LLVM 24.0.0git
AMDGPULegalizerInfo.cpp
Go to the documentation of this file.
1//===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPULegalizerInfo.h"
15
16#include "AMDGPU.h"
18#include "AMDGPUInstrInfo.h"
19#include "AMDGPUMachineInstrs.h"
20#include "AMDGPUMemoryUtils.h"
21#include "AMDGPUTargetMachine.h"
22#include "SIInstrInfo.h"
24#include "SIRegisterInfo.h"
26#include "llvm/ADT/ScopeExit.h"
37#include "llvm/IR/IntrinsicsAMDGPU.h"
38#include "llvm/IR/IntrinsicsR600.h"
40
41#define DEBUG_TYPE "amdgpu-legalinfo"
42
43using namespace llvm;
44using namespace LegalizeActions;
45using namespace LegalizeMutations;
46using namespace LegalityPredicates;
47using namespace MIPatternMatch;
48
49static constexpr unsigned MaxRegisterSize = 1024;
50
51// Round the number of elements to the next power of two elements
53 unsigned NElts = Ty.getNumElements();
54 unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts);
55 return Ty.changeElementCount(ElementCount::getFixed(Pow2NElts));
56}
57
58// Round the number of bits to the next power of two bits
60 unsigned Bits = Ty.getSizeInBits();
61 unsigned Pow2Bits = 1 << Log2_32_Ceil(Bits);
62 return LLT::scalar(Pow2Bits);
63}
64
65/// \returns true if this is an odd sized vector which should widen by adding an
66/// additional element. This is mostly to handle <3 x s16> -> <4 x s16>. This
67/// excludes s1 vectors, which should always be scalarized.
68static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
69 return [=](const LegalityQuery &Query) {
70 const LLT Ty = Query.Types[TypeIdx];
71 if (!Ty.isVector())
72 return false;
73
74 const LLT EltTy = Ty.getElementType();
75 const unsigned EltSize = EltTy.getSizeInBits();
76 return Ty.getNumElements() % 2 != 0 &&
77 EltSize > 1 && EltSize < 32 &&
78 Ty.getSizeInBits() % 32 != 0;
79 };
80}
81
82static LegalityPredicate sizeIsMultipleOf32(unsigned TypeIdx) {
83 return [=](const LegalityQuery &Query) {
84 const LLT Ty = Query.Types[TypeIdx];
85 return Ty.getSizeInBits() % 32 == 0;
86 };
87}
88
89static LegalityPredicate isWideVec16(unsigned TypeIdx) {
90 return [=](const LegalityQuery &Query) {
91 const LLT Ty = Query.Types[TypeIdx];
92 const LLT EltTy = Ty.getScalarType();
93 return EltTy.getSizeInBits() == 16 && Ty.getNumElements() > 2;
94 };
95}
96
97static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
98 return [=](const LegalityQuery &Query) {
99 const LLT Ty = Query.Types[TypeIdx];
100 const LLT EltTy = Ty.getElementType();
101 return std::pair(TypeIdx,
102 LLT::fixed_vector(Ty.getNumElements() + 1, EltTy));
103 };
104}
105
107 return [=](const LegalityQuery &Query) {
108 const LLT Ty = Query.Types[TypeIdx];
109 const LLT EltTy = Ty.getElementType();
110 unsigned Size = Ty.getSizeInBits();
111 unsigned Pieces = (Size + 63) / 64;
112 unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
113 return std::pair(TypeIdx, LLT::scalarOrVector(
114 ElementCount::getFixed(NewNumElts), EltTy));
115 };
116}
117
118// Increase the number of vector elements to reach the next multiple of 32-bit
119// type.
120static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
121 return [=](const LegalityQuery &Query) {
122 const LLT Ty = Query.Types[TypeIdx];
123
124 const LLT EltTy = Ty.getElementType();
125 const int Size = Ty.getSizeInBits();
126 const int EltSize = EltTy.getSizeInBits();
127 const int NextMul32 = (Size + 31) / 32;
128
129 assert(EltSize < 32);
130
131 const int NewNumElts = (32 * NextMul32 + EltSize - 1) / EltSize;
132 return std::pair(TypeIdx, LLT::fixed_vector(NewNumElts, EltTy));
133 };
134}
135
136// Retrieves the scalar type that's the same size as the mem desc
138 return [=](const LegalityQuery &Query) {
139 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
140 return std::make_pair(TypeIdx, LLT::integer(MemSize));
141 };
142}
143
144// Increase the number of vector elements to reach the next legal RegClass.
146 return [=](const LegalityQuery &Query) {
147 const LLT Ty = Query.Types[TypeIdx];
148 const unsigned NumElts = Ty.getNumElements();
149 const unsigned EltSize = Ty.getElementType().getSizeInBits();
150 const unsigned MaxNumElts = MaxRegisterSize / EltSize;
151
152 assert(EltSize == 32 || EltSize == 64);
153 assert(Ty.getSizeInBits() < MaxRegisterSize);
154
155 unsigned NewNumElts;
156 // Find the nearest legal RegClass that is larger than the current type.
157 for (NewNumElts = NumElts; NewNumElts < MaxNumElts; ++NewNumElts) {
158 if (SIRegisterInfo::getSGPRClassForBitWidth(NewNumElts * EltSize))
159 break;
160 }
161 return std::pair(TypeIdx,
162 LLT::fixed_vector(NewNumElts, Ty.getElementType()));
163 };
164}
165
167 if (!Ty.isVector())
168 return LLT::scalar(128);
169 const ElementCount NumElems = Ty.getElementCount();
170 return LLT::vector(NumElems, LLT::scalar(128));
171}
172
174 if (!Ty.isVector())
175 return LLT::fixed_vector(4, LLT::integer(32));
176 const unsigned NumElems = Ty.getElementCount().getFixedValue();
177 return LLT::fixed_vector(NumElems * 4, LLT::integer(32));
178}
179
181 const unsigned Size = Ty.getSizeInBits();
182
183 if (Size <= 32) {
184 // <2 x i8> -> i16
185 // <4 x i8> -> i32
186 return LLT::integer(Size);
187 }
188
189 return LLT::fixed_vector(Size / 32, LLT::integer(32));
190}
191
192static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
193 return [=](const LegalityQuery &Query) {
194 const LLT Ty = Query.Types[TypeIdx];
195 return std::pair(TypeIdx, getBitcastRegisterType(Ty));
196 };
197}
198
200 return [=](const LegalityQuery &Query) {
201 const LLT Ty = Query.Types[TypeIdx];
202 unsigned Size = Ty.getSizeInBits();
203 assert(Size % 32 == 0);
204 return std::pair(TypeIdx,
206 LLT::integer(32)));
207 };
208}
209
210static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size) {
211 return [=](const LegalityQuery &Query) {
212 const LLT QueryTy = Query.Types[TypeIdx];
213 return QueryTy.isVector() && QueryTy.getSizeInBits() < Size;
214 };
215}
216
217static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size) {
218 return [=](const LegalityQuery &Query) {
219 const LLT QueryTy = Query.Types[TypeIdx];
220 return QueryTy.isVector() && QueryTy.getSizeInBits() > Size;
221 };
222}
223
224static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
225 return [=](const LegalityQuery &Query) {
226 const LLT QueryTy = Query.Types[TypeIdx];
227 return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
228 };
229}
230
231static bool isRegisterSize(const GCNSubtarget &ST, unsigned Size) {
232 return ((ST.useRealTrue16Insts() && Size == 16) || Size % 32 == 0) &&
234}
235
237 const int EltSize = EltTy.getSizeInBits();
238 return EltSize == 16 || EltSize % 32 == 0;
239}
240
241static bool isRegisterVectorType(LLT Ty) {
242 const int EltSize = Ty.getElementType().getSizeInBits();
243 return EltSize == 32 || EltSize == 64 ||
244 (EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
245 EltSize == 128 || EltSize == 256;
246}
247
248// TODO: replace all uses of isRegisterType with isRegisterClassType
249static bool isRegisterType(const GCNSubtarget &ST, LLT Ty) {
250 if (!isRegisterSize(ST, Ty.getSizeInBits()))
251 return false;
252
253 if (Ty.isVector())
254 return isRegisterVectorType(Ty);
255
256 return true;
257}
258
259// Any combination of 32 or 64-bit elements up the maximum register size, and
260// multiples of v2s16.
262 unsigned TypeIdx) {
263 return [=, &ST](const LegalityQuery &Query) {
264 return isRegisterType(ST, Query.Types[TypeIdx]);
265 };
266}
267
268// RegisterType that doesn't have a corresponding RegClass.
269// TODO: Once `isRegisterType` is replaced with `isRegisterClassType` this
270// should be removed.
272 unsigned TypeIdx) {
273 return [=, &ST](const LegalityQuery &Query) {
274 LLT Ty = Query.Types[TypeIdx];
275 return isRegisterType(ST, Ty) &&
276 !SIRegisterInfo::getSGPRClassForBitWidth(Ty.getSizeInBits());
277 };
278}
279
280static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
281 return [=](const LegalityQuery &Query) {
282 const LLT QueryTy = Query.Types[TypeIdx];
283 if (!QueryTy.isVector())
284 return false;
285 const LLT EltTy = QueryTy.getElementType();
286 return EltTy == LLT::scalar(16) || EltTy.getSizeInBits() >= 32;
287 };
288}
289
290constexpr LLT F16 = LLT::float16();
291constexpr LLT BF16 = LLT::bfloat16();
292constexpr LLT F32 = LLT::float32();
293constexpr LLT F64 = LLT::float64();
298
299constexpr LLT S1 = LLT::scalar(1);
300constexpr LLT S8 = LLT::scalar(8);
301constexpr LLT S16 = LLT::scalar(16);
302constexpr LLT S32 = LLT::scalar(32);
303constexpr LLT S64 = LLT::scalar(64);
304constexpr LLT S96 = LLT::scalar(96);
305constexpr LLT S128 = LLT::scalar(128);
306constexpr LLT S160 = LLT::scalar(160);
307constexpr LLT S192 = LLT::scalar(192);
308constexpr LLT S224 = LLT::scalar(224);
309constexpr LLT S256 = LLT::scalar(256);
310constexpr LLT S512 = LLT::scalar(512);
311constexpr LLT S1024 = LLT::scalar(1024);
313
314constexpr LLT V2S8 = LLT::fixed_vector(2, 8);
315constexpr LLT V2S16 = LLT::fixed_vector(2, 16);
316constexpr LLT V4S16 = LLT::fixed_vector(4, 16);
317constexpr LLT V6S16 = LLT::fixed_vector(6, 16);
318constexpr LLT V8S16 = LLT::fixed_vector(8, 16);
319constexpr LLT V10S16 = LLT::fixed_vector(10, 16);
320constexpr LLT V12S16 = LLT::fixed_vector(12, 16);
321constexpr LLT V16S16 = LLT::fixed_vector(16, 16);
322
323constexpr LLT V2S32 = LLT::fixed_vector(2, 32);
324constexpr LLT V3S32 = LLT::fixed_vector(3, 32);
325constexpr LLT V4S32 = LLT::fixed_vector(4, 32);
326constexpr LLT V5S32 = LLT::fixed_vector(5, 32);
327constexpr LLT V6S32 = LLT::fixed_vector(6, 32);
328constexpr LLT V7S32 = LLT::fixed_vector(7, 32);
329constexpr LLT V8S32 = LLT::fixed_vector(8, 32);
330constexpr LLT V9S32 = LLT::fixed_vector(9, 32);
331constexpr LLT V10S32 = LLT::fixed_vector(10, 32);
332constexpr LLT V11S32 = LLT::fixed_vector(11, 32);
333constexpr LLT V12S32 = LLT::fixed_vector(12, 32);
334constexpr LLT V16S32 = LLT::fixed_vector(16, 32);
335constexpr LLT V32S32 = LLT::fixed_vector(32, 32);
336
337constexpr LLT V2S64 = LLT::fixed_vector(2, 64);
338constexpr LLT V3S64 = LLT::fixed_vector(3, 64);
339constexpr LLT V4S64 = LLT::fixed_vector(4, 64);
340constexpr LLT V5S64 = LLT::fixed_vector(5, 64);
341constexpr LLT V6S64 = LLT::fixed_vector(6, 64);
342constexpr LLT V7S64 = LLT::fixed_vector(7, 64);
343constexpr LLT V8S64 = LLT::fixed_vector(8, 64);
344constexpr LLT V16S64 = LLT::fixed_vector(16, 64);
345
346constexpr LLT V2S128 = LLT::fixed_vector(2, 128);
347constexpr LLT V4S128 = LLT::fixed_vector(4, 128);
348
349constexpr std::initializer_list<LLT> AllScalarTypes = {
351
352constexpr std::initializer_list<LLT> AllS16Vectors{
354
355constexpr std::initializer_list<LLT> AllS32Vectors = {
358
359constexpr std::initializer_list<LLT> AllS64Vectors = {
361
367
368// Checks whether a type is in the list of legal register types.
369static bool isRegisterClassType(const GCNSubtarget &ST, LLT Ty) {
370 if (Ty.isPointerOrPointerVector())
371 Ty = Ty.changeElementType(LLT::scalar(Ty.getScalarSizeInBits()));
372
375 (ST.useRealTrue16Insts() && Ty == S16) ||
377}
378
380 unsigned TypeIdx) {
381 return [&ST, TypeIdx](const LegalityQuery &Query) {
382 return isRegisterClassType(ST, Query.Types[TypeIdx]);
383 };
384}
385
386// If we have a truncating store or an extending load with a data size larger
387// than 32-bits, we need to reduce to a 32-bit type.
389 return [=](const LegalityQuery &Query) {
390 const LLT Ty = Query.Types[TypeIdx];
391 return !Ty.isVector() && Ty.getSizeInBits() > 32 &&
392 Query.MMODescrs[0].MemoryTy.getSizeInBits() < Ty.getSizeInBits();
393 };
394}
395
396// If we have a truncating store or an extending load with a data size larger
397// than 32-bits and mem location is a power of 2
399 return [=](const LegalityQuery &Query) {
400 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
401 return isWideScalarExtLoadTruncStore(TypeIdx)(Query) &&
402 isPowerOf2_64(MemSize);
403 };
404}
405
406// TODO: Should load to s16 be legal? Most loads extend to 32-bits, but we
407// handle some operations by just promoting the register during
408// selection. There are also d16 loads on GFX9+ which preserve the high bits.
409static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
410 bool IsLoad, bool IsAtomic) {
411 switch (AS) {
413 // FIXME: Private element size.
414 return ST.hasFlatScratchEnabled() ? 128 : 32;
416 return ST.useDS128() ? 128 : 64;
421 // Treat constant and global as identical. SMRD loads are sometimes usable for
422 // global loads (ideally constant address space should be eliminated)
423 // depending on the context. Legality cannot be context dependent, but
424 // RegBankSelect can split the load as necessary depending on the pointer
425 // register bank/uniformity and if the memory is invariant or not written in a
426 // kernel.
427 return IsLoad ? 512 : 128;
428 default:
429 // FIXME: Flat addresses may contextually need to be split to 32-bit parts
430 // if they may alias scratch depending on the subtarget. This needs to be
431 // moved to custom handling to use addressMayBeAccessedAsPrivate
432 return ST.hasMultiDwordFlatScratchAddressing() || IsAtomic ? 128 : 32;
433 }
434}
435
436static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
437 const LegalityQuery &Query) {
438 const LLT Ty = Query.Types[0];
439
440 // Handle G_LOAD, G_ZEXTLOAD, G_SEXTLOAD
441 const bool IsLoad = Query.Opcode != AMDGPU::G_STORE;
442
443 unsigned RegSize = Ty.getSizeInBits();
444 uint64_t MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
445 uint64_t AlignBits = Query.MMODescrs[0].AlignInBits;
446 unsigned AS = Query.Types[1].getAddressSpace();
447
448 // All of these need to be custom lowered to cast the pointer operand.
450 return false;
451
452 // Never plain-legal; custom-lowered to G_AMDGPU_REG_LOAD/STORE.
453 if (AS == AMDGPUAS::VGPR)
454 return false;
455
456 // Do not handle extending vector loads.
457 if (Ty.isVector() && MemSize != RegSize)
458 return false;
459
460 // TODO: We should be able to widen loads if the alignment is high enough, but
461 // we also need to modify the memory access size.
462#if 0
463 // Accept widening loads based on alignment.
464 if (IsLoad && MemSize < Size)
465 MemSize = std::max(MemSize, Align);
466#endif
467
468 // Only 1-byte and 2-byte to 32-bit extloads are valid.
469 if (MemSize != RegSize && RegSize != 32)
470 return false;
471
472 if (IsLoad && RegSize == 16 && !ST.d16PreservesUnusedBits())
473 return false;
474
475 if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
476 Query.MMODescrs[0].Ordering !=
478 return false;
479
480 switch (MemSize) {
481 case 8:
482 case 16:
483 case 32:
484 case 64:
485 case 128:
486 break;
487 case 96:
488 if (!ST.hasDwordx3LoadStores())
489 return false;
490 break;
491 case 256:
492 case 512:
493 // These may contextually need to be broken down.
494 break;
495 default:
496 return false;
497 }
498
499 assert(RegSize >= MemSize);
500
501 if (AlignBits < MemSize) {
502 const SITargetLowering *TLI = ST.getTargetLowering();
503 if (!TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS,
504 Align(AlignBits / 8)))
505 return false;
506 }
507
508 return true;
509}
510
511// The newer buffer intrinsic forms take their resource arguments as
512// pointers in address space 8, aka s128 values. However, in order to not break
513// SelectionDAG, the underlying operations have to continue to take v4i32
514// arguments. Therefore, we convert resource pointers - or vectors of them
515// to integer values here.
516static bool hasBufferRsrcWorkaround(const LLT Ty) {
517 if (Ty.isPointer() && Ty.getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
518 return true;
519 if (Ty.isVector()) {
520 const LLT ElemTy = Ty.getElementType();
522 }
523 return false;
524}
525
526// The current selector can't handle <6 x s16>, <8 x s16>, s96, s128 etc, so
527// workaround this. Eventually it should ignore the type for loads and only care
528// about the size. Return true in cases where we will workaround this for now by
529// bitcasting.
530static bool loadStoreBitcastWorkaround(const GCNSubtarget &ST, const LLT Ty) {
531 if (ST.getCLOpts().global_isel_new_legality)
532 return false;
533
534 const unsigned Size = Ty.getSizeInBits();
535 if (Ty.isPointerVector())
536 return true;
537 if (Size <= 64)
538 return false;
539 // Address space 8 pointers get their own workaround.
541 return false;
542 if (!Ty.isVector())
543 return true;
544
545 unsigned EltSize = Ty.getScalarSizeInBits();
546 return EltSize != 32 && EltSize != 64;
547}
548
549static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
550 const LLT Ty = Query.Types[0];
551 return isRegisterType(ST, Ty) && isLoadStoreSizeLegal(ST, Query) &&
553}
554
555// Whole-dword accesses only, for now.
556static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize) {
557 return MemSize == ValSize && MemSize <= MaxRegisterSize &&
559}
560
561/// Return true if a load or store of the type should be lowered with a bitcast
562/// to a different type.
563static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
564 const LLT MemTy) {
565 const unsigned MemSizeInBits = MemTy.getSizeInBits();
566 const unsigned Size = Ty.getSizeInBits();
567 if (Size != MemSizeInBits)
568 return Size <= 32 && Ty.isVector();
569
570 if (loadStoreBitcastWorkaround(ST, Ty) && isRegisterType(ST, Ty))
571 return true;
572
573 // Don't try to handle bitcasting vector ext loads for now.
574 return Ty.isVector() && (!MemTy.isVector() || MemTy == Ty) &&
575 (Size <= 32 || isRegisterSize(ST, Size)) &&
576 !isRegisterVectorElementType(Ty.getElementType());
577}
578
579/// Return true if we should legalize a load by widening an odd sized memory
580/// access up to the alignment. Note this case when the memory access itself
581/// changes, not the size of the result register.
582static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy,
583 uint64_t AlignInBits, unsigned AddrSpace,
584 unsigned Opcode) {
585 unsigned SizeInBits = MemoryTy.getSizeInBits();
586 // We don't want to widen cases that are naturally legal.
587 if (isPowerOf2_32(SizeInBits))
588 return false;
589
590 // If we have 96-bit memory operations, we shouldn't touch them. Note we may
591 // end up widening these for a scalar load during RegBankSelect, if we don't
592 // have 96-bit scalar loads.
593 if (SizeInBits == 96 && ST.hasDwordx3LoadStores())
594 return false;
595
596 if (SizeInBits >= maxSizeForAddrSpace(ST, AddrSpace, Opcode, false))
597 return false;
598
599 // A load is known dereferenceable up to the alignment, so it's legal to widen
600 // to it.
601 //
602 // TODO: Could check dereferenceable for less aligned cases.
603 unsigned RoundedSize = NextPowerOf2(SizeInBits);
604 if (AlignInBits < RoundedSize)
605 return false;
606
607 // Do not widen if it would introduce a slow unaligned load.
608 const SITargetLowering *TLI = ST.getTargetLowering();
609 unsigned Fast = 0;
610 return TLI->allowsMisalignedMemoryAccessesImpl(
611 RoundedSize, AddrSpace, Align(AlignInBits / 8),
613 Fast;
614}
615
616static bool shouldWidenLoad(const GCNSubtarget &ST, const LegalityQuery &Query,
617 unsigned Opcode) {
618 if (Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic)
619 return false;
620
621 return shouldWidenLoad(ST, Query.MMODescrs[0].MemoryTy,
622 Query.MMODescrs[0].AlignInBits,
623 Query.Types[1].getAddressSpace(), Opcode);
624}
625
626/// Mutates IR (typicaly a load instruction) to use a <4 x s32> as the initial
627/// type of the operand `idx` and then to transform it to a `p8` via bitcasts
628/// and inttoptr. In addition, handle vectors of p8. Returns the new type.
630 MachineRegisterInfo &MRI, unsigned Idx) {
631 MachineOperand &MO = MI.getOperand(Idx);
632
633 const LLT PointerTy = MRI.getType(MO.getReg());
634
635 // Paranoidly prevent us from doing this multiple times.
637 return PointerTy;
638
639 const LLT ScalarTy = getBufferRsrcScalarType(PointerTy);
640 const LLT VectorTy = getBufferRsrcRegisterType(PointerTy);
641 if (!PointerTy.isVector()) {
642 // Happy path: (4 x s32) -> (s32, s32, s32, s32) -> (p8)
643 const unsigned NumParts = PointerTy.getSizeInBits() / 32;
644 const LLT I32 = LLT::integer(32);
645
646 Register VectorReg = MRI.createGenericVirtualRegister(VectorTy);
647 std::array<Register, 4> VectorElems;
648 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
649 for (unsigned I = 0; I < NumParts; ++I)
650 VectorElems[I] =
651 B.buildExtractVectorElementConstant(I32, VectorReg, I).getReg(0);
652 B.buildMergeValues(MO, VectorElems);
653 MO.setReg(VectorReg);
654 return VectorTy;
655 }
656 Register BitcastReg = MRI.createGenericVirtualRegister(VectorTy);
657 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
658 auto Scalar = B.buildBitcast(ScalarTy, BitcastReg);
659 B.buildIntToPtr(MO, Scalar);
660 MO.setReg(BitcastReg);
661
662 return VectorTy;
663}
664
665/// Cast a buffer resource (an address space 8 pointer) into a 4xi32, which is
666/// the form in which the value must be in order to be passed to the low-level
667/// representations used for MUBUF/MTBUF intrinsics. This is a hack, which is
668/// needed in order to account for the fact that we can't define a register
669/// class for s128 without breaking SelectionDAG.
671 MachineRegisterInfo &MRI = *B.getMRI();
672 const LLT PointerTy = MRI.getType(Pointer);
673 const LLT ScalarTy = getBufferRsrcScalarType(PointerTy);
674 const LLT VectorTy = getBufferRsrcRegisterType(PointerTy);
675
676 if (!PointerTy.isVector()) {
677 // Special case: p8 -> (s32, s32, s32, s32) -> (4xs32)
678 SmallVector<Register, 4> PointerParts;
679 const unsigned NumParts = PointerTy.getSizeInBits() / 32;
680 auto Unmerged = B.buildUnmerge(LLT::integer(32), Pointer);
681 for (unsigned I = 0; I < NumParts; ++I)
682 PointerParts.push_back(Unmerged.getReg(I));
683 return B.buildBuildVector(VectorTy, PointerParts).getReg(0);
684 }
685 Register Scalar = B.buildPtrToInt(ScalarTy, Pointer).getReg(0);
686 return B.buildBitcast(VectorTy, Scalar).getReg(0);
687}
688
690 unsigned Idx) {
691 MachineOperand &MO = MI.getOperand(Idx);
692
693 const LLT PointerTy = B.getMRI()->getType(MO.getReg());
694 // Paranoidly prevent us from doing this multiple times.
696 return;
698}
699
701 const GCNTargetMachine &TM)
702 : ST(ST_) {
703 using namespace TargetOpcode;
704
705 const LLT GlobalPtr = LLT::pointer(AMDGPUAS::GLOBAL_ADDRESS, 64);
706 const LLT ConstantPtr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
707 const LLT Constant32Ptr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS_32BIT, 32);
708 const LLT LocalPtr = LLT::pointer(AMDGPUAS::LOCAL_ADDRESS, 32);
709 const LLT RegionPtr = LLT::pointer(AMDGPUAS::REGION_ADDRESS, 32);
710 const LLT FlatPtr = LLT::pointer(AMDGPUAS::FLAT_ADDRESS, 64);
711 const LLT PrivatePtr = LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32);
712 const LLT VGPRPtr = LLT::pointer(AMDGPUAS::VGPR, 32);
713 const LLT BufferFatPtr = LLT::pointer(AMDGPUAS::BUFFER_FAT_POINTER, 160);
714 const LLT RsrcPtr = LLT::pointer(AMDGPUAS::BUFFER_RESOURCE, 128);
715 const LLT BufferStridedPtr =
717
718 const LLT CodePtr = FlatPtr;
719
720 const std::initializer_list<LLT> AddrSpaces64 = {
721 GlobalPtr, ConstantPtr, FlatPtr
722 };
723
724 const std::initializer_list<LLT> AddrSpaces32 = {
725 LocalPtr, PrivatePtr, Constant32Ptr, RegionPtr
726 };
727
728 const std::initializer_list<LLT> AddrSpaces128 = {RsrcPtr};
729
730 const std::initializer_list<LLT> FPTypesBase = {F32, F64};
731 const std::initializer_list<LLT> FPTypes16 = {F32, F64, F16};
732 const std::initializer_list<LLT> FPTypesPK16 = {F32, F64, F16, V2F16};
733
734 const LLT I1 = LLT::integer(1);
735 const LLT I16 = LLT::integer(16);
736 const LLT I32 = LLT::integer(32);
737 const LLT I64 = LLT::integer(64);
738 const LLT V2I16 = LLT::fixed_vector(2, I16);
739
741
742 // s1 for VCC branches, s32 for SCC branches.
744
745 // TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
746 // elements for v3s16
749 .legalFor(AllS32Vectors)
751 .legalFor(AddrSpaces64)
752 .legalFor(AddrSpaces32)
753 .legalFor(AddrSpaces128)
754 .legalIf(isPointer(0))
755 .clampScalar(0, S16, S256)
757 .clampMaxNumElements(0, S32, 16)
759 .scalarize(0);
760
761 if (ST.hasVOP3PInsts() && ST.hasAddNoCarryInsts() && ST.hasIntClamp()) {
762 // Full set of gfx9 features.
763 if (ST.hasAnyPackedU64Ops()) {
764 getActionDefinitionsBuilder({G_ADD, G_SUB})
765 .legalFor({S64, S32, S16, V2S16, V2S64})
766 .clampMaxNumElementsStrict(0, S16, 2)
768 .scalarize(0)
769 .minScalar(0, S16)
771 .maxScalar(0, S32);
772 } else if (ST.hasScalarAddSub64()) {
773 getActionDefinitionsBuilder({G_ADD, G_SUB})
774 .legalFor({S64, S32, S16, V2S16})
775 .clampMaxNumElementsStrict(0, S16, 2)
776 .scalarize(0)
777 .minScalar(0, S16)
779 .maxScalar(0, S32);
780 } else {
781 getActionDefinitionsBuilder({G_ADD, G_SUB})
782 .legalFor({S32, S16, V2S16})
783 .clampMaxNumElementsStrict(0, S16, 2)
784 .scalarize(0)
785 .minScalar(0, S16)
787 .maxScalar(0, S32);
788 }
789
790 if (ST.hasScalarSMulU64()) {
792 .legalFor({S64, S32, S16, V2S16})
793 .clampMaxNumElementsStrict(0, S16, 2)
794 .scalarize(0)
795 .minScalar(0, S16)
797 .custom();
798 } else {
800 .legalFor({S32, S16, V2S16})
801 .clampMaxNumElementsStrict(0, S16, 2)
802 .scalarize(0)
803 .minScalar(0, S16)
805 .custom();
806 }
807 assert(ST.hasMad64_32());
808
809 getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT})
810 .legalFor({S32, S16, V2S16}) // Clamp modifier
811 .minScalarOrElt(0, S16)
813 .scalarize(0)
815 .lower();
816 } else if (ST.has16BitInsts()) {
817 getActionDefinitionsBuilder({G_ADD, G_SUB})
818 .legalFor({S32, S16})
819 .minScalar(0, S16)
821 .maxScalar(0, S32)
822 .scalarize(0);
823
825 .legalFor({S32, S16})
826 .scalarize(0)
827 .minScalar(0, S16)
829 .custom();
830 assert(ST.hasMad64_32());
831
832 // Technically the saturating operations require clamp bit support, but this
833 // was introduced at the same time as 16-bit operations.
834 getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
835 .legalFor({S32, S16}) // Clamp modifier
836 .minScalar(0, S16)
837 .scalarize(0)
839 .lower();
840
841 // We're just lowering this, but it helps get a better result to try to
842 // coerce to the desired type first.
843 getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
844 .minScalar(0, S16)
845 .scalarize(0)
846 .lower();
847 } else {
848 getActionDefinitionsBuilder({G_ADD, G_SUB})
849 .legalFor({S32})
850 .widenScalarToNextMultipleOf(0, 32)
851 .clampScalar(0, S32, S32)
852 .scalarize(0);
853
854 auto &Mul = getActionDefinitionsBuilder(G_MUL)
855 .legalFor({S32})
856 .scalarize(0)
857 .minScalar(0, S32)
859
860 if (ST.hasMad64_32())
861 Mul.custom();
862 else
863 Mul.maxScalar(0, S32);
864
865 if (ST.hasIntClamp()) {
866 getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
867 .legalFor({S32}) // Clamp modifier.
868 .scalarize(0)
870 .lower();
871 } else {
872 // Clamp bit support was added in VI, along with 16-bit operations.
873 getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
874 .minScalar(0, S32)
875 .scalarize(0)
876 .lower();
877 }
878
879 // FIXME: DAG expansion gets better results. The widening uses the smaller
880 // range values and goes for the min/max lowering directly.
881 getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
882 .minScalar(0, S32)
883 .scalarize(0)
884 .lower();
885 }
886
888 {G_SDIV, G_UDIV, G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
889 .customFor({S32, S64})
890 .clampScalar(0, S32, S64)
892 .scalarize(0);
893
894 auto &Mulh = getActionDefinitionsBuilder({G_UMULH, G_SMULH})
895 .legalFor({S32})
896 .maxScalar(0, S32);
897
898 if (ST.hasVOP3PInsts()) {
899 Mulh
900 .clampMaxNumElements(0, S8, 2)
901 .lowerFor({V2S8});
902 }
903
904 Mulh
905 .scalarize(0)
906 .lower();
907
908 // Report legal for any types we can handle anywhere. For the cases only legal
909 // on the SALU, RegBankSelect will be able to re-legalize.
910 getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
911 .legalFor({S32, S1, S64, V2S32, S16, V2S16, V4S16})
912 .clampScalar(0, S32, S64)
918 .scalarize(0);
919
921 {G_UADDO, G_USUBO, G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
922 .legalFor({{S32, S1}, {S32, S32}})
923 .clampScalar(0, S32, S32)
924 .scalarize(0);
925
927 // Don't worry about the size constraint.
929 .widenScalarIf(all(typeInSet(0, {I16, F16, BF16}), isScalar(1)),
930 changeTo(0, LLT::integer(32)))
931 .widenScalarIf(all(isScalar(0), typeInSet(1, {I16, F16, BF16})),
932 changeTo(1, LLT::integer(32)))
933 .lower();
934
936 .legalFor({S1, S32, S64, S16, GlobalPtr,
937 LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
938 .legalIf(isPointer(0))
939 .clampScalar(0, S32, S64)
941
943
944 getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
945 .legalIf(isRegisterClassType(ST, 0))
946 // s1 and s16 are special cases because they have legal operations on
947 // them, but don't really occupy registers in the normal way.
948 .legalFor({S1, S16})
949 .clampNumElements(0, V16S32, V32S32)
953 .clampMaxNumElements(0, S32, 16);
954
955 // An alloca in the VGPR ("as memory") address space is diagnosed; see
956 // legalizeFrameIndexVGPR.
957 getActionDefinitionsBuilder(G_FRAME_INDEX)
958 .legalFor({PrivatePtr})
959 .customFor({VGPRPtr});
960
961 // If the amount is divergent, we have to do a wave reduction to get the
962 // maximum value, so this is expanded during RegBankSelect.
963 getActionDefinitionsBuilder(G_DYN_STACKALLOC)
964 .legalFor({{PrivatePtr, S32}});
965
966 getActionDefinitionsBuilder(G_STACKSAVE)
967 .customFor({PrivatePtr});
968 getActionDefinitionsBuilder(G_STACKRESTORE)
969 .legalFor({PrivatePtr});
970
971 getActionDefinitionsBuilder(G_WRITE_REGISTER).legalFor({S32, S64});
972
973 getActionDefinitionsBuilder({G_GET_FPENV, G_SET_FPENV}).customFor({S64});
974
975 getActionDefinitionsBuilder({G_GET_ROUNDING, G_SET_ROUNDING}).legalFor({S32});
976
977 getActionDefinitionsBuilder(G_GLOBAL_VALUE)
978 .customIf(typeIsNot(0, PrivatePtr));
979
980 getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({CodePtr});
981
982 auto &FPOpActions =
983 getActionDefinitionsBuilder({G_FADD, G_FMUL, G_FMA}).legalFor({F32, F64});
984 auto &FCanonicalizeActions =
985 getActionDefinitionsBuilder(G_FCANONICALIZE).legalFor({F32, F64});
986 auto &StrictFPOpActions =
987 getActionDefinitionsBuilder({G_STRICT_FADD, G_STRICT_FMUL, G_STRICT_FMA})
988 .legalFor({F32, F64});
989 auto &TrigActions =
990 getActionDefinitionsBuilder({G_FSIN, G_FCOS}).customFor({F32, F64});
991 auto &FDIVActions = getActionDefinitionsBuilder(G_FDIV).customFor({F32, F64});
992
993 if (ST.has16BitInsts()) {
994 if (ST.hasVOP3PInsts()) {
995 FPOpActions.legalFor({F16, V2F16});
996 FCanonicalizeActions.legalFor({F16, V2F16});
997 StrictFPOpActions.legalFor({F16, V2F16});
998 } else {
999 FPOpActions.legalFor({F16});
1000 FCanonicalizeActions.legalFor({F16});
1001 StrictFPOpActions.legalFor({F16});
1002 }
1003
1004 TrigActions.customFor({F16});
1005 FDIVActions.customFor({F16});
1006 }
1007
1008 if (ST.hasBF16PackedInsts()) {
1009 // Promote scalar bf16 operations to v2bf16 (packed) operations
1010 FPOpActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
1011 .legalFor({V2BF16})
1012 .clampMaxNumElementsStrict(0, BF16, 2);
1013 FCanonicalizeActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
1014 .legalFor({V2BF16})
1015 .clampMaxNumElementsStrict(0, BF16, 2);
1016 StrictFPOpActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
1017 .legalFor({V2BF16})
1018 .clampMaxNumElementsStrict(0, BF16, 2);
1019 } else {
1020 FPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
1021 FCanonicalizeActions.widenScalarFor({BF16}, changeElementTo(0, F32));
1022 }
1023
1024 if (ST.hasAnyPackedFP32Ops()) {
1025 FPOpActions.legalFor({V2F32});
1026 FCanonicalizeActions.legalFor({V2F32});
1027 StrictFPOpActions.legalFor({V2F32});
1028 FPOpActions.clampMaxNumElementsStrict(0, F32, 2);
1029 FCanonicalizeActions.clampMaxNumElementsStrict(0, F32, 2);
1030 StrictFPOpActions.clampMaxNumElementsStrict(0, F32, 2);
1031 }
1032
1033 if (ST.hasAnyPackedFP64Ops()) {
1034 FPOpActions.legalFor({V2F64});
1035 FCanonicalizeActions.legalFor({V2F64});
1036 StrictFPOpActions.legalFor({V2F64});
1037 FPOpActions.clampMaxNumElementsStrict(0, F64, 2);
1038 FCanonicalizeActions.clampMaxNumElementsStrict(0, F64, 2);
1039 StrictFPOpActions.clampMaxNumElementsStrict(0, F64, 2);
1040 }
1041
1042 auto &MinNumMaxNumIeee =
1043 getActionDefinitionsBuilder({G_FMINNUM_IEEE, G_FMAXNUM_IEEE});
1044 auto &MinNumMaxNum = getActionDefinitionsBuilder(
1045 {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});
1046
1047 MinNumMaxNumIeee.legalFor(!ST.hasIEEEMinimumMaximumInsts(), {F32, F64});
1048 MinNumMaxNum.legalFor(ST.hasIEEEMinimumMaximumInsts(), {F32, F64})
1049 .customFor({F32, F64});
1050
1051 if (ST.has16BitInsts()) {
1052 MinNumMaxNumIeee.legalFor(!ST.hasIEEEMinimumMaximumInsts(), {F16});
1053 MinNumMaxNum.legalFor(ST.hasIEEEMinimumMaximumInsts(), {F16})
1054 .customFor({F16});
1055 }
1056
1057 // V2F16
1058 if (ST.hasVOP3PInsts()) {
1059 MinNumMaxNumIeee.legalFor(!ST.hasIEEEMinimumMaximumInsts(), {V2F16})
1060 .moreElementsIf(all(elementTypeIs(0, F16), isSmallOddVector(0)),
1061 oneMoreElement(0))
1062 .clampMaxNumElements(0, F16, 2);
1063 MinNumMaxNum.legalFor(ST.hasIEEEMinimumMaximumInsts(), {V2F16})
1064 .customFor({V2F16})
1065 .moreElementsIf(all(elementTypeIs(0, F16), isSmallOddVector(0)),
1066 oneMoreElement(0))
1067 .clampMaxNumElements(0, F16, 2);
1068 }
1069
1070 // V2F64
1071 if (ST.hasAnyPackedFP64Ops()) {
1072 MinNumMaxNum.legalFor(ST.hasIEEEMinimumMaximumInsts(), {V2F64})
1073 .customFor({V2F64})
1074 .moreElementsIf(all(elementTypeIs(0, F64), isSmallOddVector(0)),
1075 oneMoreElement(0))
1076 .clampMaxNumElements(0, F64, 2);
1077 }
1078
1079 // V2BF16
1080 if (ST.hasBF16PackedInsts()) {
1081 MinNumMaxNumIeee.legalFor(!ST.hasIEEEMinimumMaximumInsts(), {V2BF16})
1082 .clampMaxNumElementsStrict(0, BF16, 2);
1083
1084 MinNumMaxNum.legalFor(ST.hasIEEEMinimumMaximumInsts(), {V2BF16})
1085 .moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
1086 .customFor({V2BF16})
1087 .clampMaxNumElementsStrict(0, BF16, 2);
1088 } else {
1089 MinNumMaxNum.widenScalarFor({BF16}, changeElementTo(0, F32));
1090 }
1091
1092 MinNumMaxNumIeee.scalarize(0);
1093 MinNumMaxNum.scalarize(0);
1094
1095 if (!ST.has16BitInsts()) {
1096 MinNumMaxNumIeee.minScalar(0, F32);
1097 MinNumMaxNum.minScalar(0, F32);
1098 }
1099
1100 if (ST.hasVOP3PInsts()) {
1101 FPOpActions.clampMaxNumElementsStrict(0, F16, 2);
1102 FCanonicalizeActions.clampMaxNumElementsStrict(0, F16, 2);
1103 StrictFPOpActions.clampMaxNumElementsStrict(0, F16, 2);
1104 }
1105
1106 FPOpActions.scalarize(0);
1107 FCanonicalizeActions.scalarize(0);
1108 StrictFPOpActions.scalarize(0);
1109 TrigActions.scalarize(0);
1110 FDIVActions.scalarize(0);
1111 if (!ST.has16BitInsts()) {
1112 FPOpActions.widenScalarFor({F16}, changeElementTo(0, F32));
1113 FCanonicalizeActions.widenScalarFor({F16}, changeElementTo(0, F32));
1114 StrictFPOpActions.widenScalarFor({F16}, changeElementTo(0, F32));
1115 TrigActions.widenScalarFor({F16}, changeElementTo(0, F32));
1116 FDIVActions.widenScalarFor({F16}, changeElementTo(0, F32));
1117 }
1118
1119 auto &FNegAbs = getActionDefinitionsBuilder({G_FNEG, G_FABS});
1120 FNegAbs.legalFor(FPTypesPK16)
1121 .legalFor({BF16, V2BF16})
1122 .legalFor(ST.hasAnyPackedFP32Ops(), {V2F32})
1125 if (ST.hasAnyPackedFP32Ops())
1126 FNegAbs.clampMaxNumElementsStrict(0, F32, 2);
1127 FNegAbs.scalarize(0);
1128
1129 if (ST.has16BitInsts()) {
1131 .legalFor({F16})
1132 .legalFor(ST.hasBF16TransInsts(), {BF16})
1133 .customFor({F32, F64})
1134 .scalarize(0)
1136 .unsupported();
1138 .legalFor({F32, F64, F16})
1139 .scalarize(0);
1140
1141 getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
1142 .legalFor({{F32, I32}, {F64, I32}, {F16, I16}})
1143 .scalarize(0)
1144 .maxScalarIf(typeIs(0, F16), 1, I16)
1145 .clampScalar(1, I32, I32)
1146 .lower();
1147
1149 .customFor({{F32, I32}, {F64, I32}, {F16, I16}, {F16, I32}})
1150 .scalarize(0)
1151 .lower();
1152
1154 .lowerFor({F16, F32, F64})
1155 .scalarize(0)
1156 .lower();
1157 } else {
1159 .customFor({F32, F64, F16})
1160 .scalarize(0)
1162 .unsupported();
1163
1164 if (ST.hasFractBug()) {
1166 .customFor({F64})
1167 .legalFor({F32, F64})
1168 .scalarize(0)
1169 .minScalar(0, F32);
1170 } else {
1172 .legalFor({F32, F64})
1173 .scalarize(0)
1174 .minScalar(0, F32);
1175 }
1176
1177 getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
1178 .legalFor({{F32, I32}, {F64, I32}})
1179 .scalarize(0)
1180 .minScalar(0, F32)
1181 .clampScalar(1, I32, I32)
1182 .lower();
1183
1185 .customFor({{F32, I32}, {F64, I32}})
1186 .scalarize(0)
1187 .minScalar(0, F32)
1188 .clampScalar(1, I32, I32)
1189 .lower();
1190
1192 .lowerFor({F32, F64})
1193 .scalarize(0)
1194 .lower();
1195 }
1196
1197 auto &FPTruncActions = getActionDefinitionsBuilder(G_FPTRUNC);
1198 if (ST.hasCvtPkF16F32Inst()) {
1199 FPTruncActions.legalFor({{F32, F64}, {F16, F32}, {V2F16, V2F32}})
1200 .clampMaxNumElements(0, F16, 2);
1201 } else {
1202 FPTruncActions.legalFor({{F32, F64}, {F16, F32}});
1203 }
1204 FPTruncActions.lowerFor({{BF16, F32}, {BF16, F64}, {F16, F64}}).scalarize(0);
1205
1207 .legalFor({{F64, F32}, {F32, F16}})
1208 .narrowScalarFor({{F64, F16}}, changeElementSizeTo(0, F32))
1209 .lowerFor({{F32, BF16}, {F64, BF16}})
1210 .scalarize(0);
1211
1212 auto &FSubActions = getActionDefinitionsBuilder({G_FSUB, G_STRICT_FSUB});
1213 if (ST.has16BitInsts()) {
1214 FSubActions
1215 // Use actual fsub instruction
1216 .legalFor({F32, F16})
1217 // Must use fadd + fneg
1218 .lowerFor({F64, V2F16});
1219 } else {
1220 FSubActions
1221 // Use actual fsub instruction
1222 .legalFor({F32})
1223 // Must use fadd + fneg
1224 .lowerFor({F64, F16, V2F16});
1225 }
1226
1227 if (ST.hasBF16PackedInsts()) {
1228 FSubActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
1229 .lowerFor({V2BF16})
1230 .clampMaxNumElementsStrict(0, BF16, 2);
1231 } else {
1232 FSubActions.widenScalarFor({BF16}, changeElementTo(0, F32));
1233 }
1234
1235 if (ST.hasAnyPackedFP32Ops())
1236 FSubActions.lowerFor({V2F32}).clampMaxNumElements(0, F32, 2);
1237
1238 FSubActions.clampMaxNumElements(0, F16, 2).scalarize(0).clampScalar(0, F32,
1239 F64);
1240
1241 // Whether this is legal depends on the floating point mode for the function.
1242 auto &FMad = getActionDefinitionsBuilder(G_FMAD);
1243 if (ST.hasMadF16() && ST.hasMadMacF32Insts())
1244 FMad.customFor({F32, F16});
1245 else if (ST.hasMadMacF32Insts())
1246 FMad.customFor({F32});
1247 else if (ST.hasMadF16())
1248 FMad.customFor({F16});
1249 FMad.scalarize(0)
1250 .lower();
1251
1252 auto &FRem = getActionDefinitionsBuilder(G_FREM);
1253 if (ST.has16BitInsts()) {
1254 FRem.customFor({F16, F32, F64});
1255 } else {
1256 FRem.minScalar(0, F32).customFor({F32, F64});
1257 }
1258 FRem.scalarize(0);
1259
1260 // TODO: Do we need to clamp maximum bitwidth?
1262 .legalIf(isScalar(0))
1263 .legalFor({{V2S16, V2S32}})
1264 .clampMaxNumElements(0, S16, 2)
1265 // Avoid scalarizing in cases that should be truly illegal. In unresolvable
1266 // situations (like an invalid implicit use), we don't want to infinite loop
1267 // in the legalizer.
1269 .alwaysLegal();
1270
1271 getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
1272 .legalFor({{S64, S32}, {S32, S16}, {S64, S16},
1273 {S32, S1}, {S64, S1}, {S16, S1}})
1274 .scalarize(0)
1275 .clampScalar(0, S32, S64)
1276 .widenScalarToNextPow2(1, 32);
1277
1278 // TODO: Split s1->s64 during regbankselect for VALU.
1279 auto &IToFP = getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
1280 .legalFor({{F32, I32}, {F64, I32}})
1281 .widenScalarFor({{F16, I32}}, changeElementSizeTo(0, F32))
1282 .lowerIf(typeIs(1, I1))
1283 .customFor({{F32, I64}, {F64, I64}});
1284 if (ST.has16BitInsts())
1285 IToFP.legalFor({{F16, I16}});
1286 IToFP.clampScalar(1, I32, I64)
1287 .minScalar(0, F32)
1288 .scalarize(0)
1290
1291 auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
1292 .legalFor({{I32, F32}, {I32, F64}})
1293 .customFor({{I64, F32}, {I64, F64}})
1294 .widenScalarFor({{I32, F16}}, changeElementSizeTo(1, F32))
1295 .narrowScalarFor({{I64, F16}}, changeElementSizeTo(0, I32))
1296 .widenScalarFor({{I16, BF16}, {I32, BF16}, {I64, BF16}},
1297 changeElementTo(1, F32));
1298 if (ST.has16BitInsts())
1299 FPToI.legalFor({{I16, F16}});
1300 else
1301 FPToI.minScalar(1, F32);
1302
1303 FPToI.minScalar(0, I32).widenScalarToNextPow2(0, 32).scalarize(0).lower();
1304
1305 // clang-format off
1306 auto &FPToISat = getActionDefinitionsBuilder({G_FPTOSI_SAT, G_FPTOUI_SAT})
1307 .legalFor({{I32, F32}, {I32, F64}, {I16, F32}})
1308 .legalFor(ST.has16BitInsts(), {{I16, F16}})
1309 .legalFor(ST.hasVCvtPkIU16F32(), {{V2I16, V2F32}})
1310 .narrowScalarFor({{I64, F16}}, changeElementSizeTo(0, I32));
1311
1312 // If available, widen width <16 to i16, intead of i32 so v_cvt_i16/u16_f16 can be used.
1313 if (ST.has16BitInsts())
1314 FPToISat.minScalarIf(typeIs(1, F16), 0, I16);
1315
1316 if (ST.hasVCvtPkIU16F32())
1317 FPToISat.clampMaxNumElements(0, I16, 2);
1318
1319 FPToISat.minScalar(1, F32);
1320 FPToISat.minScalar(0, I32)
1321 .widenScalarToNextPow2(0, 32)
1322 .scalarize(0)
1323 .lower();
1324 // clang-format on
1325
1326 getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
1327 .clampScalar(0, I16, I64)
1328 .scalarize(0)
1329 .lower();
1330
1331 getActionDefinitionsBuilder(G_INTRINSIC_FPTRUNC_ROUND)
1332 .legalFor({F16, F32})
1333 .scalarize(0)
1334 .lower();
1335
1336 // Lower G_FNEARBYINT and G_FRINT into G_INTRINSIC_ROUNDEVEN
1337 getActionDefinitionsBuilder({G_INTRINSIC_ROUND, G_FRINT, G_FNEARBYINT})
1338 .scalarize(0)
1339 .lower();
1340
1341 getActionDefinitionsBuilder({G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
1342 .clampScalar(0, I16, I64)
1343 .scalarize(0)
1344 .lower();
1345
1346 auto &RoundingActions = getActionDefinitionsBuilder(
1347 {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN});
1348 if (ST.has16BitInsts())
1349 RoundingActions.legalFor({F16, F32, F64});
1350 else if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS)
1351 RoundingActions.legalFor({F32, F64});
1352 else
1353 RoundingActions.legalFor({F32}).customFor({F64});
1354
1355 RoundingActions.scalarize(0);
1356 if (!ST.has16BitInsts())
1357 RoundingActions.minScalar(0, F32);
1358
1359 getActionDefinitionsBuilder(G_PTR_ADD)
1360 .unsupportedFor({BufferFatPtr, BufferStridedPtr, RsrcPtr})
1361 .legalIf(all(isPointer(0), sameSize(0, 1)))
1362 .scalarize(0)
1363 .scalarSameSizeAs(1, 0);
1364
1365 getActionDefinitionsBuilder(G_PTRMASK)
1366 .legalIf(all(sameSize(0, 1), typeInSet(1, {S64, S32})))
1367 .scalarSameSizeAs(1, 0)
1368 .scalarize(0);
1369
1370 auto &CmpBuilder =
1371 getActionDefinitionsBuilder(G_ICMP)
1372 // The compare output type differs based on the register bank of the output,
1373 // so make both s1 and s32 legal.
1374 //
1375 // Scalar compares producing output in scc will be promoted to s32, as that
1376 // is the allocatable register type that will be needed for the copy from
1377 // scc. This will be promoted during RegBankSelect, and we assume something
1378 // before that won't try to use s32 result types.
1379 //
1380 // Vector compares producing an output in vcc/SGPR will use s1 in VCC reg
1381 // bank.
1383 {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
1384 .legalForCartesianProduct(
1385 {S32}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr});
1386 if (ST.has16BitInsts()) {
1387 CmpBuilder.legalFor({{S1, S16}});
1388 }
1389
1390 CmpBuilder
1392 .clampScalar(1, S32, S64)
1393 .scalarize(0)
1394 .legalIf(all(typeInSet(0, {S1, S32}), isPointer(1)));
1395
1396 getActionDefinitionsBuilder({G_SCMP, G_UCMP}).lower();
1397
1398 auto &FCmpBuilder =
1399 getActionDefinitionsBuilder(G_FCMP).legalForCartesianProduct(
1400 {I1}, ST.has16BitInsts() ? FPTypes16 : FPTypesBase);
1401
1402 if (ST.hasSALUFloatInsts())
1403 FCmpBuilder.legalForCartesianProduct({I32}, {F16, F32});
1404
1405 FCmpBuilder.widenScalarFor({{I1, BF16}}, changeElementTo(1, F32))
1406 .widenScalarToNextPow2(1)
1407 .minScalar(1, F32)
1408 .scalarize(0);
1409
1410 getActionDefinitionsBuilder(G_FPOW)
1411 .customFor({F32})
1412 .clampScalar(0, F32, F32)
1413 .scalarize(0);
1414
1415 getActionDefinitionsBuilder(G_FPOWI)
1416 .clampScalar(0, F32, F32)
1417 .scalarize(0)
1418 .lower();
1419
1420 getActionDefinitionsBuilder(G_FLOG2)
1421 .legalFor(ST.has16BitInsts(), {F16})
1422 .legalFor(ST.hasBF16TransInsts(), {BF16})
1423 .customFor({F32, F16})
1424 .scalarize(0)
1425 .widenScalarFor({BF16}, changeElementTo(0, F32))
1426 .lower();
1427
1428 getActionDefinitionsBuilder(G_FEXP2)
1429 .legalFor(ST.has16BitInsts(), {F16})
1430 .legalFor(ST.hasBF16TransInsts(), {BF16})
1431 .customFor({F32, F64, F16})
1432 .scalarize(0)
1433 .widenScalarFor({BF16}, changeElementTo(0, F32))
1434 .lower();
1435
1436 getActionDefinitionsBuilder({G_FLOG, G_FLOG10})
1437 .customFor({F16, F32})
1438 .scalarize(0);
1439
1440 getActionDefinitionsBuilder({G_FEXP, G_FEXP10})
1441 .customFor({F16, F32, F64})
1442 .scalarize(0);
1443
1444 // The 64-bit versions produce 32-bit results, but only on the SALU.
1445 getActionDefinitionsBuilder(G_CTPOP)
1446 .legalFor({{S32, S32}, {S32, S64}})
1447 .clampScalar(0, S32, S32)
1448 .widenScalarToNextPow2(1, 32)
1449 .clampScalar(1, S32, S64)
1450 .scalarize(0)
1451 .widenScalarToNextPow2(0, 32);
1452
1453 // If no 16 bit instr is available, lower into different instructions.
1454 if (ST.has16BitInsts())
1455 getActionDefinitionsBuilder(G_IS_FPCLASS)
1456 .legalForCartesianProduct({I1}, FPTypes16)
1457 .widenScalarToNextPow2(1)
1458 .scalarize(0)
1459 .lower();
1460 else
1461 getActionDefinitionsBuilder(G_IS_FPCLASS)
1462 .legalForCartesianProduct({I1}, FPTypesBase)
1463 .lowerFor({I1, F16})
1464 .widenScalarToNextPow2(1)
1465 .scalarize(0)
1466 .lower();
1467
1468 // The hardware instructions return a different result on 0 than the generic
1469 // instructions expect. The hardware produces -1, but these produce the
1470 // bitwidth.
1471 getActionDefinitionsBuilder({G_CTLZ, G_CTTZ})
1472 .scalarize(0)
1473 .clampScalar(0, S32, S32)
1474 .clampScalar(1, S32, S64)
1475 .widenScalarToNextPow2(0, 32)
1476 .widenScalarToNextPow2(1, 32)
1477 .custom();
1478
1479 // The 64-bit versions produce 32-bit results, but only on the SALU.
1480 getActionDefinitionsBuilder(G_CTLZ_ZERO_POISON)
1481 .legalFor({{S32, S32}, {S32, S64}})
1482 .customIf(scalarNarrowerThan(1, 32))
1483 .clampScalar(0, S32, S32)
1484 .clampScalar(1, S32, S64)
1485 .scalarize(0)
1486 .widenScalarToNextPow2(0, 32)
1487 .widenScalarToNextPow2(1, 32);
1488
1489 getActionDefinitionsBuilder(G_CTTZ_ZERO_POISON)
1490 .legalFor({{S32, S32}, {S32, S64}})
1491 .clampScalar(0, S32, S32)
1492 .clampScalar(1, S32, S64)
1493 .scalarize(0)
1494 .widenScalarToNextPow2(0, 32)
1495 .widenScalarToNextPow2(1, 32);
1496
1497 getActionDefinitionsBuilder(G_CTLS)
1498 .customFor({{S32, S32}})
1499 .scalarize(0)
1500 .clampScalar(0, S32, S32)
1501 .clampScalar(1, S32, S32);
1502
1503 // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
1504 // RegBankSelect.
1505 getActionDefinitionsBuilder(G_BITREVERSE)
1506 .legalFor({S32, S64})
1507 .clampScalar(0, S32, S64)
1508 .scalarize(0)
1509 .widenScalarToNextPow2(0);
1510
1511 if (ST.has16BitInsts()) {
1512 getActionDefinitionsBuilder(G_BSWAP)
1513 .legalFor({S16, S32, V2S16})
1514 .clampMaxNumElementsStrict(0, S16, 2)
1515 // FIXME: Fixing non-power-of-2 before clamp is workaround for
1516 // narrowScalar limitation.
1517 .widenScalarToNextPow2(0)
1518 .clampScalar(0, S16, S32)
1519 .scalarize(0);
1520
1521 if (ST.hasVOP3PInsts()) {
1522 getActionDefinitionsBuilder(G_ABS)
1523 .legalFor({S32, S16, V2S16})
1524 .clampMaxNumElements(0, S16, 2)
1525 .minScalar(0, S16)
1526 .widenScalarToNextPow2(0)
1527 .scalarize(0)
1528 .lower();
1529 if (ST.useMinMaxI64Insts()) {
1530 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1531 .legalFor({S32, S16, S64, V2S16})
1532 .clampMaxNumElements(0, S16, 2)
1533 .minScalar(0, S16)
1534 .widenScalarToNextPow2(0)
1535 .scalarize(0)
1536 .lower();
1537 } else {
1538 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1539 .legalFor({S32, S16, V2S16})
1540 .clampMaxNumElements(0, S16, 2)
1541 .minScalar(0, S16)
1542 .widenScalarToNextPow2(0)
1543 .scalarize(0)
1544 .lower();
1545 }
1546 } else {
1547 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
1548 .legalFor({S32, S16})
1549 .widenScalarToNextPow2(0)
1550 .minScalar(0, S16)
1551 .scalarize(0)
1552 .lower();
1553 }
1554 } else {
1555 // TODO: Should have same legality without v_perm_b32
1556 getActionDefinitionsBuilder(G_BSWAP)
1557 .legalFor({S32})
1558 .lowerIf(scalarNarrowerThan(0, 32))
1559 // FIXME: Fixing non-power-of-2 before clamp is workaround for
1560 // narrowScalar limitation.
1561 .widenScalarToNextPow2(0)
1562 .maxScalar(0, S32)
1563 .scalarize(0)
1564 .lower();
1565
1566 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
1567 .legalFor({S32})
1568 .minScalar(0, S32)
1569 .widenScalarToNextPow2(0)
1570 .scalarize(0)
1571 .lower();
1572 }
1573
1574 getActionDefinitionsBuilder(G_INTTOPTR)
1575 // List the common cases
1576 .legalForCartesianProduct(AddrSpaces64, {S64})
1577 .legalForCartesianProduct(AddrSpaces32, {S32})
1578 .scalarize(0)
1579 // Accept any address space as long as the size matches
1580 .legalIf(sameSize(0, 1))
1581 .widenScalarIf(smallerThan(1, 0),
1582 [](const LegalityQuery &Query) {
1583 return std::pair(
1584 1, LLT::scalar(Query.Types[0].getSizeInBits()));
1585 })
1586 .narrowScalarIf(largerThan(1, 0), [](const LegalityQuery &Query) {
1587 return std::pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
1588 });
1589
1590 getActionDefinitionsBuilder(G_PTRTOINT)
1591 // List the common cases
1592 .legalForCartesianProduct(AddrSpaces64, {S64})
1593 .legalForCartesianProduct(AddrSpaces32, {S32})
1594 .scalarize(0)
1595 // Accept any address space as long as the size matches
1596 .legalIf(sameSize(0, 1))
1597 .widenScalarIf(smallerThan(0, 1),
1598 [](const LegalityQuery &Query) {
1599 return std::pair(
1600 0, LLT::scalar(Query.Types[1].getSizeInBits()));
1601 })
1602 .narrowScalarIf(largerThan(0, 1), [](const LegalityQuery &Query) {
1603 return std::pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
1604 });
1605
1606 getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
1607 .scalarize(0)
1608 .custom();
1609
1610 const auto needToSplitMemOp = [=](const LegalityQuery &Query,
1611 bool IsLoad) -> bool {
1612 const LLT DstTy = Query.Types[0];
1613
1614 // Split vector extloads.
1615 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
1616
1617 if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
1618 return true;
1619
1620 const LLT PtrTy = Query.Types[1];
1621 unsigned AS = PtrTy.getAddressSpace();
1622 if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
1623 Query.MMODescrs[0].Ordering !=
1625 return true;
1626
1627 if (AS == AMDGPUAS::LOCAL_ADDRESS && MemSize == 64 &&
1628 !ST.hasUsableDSOffset() && Query.MMODescrs[0].AlignInBits == 32)
1629 return true;
1630
1631 // Catch weird sized loads that don't evenly divide into the access sizes
1632 // TODO: May be able to widen depending on alignment etc.
1633 unsigned NumRegs = (MemSize + 31) / 32;
1634 if (NumRegs == 3) {
1635 if (!ST.hasDwordx3LoadStores())
1636 return true;
1637 } else {
1638 // If the alignment allows, these should have been widened.
1639 if (!isPowerOf2_32(NumRegs))
1640 return true;
1641 }
1642
1643 return false;
1644 };
1645
1646 unsigned GlobalAlign32 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 32;
1647 unsigned GlobalAlign16 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 16;
1648 unsigned GlobalAlign8 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 8;
1649 unsigned LocalAlign64 = ST.hasUsableDSOffset() ? 32 : 64;
1650
1651 // TODO: Refine based on subtargets which support unaligned access or 128-bit
1652 // LDS
1653 // TODO: Unsupported flat for SI.
1654
1655 for (unsigned Op : {G_LOAD, G_STORE}) {
1656 const bool IsStore = Op == G_STORE;
1657
1658 auto &Actions = getActionDefinitionsBuilder(Op);
1659 // Explicitly list some common cases.
1660 // TODO: Does this help compile time at all?
1661 Actions.legalForTypesWithMemDesc(
1662 {{S32, GlobalPtr, S32, GlobalAlign32},
1663 {V2S32, GlobalPtr, V2S32, GlobalAlign32},
1664 {V4S32, GlobalPtr, V4S32, GlobalAlign32},
1665 {S64, GlobalPtr, S64, GlobalAlign32},
1666 {V2S64, GlobalPtr, V2S64, GlobalAlign32},
1667 {V2S16, GlobalPtr, V2S16, GlobalAlign32},
1668 {S32, GlobalPtr, S8, GlobalAlign8},
1669 {S32, GlobalPtr, S16, GlobalAlign16},
1670
1671 {S32, LocalPtr, S32, 32},
1672 {S64, LocalPtr, S64, LocalAlign64},
1673 {V2S32, LocalPtr, V2S32, LocalAlign64},
1674 {S32, LocalPtr, S8, 8},
1675 {S32, LocalPtr, S16, 16},
1676 {V2S16, LocalPtr, S32, 32},
1677
1678 {S32, PrivatePtr, S32, 32},
1679 {S32, PrivatePtr, S8, 8},
1680 {S32, PrivatePtr, S16, 16},
1681 {V2S16, PrivatePtr, S32, 32},
1682
1683 {S32, ConstantPtr, S32, GlobalAlign32},
1684 {V2S32, ConstantPtr, V2S32, GlobalAlign32},
1685 {V4S32, ConstantPtr, V4S32, GlobalAlign32},
1686 {S64, ConstantPtr, S64, GlobalAlign32}});
1687
1688 if (IsStore || ST.d16PreservesUnusedBits()) {
1689 Actions.legalForTypesWithMemDesc(ST.useRealTrue16Insts(), /* Pred */
1690 {{S16, GlobalPtr, S8, GlobalAlign8},
1691 {S16, GlobalPtr, S16, GlobalAlign16},
1692 {S16, LocalPtr, S8, 8},
1693 {S16, LocalPtr, S16, 16},
1694 {S16, PrivatePtr, S8, 8},
1695 {S16, PrivatePtr, S16, 16}});
1696 }
1697
1698 Actions.legalIf(
1699 [=](const LegalityQuery &Query) -> bool {
1700 return isLoadStoreLegal(ST, Query);
1701 });
1702
1703 // The custom pointers (fat pointers, buffer resources) don't work with load
1704 // and store at this level. Fat pointers should have been lowered to
1705 // intrinsics before the translation to MIR.
1706 Actions.unsupportedIf(
1707 typeInSet(1, {BufferFatPtr, BufferStridedPtr, RsrcPtr}));
1708
1709 // Address space 8 pointers are handled by a 4xs32 load, bitcast, and
1710 // ptrtoint. This is needed to account for the fact that we can't have i128
1711 // as a register class for SelectionDAG reasons.
1712 Actions.customIf([=](const LegalityQuery &Query) -> bool {
1713 return hasBufferRsrcWorkaround(Query.Types[0]);
1714 });
1715
1716 // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
1717 // 64-bits.
1718 //
1719 // VGPR accesses are always custom, to be lowered or diagnosed.
1720 //
1721 // TODO: Should generalize bitcast action into coerce, which will also cover
1722 // inserting addrspacecasts.
1723 Actions.customIf(typeInSet(1, {Constant32Ptr, VGPRPtr}));
1724
1725 // Turn any illegal element vectors into something easier to deal
1726 // with. These will ultimately produce 32-bit scalar shifts to extract the
1727 // parts anyway.
1728 //
1729 // For odd 16-bit element vectors, prefer to split those into pieces with
1730 // 16-bit vector parts.
1731 Actions.bitcastIf(
1732 [=](const LegalityQuery &Query) -> bool {
1733 return shouldBitcastLoadStoreType(ST, Query.Types[0],
1734 Query.MMODescrs[0].MemoryTy);
1735 }, bitcastToRegisterType(0));
1736
1737 if (!IsStore) {
1738 // Widen suitably aligned loads by loading extra bytes. The standard
1739 // legalization actions can't properly express widening memory operands.
1740 Actions.customIf([=](const LegalityQuery &Query) -> bool {
1741 return shouldWidenLoad(ST, Query, G_LOAD);
1742 });
1743 }
1744
1745 // FIXME: load/store narrowing should be moved to lower action
1746 Actions
1747 .narrowScalarIf(
1748 [=](const LegalityQuery &Query) -> bool {
1749 return !Query.Types[0].isVector() &&
1750 needToSplitMemOp(Query, Op == G_LOAD);
1751 },
1752 [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1753 const LLT DstTy = Query.Types[0];
1754 const LLT PtrTy = Query.Types[1];
1755
1756 const unsigned DstSize = DstTy.getSizeInBits();
1757 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
1758
1759 // Split extloads.
1760 if (DstSize > MemSize)
1761 return std::pair(0, LLT::integer(MemSize));
1762
1763 unsigned MaxSize = maxSizeForAddrSpace(
1764 ST, PtrTy.getAddressSpace(), Op == G_LOAD,
1765 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
1766 if (MemSize > MaxSize)
1767 return std::pair(0, LLT::integer(MaxSize));
1768
1769 uint64_t Align = Query.MMODescrs[0].AlignInBits;
1770 return std::pair(0, LLT::integer(Align));
1771 })
1772 .fewerElementsIf(
1773 [=](const LegalityQuery &Query) -> bool {
1774 return Query.Types[0].isVector() &&
1775 needToSplitMemOp(Query, Op == G_LOAD);
1776 },
1777 [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1778 const LLT DstTy = Query.Types[0];
1779 const LLT PtrTy = Query.Types[1];
1780
1781 LLT EltTy = DstTy.getElementType();
1782 unsigned MaxSize = maxSizeForAddrSpace(
1783 ST, PtrTy.getAddressSpace(), Op == G_LOAD,
1784 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
1785
1786 // FIXME: Handle widened to power of 2 results better. This ends
1787 // up scalarizing.
1788 // FIXME: 3 element stores scalarized on SI
1789
1790 // Split if it's too large for the address space.
1791 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
1792 if (MemSize > MaxSize) {
1793 unsigned NumElts = DstTy.getNumElements();
1794 unsigned EltSize = EltTy.getSizeInBits();
1795
1796 if (MaxSize % EltSize == 0) {
1797 return std::pair(
1799 ElementCount::getFixed(MaxSize / EltSize), EltTy));
1800 }
1801
1802 unsigned NumPieces = MemSize / MaxSize;
1803
1804 // FIXME: Refine when odd breakdowns handled
1805 // The scalars will need to be re-legalized.
1806 if (NumPieces == 1 || NumPieces >= NumElts ||
1807 NumElts % NumPieces != 0)
1808 return std::pair(0, EltTy);
1809
1810 return std::pair(0,
1811 LLT::fixed_vector(NumElts / NumPieces, EltTy));
1812 }
1813
1814 // FIXME: We could probably handle weird extending loads better.
1815 if (DstTy.getSizeInBits() > MemSize)
1816 return std::pair(0, EltTy);
1817
1818 unsigned EltSize = EltTy.getSizeInBits();
1819 unsigned DstSize = DstTy.getSizeInBits();
1820 if (!isPowerOf2_32(DstSize)) {
1821 // We're probably decomposing an odd sized store. Try to split
1822 // to the widest type. TODO: Account for alignment. As-is it
1823 // should be OK, since the new parts will be further legalized.
1824 unsigned FloorSize = llvm::bit_floor(DstSize);
1825 return std::pair(
1827 ElementCount::getFixed(FloorSize / EltSize), EltTy));
1828 }
1829
1830 // May need relegalization for the scalars.
1831 return std::pair(0, EltTy);
1832 })
1833 .widenScalarIf(scalarNarrowerThan(0, 32), changeTo(0, LLT::integer(32)))
1834 .narrowScalarIf(isTruncStoreToSizePowerOf2(0),
1836 .widenScalarToNextPow2(0)
1837 .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
1838 .lower();
1839 }
1840
1841 // FIXME: Unaligned accesses not lowered.
1842 auto &ExtLoads =
1843 getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
1844 .legalForTypesWithMemDesc({{S32, GlobalPtr, S8, 8},
1845 {S32, GlobalPtr, S16, 2 * 8},
1846 {S32, LocalPtr, S8, 8},
1847 {S32, LocalPtr, S16, 16},
1848 {S32, PrivatePtr, S8, 8},
1849 {S32, PrivatePtr, S16, 16},
1850 {S32, ConstantPtr, S8, 8},
1851 {S32, ConstantPtr, S16, 2 * 8}})
1852 .legalForTypesWithMemDesc(ST.useRealTrue16Insts(),
1853 {{S16, GlobalPtr, S8, GlobalAlign8},
1854 {S16, LocalPtr, S8, GlobalAlign8},
1855 {S16, PrivatePtr, S8, GlobalAlign8},
1856 {S16, ConstantPtr, S8, GlobalAlign8}})
1857 .legalIf([=](const LegalityQuery &Query) -> bool {
1858 return isLoadStoreLegal(ST, Query);
1859 });
1860
1861 if (ST.hasFlatAddressSpace()) {
1862 ExtLoads.legalForTypesWithMemDesc(
1863 {{S32, FlatPtr, S8, 8}, {S32, FlatPtr, S16, 16}});
1864
1865 ExtLoads.legalForTypesWithMemDesc(ST.useRealTrue16Insts(),
1866 {{S16, FlatPtr, S8, GlobalAlign8}});
1867 }
1868
1869 // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
1870 // 64-bits.
1871 //
1872 // VGPR accesses are always custom, to be lowered or diagnosed.
1873 //
1874 // TODO: Should generalize bitcast action into coerce, which will also cover
1875 // inserting addrspacecasts.
1876 ExtLoads.customIf(typeInSet(1, {Constant32Ptr, VGPRPtr}));
1877
1878 ExtLoads.narrowScalarIf(
1879 [](const LegalityQuery &Query) {
1880 LLT MemTy = Query.MMODescrs[0].MemoryTy;
1881 return MemTy.isScalar() && MemTy.getSizeInBits() > 32 &&
1882 Query.Types[0].getSizeInBits() > MemTy.getSizeInBits();
1883 }, // For large MemSize, narrowscalar to MemSize (load MemSize + ext)
1885 ExtLoads.clampScalar(0, S32, S32)
1886 .widenScalarToNextPow2(0)
1887 .lower();
1888
1889 auto &Atomics = getActionDefinitionsBuilder(
1890 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
1891 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
1892 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
1893 G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP, G_ATOMICRMW_UDEC_WRAP})
1894 .legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
1895 {S64, GlobalPtr}, {S64, LocalPtr},
1896 {S32, RegionPtr}, {S64, RegionPtr}});
1897 if (ST.hasFlatAddressSpace()) {
1898 Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
1899 }
1900
1901 auto &Atomics32 =
1902 getActionDefinitionsBuilder({G_ATOMICRMW_USUB_COND, G_ATOMICRMW_USUB_SAT})
1903 .legalFor({{S32, GlobalPtr}, {S32, LocalPtr}, {S32, RegionPtr}});
1904 if (ST.hasFlatAddressSpace()) {
1905 Atomics32.legalFor({{S32, FlatPtr}});
1906 }
1907
1908 // TODO: v2bf16 operations, and fat buffer pointer support.
1909 auto &Atomic = getActionDefinitionsBuilder(G_ATOMICRMW_FADD);
1910 if (ST.hasLDSFPAtomicAddF32()) {
1911 Atomic.legalFor({{F32, LocalPtr}, {F32, RegionPtr}});
1912 if (ST.hasLdsAtomicAddF64())
1913 Atomic.legalFor({{F64, LocalPtr}});
1914 if (ST.hasAtomicDsPkAdd16Insts())
1915 Atomic.legalFor({{V2F16, LocalPtr}, {V2BF16, LocalPtr}});
1916 }
1917 if (ST.hasAtomicFaddInsts())
1918 Atomic.legalFor({{F32, GlobalPtr}});
1919 if (ST.hasFlatAtomicFaddF32Inst())
1920 Atomic.legalFor({{F32, FlatPtr}});
1921
1922 if (ST.hasGFX90AInsts() || ST.hasGFX1250Insts()) {
1923 // These are legal with some caveats, and should have undergone expansion in
1924 // the IR in most situations
1925 // TODO: Move atomic expansion into legalizer
1926 Atomic.legalFor({{F32, GlobalPtr}, {F64, GlobalPtr}, {F64, FlatPtr}});
1927 }
1928
1929 if (ST.hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
1930 ST.hasAtomicBufferGlobalPkAddF16Insts())
1931 Atomic.legalFor({{V2F16, GlobalPtr}, {V2F16, BufferFatPtr}});
1932 if (ST.hasAtomicGlobalPkAddBF16Inst())
1933 Atomic.legalFor({{V2BF16, GlobalPtr}});
1934 if (ST.hasAtomicFlatPkAdd16Insts())
1935 Atomic.legalFor({{V2F16, FlatPtr}, {V2BF16, FlatPtr}});
1936
1937
1938 // Most of the legalization work here is done by AtomicExpand. We could
1939 // probably use a simpler legality rule that just assumes anything is OK.
1940 auto &AtomicFMinFMax =
1941 getActionDefinitionsBuilder({G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
1942 .legalFor({{F32, LocalPtr}, {F64, LocalPtr}});
1943
1944 if (ST.hasAtomicFMinFMaxF32GlobalInsts())
1945 AtomicFMinFMax.legalFor({{F32, GlobalPtr},{F32, BufferFatPtr}});
1946 if (ST.hasAtomicFMinFMaxF64GlobalInsts())
1947 AtomicFMinFMax.legalFor({{F64, GlobalPtr}, {F64, BufferFatPtr}});
1948 if (ST.hasAtomicFMinFMaxF32FlatInsts())
1949 AtomicFMinFMax.legalFor({F32, FlatPtr});
1950 if (ST.hasAtomicFMinFMaxF64FlatInsts())
1951 AtomicFMinFMax.legalFor({F64, FlatPtr});
1952
1953 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
1954 // demarshalling
1955 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
1956 .customFor({{S32, GlobalPtr}, {S64, GlobalPtr},
1957 {S32, FlatPtr}, {S64, FlatPtr}})
1958 .legalFor({{S32, LocalPtr}, {S64, LocalPtr},
1959 {S32, RegionPtr}, {S64, RegionPtr}});
1960 // TODO: Pointer types, any 32-bit or 64-bit vector
1961
1962 // Condition should be s32 for scalar, s1 for vector.
1963 getActionDefinitionsBuilder(G_SELECT)
1964 .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16, GlobalPtr,
1965 LocalPtr, FlatPtr, PrivatePtr,
1966 LLT::fixed_vector(2, LocalPtr),
1967 LLT::fixed_vector(2, PrivatePtr)},
1968 {S1, S32})
1969 .clampScalar(0, S16, S64)
1970 .scalarize(1)
1971 .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
1972 .fewerElementsIf(numElementsNotEven(0), scalarize(0))
1973 .clampMaxNumElements(0, S32, 2)
1974 .clampMaxNumElements(0, LocalPtr, 2)
1975 .clampMaxNumElements(0, PrivatePtr, 2)
1976 .scalarize(0)
1977 .widenScalarToNextPow2(0)
1978 .legalIf(all(isPointer(0), typeInSet(1, {S1, S32})));
1979
1980 // TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
1981 // be more flexible with the shift amount type.
1982 auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
1983 .legalFor({{S32, S32}, {S64, S32}});
1984 if (ST.has16BitInsts()) {
1985 if (ST.hasVOP3PInsts()) {
1986 Shifts.legalFor({{S16, S16}, {V2S16, V2S16}})
1987 .clampMaxNumElements(0, S16, 2);
1988 } else
1989 Shifts.legalFor({{S16, S16}});
1990
1991 // TODO: Support 16-bit shift amounts for all types
1992 Shifts.widenScalarIf(
1993 [=](const LegalityQuery &Query) {
1994 // Use 16-bit shift amounts for any 16-bit shift. Otherwise we want a
1995 // 32-bit amount.
1996 const LLT ValTy = Query.Types[0];
1997 const LLT AmountTy = Query.Types[1];
1998 return ValTy.isScalar() && ValTy.getSizeInBits() <= 16 &&
1999 AmountTy.getSizeInBits() < 16;
2000 },
2002 Shifts.maxScalarIf(typeIs(0, S16), 1, S16);
2003 Shifts.clampScalar(1, S32, S32);
2004 Shifts.widenScalarToNextPow2(0, 16);
2005 Shifts.clampScalar(0, S16, S64);
2006
2007 getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
2008 .minScalar(0, S16)
2009 .scalarize(0)
2010 .lower();
2011 } else {
2012 // Make sure we legalize the shift amount type first, as the general
2013 // expansion for the shifted type will produce much worse code if it hasn't
2014 // been truncated already.
2015 Shifts.clampScalar(1, S32, S32);
2016 Shifts.widenScalarToNextPow2(0, 32);
2017 Shifts.clampScalar(0, S32, S64);
2018
2019 getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
2020 .minScalar(0, S32)
2021 .scalarize(0)
2022 .lower();
2023 }
2024 Shifts.scalarize(0);
2025
2026 for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
2027 unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
2028 unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
2029 unsigned IdxTypeIdx = 2;
2030
2031 getActionDefinitionsBuilder(Op)
2032 .customIf([=](const LegalityQuery &Query) {
2033 const LLT EltTy = Query.Types[EltTypeIdx];
2034 const LLT VecTy = Query.Types[VecTypeIdx];
2035 const LLT IdxTy = Query.Types[IdxTypeIdx];
2036 const unsigned EltSize = EltTy.getSizeInBits();
2037 const bool isLegalVecType =
2039 // Address space 8 pointers are 128-bit wide values, but the logic
2040 // below will try to bitcast them to 2N x s64, which will fail.
2041 // Therefore, as an intermediate step, wrap extracts/insertions from a
2042 // ptrtoint-ing the vector and scalar arguments (or inttoptring the
2043 // extraction result) in order to produce a vector operation that can
2044 // be handled by the logic below.
2045 if (EltTy.isPointer() && EltSize > 64)
2046 return true;
2047 return (EltSize == 32 || EltSize == 64) &&
2048 VecTy.getSizeInBits() % 32 == 0 &&
2049 VecTy.getSizeInBits() <= MaxRegisterSize &&
2050 IdxTy.getSizeInBits() == 32 &&
2051 isLegalVecType;
2052 })
2053 .bitcastIf(all(sizeIsMultipleOf32(VecTypeIdx),
2054 scalarOrEltNarrowerThan(VecTypeIdx, 32)),
2055 bitcastToVectorElement32(VecTypeIdx))
2056 //.bitcastIf(vectorSmallerThan(1, 32), bitcastToScalar(1))
2057 .bitcastIf(all(sizeIsMultipleOf32(VecTypeIdx),
2058 scalarOrEltWiderThan(VecTypeIdx, 64)),
2059 [=](const LegalityQuery &Query) {
2060 // For > 64-bit element types, try to turn this into a
2061 // 64-bit element vector since we may be able to do better
2062 // indexing if this is scalar. If not, fall back to 32.
2063 const LLT EltTy = Query.Types[EltTypeIdx];
2064 const LLT VecTy = Query.Types[VecTypeIdx];
2065 const unsigned DstEltSize = EltTy.getSizeInBits();
2066 const unsigned VecSize = VecTy.getSizeInBits();
2067
2068 const unsigned TargetEltSize =
2069 DstEltSize % 64 == 0 ? 64 : 32;
2070 return std::pair(VecTypeIdx,
2071 LLT::fixed_vector(VecSize / TargetEltSize,
2072 TargetEltSize));
2073 })
2074 .clampScalar(EltTypeIdx, S32, S64)
2075 .clampScalar(VecTypeIdx, S32, S64)
2076 .clampScalar(IdxTypeIdx, S32, S32)
2077 .clampMaxNumElements(VecTypeIdx, S32, 32)
2078 // TODO: Clamp elements for 64-bit vectors?
2079 .moreElementsIf(isIllegalRegisterType(ST, VecTypeIdx),
2081 // It should only be necessary with variable indexes.
2082 // As a last resort, lower to the stack
2083 .lower();
2084 }
2085
2086 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
2087 .unsupportedIf([=](const LegalityQuery &Query) {
2088 const LLT &EltTy = Query.Types[1].getElementType();
2089 return Query.Types[0] != EltTy;
2090 });
2091
2092 for (unsigned Op : {G_EXTRACT, G_INSERT}) {
2093 unsigned BigTyIdx = Op == G_EXTRACT ? 1 : 0;
2094 unsigned LitTyIdx = Op == G_EXTRACT ? 0 : 1;
2095 getActionDefinitionsBuilder(Op)
2096 .widenScalarIf(
2097 [=](const LegalityQuery &Query) {
2098 const LLT BigTy = Query.Types[BigTyIdx];
2099 return (BigTy.getScalarSizeInBits() < 16);
2100 },
2102 .widenScalarIf(
2103 [=](const LegalityQuery &Query) {
2104 const LLT LitTy = Query.Types[LitTyIdx];
2105 return (LitTy.getScalarSizeInBits() < 16);
2106 },
2108 .moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
2109 .widenScalarToNextPow2(BigTyIdx, 32)
2110 .customIf([=](const LegalityQuery &Query) {
2111 // Generic lower operates on the full-width value, producing
2112 // shift+trunc/mask sequences. For simple cases where extract/insert
2113 // values are 32-bit aligned, we can instead unmerge/merge and work on
2114 // the 32-bit components. However, we can't check the offset here so
2115 // custom lower function will have to call generic lowering if offset
2116 // is not 32-bit aligned.
2117 const LLT BigTy = Query.Types[BigTyIdx];
2118 const LLT LitTy = Query.Types[LitTyIdx];
2119 return !BigTy.isVector() && BigTy.getSizeInBits() % 32 == 0 &&
2120 LitTy.getSizeInBits() % 32 == 0;
2121 })
2122 .lower();
2123 }
2124
2125 auto &BuildVector =
2126 getActionDefinitionsBuilder(G_BUILD_VECTOR)
2127 .legalForCartesianProduct(AllS32Vectors, {S32})
2128 .legalForCartesianProduct(AllS64Vectors, {S64})
2129 .clampNumElements(0, V16S32, V32S32)
2130 .clampNumElements(0, V2S64, V16S64)
2131 .fewerElementsIf(isWideVec16(0),
2133 .moreElementsIf(isIllegalRegisterType(ST, 0),
2135
2136 if (ST.hasScalarPackInsts()) {
2137 BuildVector
2138 // FIXME: Should probably widen s1 vectors straight to s32
2139 .minScalarOrElt(0, S16)
2140 .minScalar(1, S16);
2141
2142 getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
2143 .legalFor({V2S16, S32})
2144 .lower();
2145 } else {
2146 BuildVector.customFor({V2S16, S16});
2147 BuildVector.minScalarOrElt(0, S32);
2148
2149 getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
2150 .customFor({V2S16, S32})
2151 .lower();
2152 }
2153
2154 BuildVector.legalIf(isRegisterType(ST, 0));
2155
2156 // FIXME: Clamp maximum size
2157 getActionDefinitionsBuilder(G_CONCAT_VECTORS)
2158 .legalIf(all(isRegisterType(ST, 0), isRegisterType(ST, 1)))
2159 .clampMaxNumElements(0, S32, 32)
2160 .clampMaxNumElements(1, S16, 2) // TODO: Make 4?
2161 .clampMaxNumElements(0, S16, 64);
2162
2163 getActionDefinitionsBuilder(G_SHUFFLE_VECTOR).lower();
2164
2165 // Merge/Unmerge
2166 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
2167 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
2168 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
2169
2170 auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
2171 const LLT Ty = Query.Types[TypeIdx];
2172 if (Ty.isVector()) {
2173 const LLT &EltTy = Ty.getElementType();
2174 if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 512)
2175 return true;
2177 return true;
2178 }
2179 return false;
2180 };
2181
2182 auto &Builder =
2183 getActionDefinitionsBuilder(Op)
2184 .legalIf(all(isRegisterType(ST, 0), isRegisterType(ST, 1)))
2185 .lowerFor({{S16, V2S16}})
2186 .lowerIf([=](const LegalityQuery &Query) {
2187 const LLT BigTy = Query.Types[BigTyIdx];
2188 return BigTy.getSizeInBits() == 32;
2189 })
2190 // Try to widen to s16 first for small types.
2191 // TODO: Only do this on targets with legal s16 shifts
2192 .minScalarOrEltIf(scalarNarrowerThan(LitTyIdx, 16), LitTyIdx, S16)
2193 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 16)
2194 .moreElementsIf(isSmallOddVector(BigTyIdx),
2195 oneMoreElement(BigTyIdx))
2196 .fewerElementsIf(all(typeIs(0, S16), vectorWiderThan(1, 32),
2197 elementTypeIs(1, S16)),
2199 // Clamp the little scalar to s8-s256 and make it a power of 2. It's
2200 // not worth considering the multiples of 64 since 2*192 and 2*384
2201 // are not valid.
2202 .clampScalar(LitTyIdx, S32, S512)
2203 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 32)
2204 // Break up vectors with weird elements into scalars
2205 .fewerElementsIf(
2206 [=](const LegalityQuery &Query) {
2207 return notValidElt(Query, LitTyIdx);
2208 },
2209 scalarize(0))
2210 .fewerElementsIf(
2211 [=](const LegalityQuery &Query) {
2212 return notValidElt(Query, BigTyIdx);
2213 },
2214 scalarize(1))
2215 .clampScalar(BigTyIdx, S32, MaxScalar);
2216
2217 if (Op == G_MERGE_VALUES) {
2218 Builder.widenScalarIf(
2219 // TODO: Use 16-bit shifts if legal for 8-bit values?
2220 [=](const LegalityQuery &Query) {
2221 const LLT Ty = Query.Types[LitTyIdx];
2222 return Ty.getSizeInBits() < 32;
2223 },
2224 changeElementSizeTo(LitTyIdx, S32));
2225 }
2226
2227 Builder.widenScalarIf(
2228 [=](const LegalityQuery &Query) {
2229 const LLT Ty = Query.Types[BigTyIdx];
2230 return Ty.getSizeInBits() % 16 != 0;
2231 },
2232 [=](const LegalityQuery &Query) {
2233 // Pick the next power of 2, or a multiple of 64 over 128.
2234 // Whichever is smaller.
2235 const LLT &Ty = Query.Types[BigTyIdx];
2236 unsigned NewSizeInBits = 1 << Log2_32_Ceil(Ty.getSizeInBits() + 1);
2237 if (NewSizeInBits >= 256) {
2238 unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
2239 if (RoundedTo < NewSizeInBits)
2240 NewSizeInBits = RoundedTo;
2241 }
2242 return std::pair(BigTyIdx, LLT::scalar(NewSizeInBits));
2243 })
2244 // Any vectors left are the wrong size. Scalarize them.
2245 .scalarize(0)
2246 .scalarize(1);
2247 }
2248
2249 // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
2250 // RegBankSelect.
2251 auto &SextInReg = getActionDefinitionsBuilder(G_SEXT_INREG)
2252 .legalFor({{S32}, {S64}})
2253 .clampScalar(0, S32, S64);
2254
2255 if (ST.hasVOP3PInsts()) {
2256 SextInReg.lowerFor({{V2S16}})
2257 // Prefer to reduce vector widths for 16-bit vectors before lowering, to
2258 // get more vector shift opportunities, since we'll get those when
2259 // expanded.
2260 .clampMaxNumElementsStrict(0, S16, 2);
2261 } else if (ST.has16BitInsts()) {
2262 SextInReg.lowerFor({{S32}, {S64}, {S16}});
2263 } else {
2264 // Prefer to promote to s32 before lowering if we don't have 16-bit
2265 // shifts. This avoid a lot of intermediate truncate and extend operations.
2266 SextInReg.lowerFor({{S32}, {S64}});
2267 }
2268
2269 SextInReg
2270 .scalarize(0)
2271 .clampScalar(0, S32, S64)
2272 .lower();
2273
2274 getActionDefinitionsBuilder({G_ROTR, G_ROTL})
2275 .scalarize(0)
2276 .lower();
2277
2278 auto &FSHRActionDefs = getActionDefinitionsBuilder(G_FSHR);
2279 FSHRActionDefs.legalFor({{S32, S32}})
2280 .clampMaxNumElementsStrict(0, S16, 2);
2281 if (ST.hasVOP3PInsts())
2282 FSHRActionDefs.lowerFor({{V2S16, V2S16}});
2283 FSHRActionDefs.scalarize(0).lower();
2284
2285 if (ST.hasVOP3PInsts()) {
2286 getActionDefinitionsBuilder(G_FSHL)
2287 .lowerFor({{V2S16, V2S16}})
2288 .clampMaxNumElementsStrict(0, S16, 2)
2289 .scalarize(0)
2290 .lower();
2291 } else {
2292 getActionDefinitionsBuilder(G_FSHL)
2293 .scalarize(0)
2294 .lower();
2295 }
2296
2297 getActionDefinitionsBuilder(G_READCYCLECOUNTER)
2298 .legalFor({S64});
2299
2300 getActionDefinitionsBuilder(G_READSTEADYCOUNTER).legalFor({S64});
2301
2302 getActionDefinitionsBuilder(G_FENCE)
2303 .alwaysLegal();
2304
2305 getActionDefinitionsBuilder({G_SMULO, G_UMULO})
2306 .scalarize(0)
2307 .minScalar(0, S32)
2308 .lower();
2309
2310 getActionDefinitionsBuilder({G_SBFX, G_UBFX})
2311 .legalFor({{S32, S32}, {S64, S32}})
2312 .clampScalar(1, S32, S32)
2313 .clampScalar(0, S32, S64)
2314 .widenScalarToNextPow2(0)
2315 .scalarize(0);
2316
2317 getActionDefinitionsBuilder(
2318 {// TODO: Verify V_BFI_B32 is generated from expanded bit ops
2319 G_FCOPYSIGN,
2320
2321 G_ATOMIC_CMPXCHG_WITH_SUCCESS, G_ATOMICRMW_NAND, G_ATOMICRMW_FSUB,
2322 G_READ_REGISTER,
2323
2324 G_SADDO, G_SSUBO})
2325 .lower();
2326
2327 if (ST.hasIEEEMinimumMaximumInsts()) {
2328 getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
2329 .legalFor(FPTypesPK16)
2330 .clampMaxNumElements(0, F16, 2)
2331 .scalarize(0);
2332 } else if (ST.hasVOP3PInsts()) {
2333 getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
2334 .lowerFor({V2F16})
2335 .clampMaxNumElementsStrict(0, F16, 2)
2336 .scalarize(0)
2337 .lower();
2338 } else {
2339 getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
2340 .scalarize(0)
2341 .clampScalar(0, F32, F64)
2342 .lower();
2343 }
2344
2345 getActionDefinitionsBuilder(
2346 {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET, G_MEMSET_INLINE})
2347 .lower();
2348
2349 getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom();
2350
2351 getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
2352 G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
2353 G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
2354 .unsupported();
2355
2356 getActionDefinitionsBuilder(G_PREFETCH)
2357 .customFor({Constant32Ptr})
2358 .alwaysLegal();
2359
2360 getActionDefinitionsBuilder(
2361 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
2362 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FADD, G_VECREDUCE_FMUL,
2363 G_VECREDUCE_FMIN, G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM,
2364 G_VECREDUCE_FMAXIMUM, G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
2365 .legalFor(AllVectors)
2366 .scalarize(1)
2367 .lower();
2368
2369 getActionDefinitionsBuilder({G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS,
2370 G_INTRINSIC_CONVERGENT,
2371 G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
2372 .alwaysLegal();
2373
2374 verify(*ST.getInstrInfo());
2375}
2376
2379 LostDebugLocObserver &LocObserver) const {
2380 MachineIRBuilder &B = Helper.MIRBuilder;
2381 MachineRegisterInfo &MRI = *B.getMRI();
2382
2383 switch (MI.getOpcode()) {
2384 case TargetOpcode::G_ADDRSPACE_CAST:
2385 return legalizeAddrSpaceCast(MI, MRI, B);
2386 case TargetOpcode::G_FRAME_INDEX:
2387 return legalizeFrameIndexVGPR(MI, B);
2388 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2389 return legalizeFroundeven(MI, MRI, B);
2390 case TargetOpcode::G_FCEIL:
2391 return legalizeFceil(MI, MRI, B);
2392 case TargetOpcode::G_FREM:
2393 return legalizeFrem(MI, MRI, B);
2394 case TargetOpcode::G_INTRINSIC_TRUNC:
2395 return legalizeIntrinsicTrunc(MI, MRI, B);
2396 case TargetOpcode::G_SITOFP:
2397 return legalizeITOFP(MI, MRI, B, true);
2398 case TargetOpcode::G_UITOFP:
2399 return legalizeITOFP(MI, MRI, B, false);
2400 case TargetOpcode::G_FPTOSI:
2401 return legalizeFPTOI(MI, MRI, B, true);
2402 case TargetOpcode::G_FPTOUI:
2403 return legalizeFPTOI(MI, MRI, B, false);
2404 case TargetOpcode::G_FMINNUM:
2405 case TargetOpcode::G_FMAXNUM:
2406 case TargetOpcode::G_FMINIMUMNUM:
2407 case TargetOpcode::G_FMAXIMUMNUM:
2408 return legalizeMinNumMaxNum(Helper, MI);
2409 case TargetOpcode::G_EXTRACT:
2410 return legalizeExtract(Helper, MI);
2411 case TargetOpcode::G_INSERT:
2412 return legalizeInsert(Helper, MI);
2413 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
2414 return legalizeExtractVectorElt(MI, MRI, B);
2415 case TargetOpcode::G_INSERT_VECTOR_ELT:
2416 return legalizeInsertVectorElt(MI, MRI, B);
2417 case TargetOpcode::G_FSIN:
2418 case TargetOpcode::G_FCOS:
2419 return legalizeSinCos(MI, MRI, B);
2420 case TargetOpcode::G_GLOBAL_VALUE:
2421 return legalizeGlobalValue(MI, MRI, B);
2422 case TargetOpcode::G_LOAD:
2423 case TargetOpcode::G_SEXTLOAD:
2424 case TargetOpcode::G_ZEXTLOAD:
2425 case TargetOpcode::G_PREFETCH:
2426 return legalizeLoad(Helper, MI);
2427 case TargetOpcode::G_STORE:
2428 return legalizeStore(Helper, MI);
2429 case TargetOpcode::G_FMAD:
2430 return legalizeFMad(MI, MRI, B);
2431 case TargetOpcode::G_FDIV:
2432 return legalizeFDIV(MI, MRI, B);
2433 case TargetOpcode::G_FFREXP:
2434 return legalizeFFREXP(MI, MRI, B);
2435 case TargetOpcode::G_FSQRT:
2436 return legalizeFSQRT(MI, MRI, B);
2437 case TargetOpcode::G_UDIV:
2438 case TargetOpcode::G_UREM:
2439 case TargetOpcode::G_UDIVREM:
2440 return legalizeUnsignedDIV_REM(MI, MRI, B);
2441 case TargetOpcode::G_SDIV:
2442 case TargetOpcode::G_SREM:
2443 case TargetOpcode::G_SDIVREM:
2444 return legalizeSignedDIV_REM(MI, MRI, B);
2445 case TargetOpcode::G_ATOMIC_CMPXCHG:
2446 return legalizeAtomicCmpXChg(MI, MRI, B);
2447 case TargetOpcode::G_FLOG2:
2448 return legalizeFlog2(MI, B);
2449 case TargetOpcode::G_FLOG:
2450 case TargetOpcode::G_FLOG10:
2451 return legalizeFlogCommon(MI, B);
2452 case TargetOpcode::G_FEXP2:
2453 return legalizeFExp2(MI, B);
2454 case TargetOpcode::G_FEXP:
2455 case TargetOpcode::G_FEXP10:
2456 return legalizeFExp(MI, B);
2457 case TargetOpcode::G_FPOW:
2458 return legalizeFPow(Helper, MI);
2459 case TargetOpcode::G_FFLOOR:
2460 return legalizeFFloor(MI, MRI, B);
2461 case TargetOpcode::G_BUILD_VECTOR:
2462 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
2463 return legalizeBuildVector(MI, MRI, B);
2464 case TargetOpcode::G_MUL:
2465 return legalizeMul(Helper, MI);
2466 case TargetOpcode::G_CTLZ:
2467 case TargetOpcode::G_CTTZ:
2468 return legalizeCTLZ_CTTZ(MI, MRI, B);
2469 case TargetOpcode::G_CTLS:
2470 return legalizeCTLS(MI, MRI, B);
2471 case TargetOpcode::G_CTLZ_ZERO_POISON:
2472 return legalizeCTLZ_ZERO_POISON(MI, MRI, B);
2473 case TargetOpcode::G_STACKSAVE:
2474 return legalizeStackSave(MI, B);
2475 case TargetOpcode::G_GET_FPENV:
2476 return legalizeGetFPEnv(MI, MRI, B);
2477 case TargetOpcode::G_SET_FPENV:
2478 return legalizeSetFPEnv(MI, MRI, B);
2479 case TargetOpcode::G_TRAP:
2480 return legalizeTrap(Helper, MI);
2481 case TargetOpcode::G_DEBUGTRAP:
2482 return legalizeDebugTrap(MI, MRI, B);
2483 default:
2484 return false;
2485 }
2486
2487 llvm_unreachable("expected switch to return");
2488}
2489
2492 MachineIRBuilder &B) const {
2493 unsigned BaseAS = AS;
2494 unsigned SANum = AMDGPU::getSyntheticApertureNumber(AS);
2496 BaseAS = AMDGPUAS::LOCAL_ADDRESS;
2497
2498 Register Aperture = getBaseSegmentAperture(BaseAS, MRI, B);
2499
2500 if (SANum != AMDGPU::SyntheticAperture::None) {
2501 const LLT S32 = LLT::scalar(32);
2502 auto Tag = B.buildConstant(S32, SANum);
2503 return B.buildOr(S32, Aperture, Tag).getReg(0);
2504 }
2505
2506 return Aperture;
2507}
2508
2509Register AMDGPULegalizerInfo::getBaseSegmentAperture(
2510 unsigned AS, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
2511 MachineFunction &MF = B.getMF();
2512 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2513 const LLT I32 = LLT::integer(32);
2514 const LLT I64 = LLT::integer(64);
2515
2516 bool IsLDS = (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER);
2517 assert(IsLDS || AS == AMDGPUAS::PRIVATE_ADDRESS);
2518
2519 if (ST.hasApertureRegs()) {
2520 // Note: this register is somewhat broken. When used as a 32-bit operand,
2521 // it only returns zeroes. The real value is in the upper 32 bits.
2522 // Thus, we must emit extract the high 32 bits.
2523 const unsigned ApertureRegNo =
2524 IsLDS ? AMDGPU::SRC_SHARED_BASE : AMDGPU::SRC_PRIVATE_BASE;
2525 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
2526 !ST.hasGloballyAddressableScratch()) &&
2527 "Cannot use src_private_base with globally addressable scratch!");
2528 Register Dst = MRI.createGenericVirtualRegister(I64);
2529 MRI.setRegClass(Dst, &AMDGPU::SReg_64RegClass);
2530 B.buildCopy({Dst}, {Register(ApertureRegNo)});
2531 return B.buildUnmerge(I32, Dst).getReg(1);
2532 }
2533
2536 // For code object version 5, private_base and shared_base are passed through
2537 // implicit kernargs.
2540 MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(B.getMF());
2541
2546 ST.getTargetLowering()->getImplicitParameterOffset(B.getMF(), Param);
2547
2550
2551 if (!loadInputValue(KernargPtrReg, B,
2553 return Register();
2554
2555 MachineMemOperand *MMO = MF.getMachineMemOperand(
2556 PtrInfo.getWithOffset(Offset),
2560
2561 // Pointer address
2562 B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
2563 B.buildConstant(LLT::integer(64), Offset).getReg(0));
2564 // Load address
2565 return B.buildLoad(I32, LoadAddr, *MMO).getReg(0);
2566 }
2567
2570
2572 return Register();
2573
2574 // TODO: Use custom PseudoSourceValue
2575 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
2576
2577 // Offset into amd_queue_t for group_segment_aperture_base_hi /
2578 // private_segment_aperture_base_hi.
2579 uint32_t StructOffset = IsLDS ? 0x40 : 0x44;
2580
2581 MachineMemOperand *MMO = MF.getMachineMemOperand(
2582 PtrInfo,
2585 LLT::integer(32), commonAlignment(Align(64), StructOffset));
2586
2587 B.buildObjectPtrOffset(
2588 LoadAddr, QueuePtr,
2589 B.buildConstant(LLT::integer(64), StructOffset).getReg(0));
2590 return B.buildLoad(I32, LoadAddr, *MMO).getReg(0);
2591}
2592
2593/// Return true if the value is a known valid address, such that a null check is
2594/// not necessary.
2596 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
2597 MachineInstr *Def = MRI.getVRegDef(Val);
2598 switch (Def->getOpcode()) {
2599 case AMDGPU::G_FRAME_INDEX:
2600 case AMDGPU::G_GLOBAL_VALUE:
2601 case AMDGPU::G_BLOCK_ADDR:
2602 return true;
2603 case AMDGPU::G_CONSTANT: {
2604 const ConstantInt *CI = Def->getOperand(1).getCImm();
2605 return CI->getSExtValue() != AMDGPU::getNullPointerValue(AddrSpace);
2606 }
2607 default:
2608 return false;
2609 }
2610
2611 return false;
2612}
2613
2616 MachineIRBuilder &B) const {
2617 MachineFunction &MF = B.getMF();
2618
2619 assert(MI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
2620
2621 const LLT I32 = LLT::integer(32);
2622 const LLT I64 = LLT::integer(64);
2623 Register Dst = MI.getOperand(0).getReg();
2624 Register Src = MI.getOperand(1).getReg();
2625 LLT DstTy = MRI.getType(Dst);
2626 LLT SrcTy = MRI.getType(Src);
2627 unsigned DestAS = DstTy.getAddressSpace();
2628 unsigned SrcAS = SrcTy.getAddressSpace();
2629
2630 // TODO: Avoid reloading from the queue ptr for each cast, or at least each
2631 // vector element.
2632 assert(!DstTy.isVector());
2633
2634 const AMDGPUTargetMachine &TM
2635 = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
2636
2637 // The source is known non-null for a G_ADDRSPACE_CAST carrying the nonnull
2638 // flag; otherwise we need to guess.
2639 const bool IsNonNull = MI.getFlag(MachineInstr::MIFlag::NonNull);
2640
2641 if (TM.isNoopAddrSpaceCast(MF.getDataLayout(), SrcAS, DestAS)) {
2642 MI.setDesc(B.getTII().get(TargetOpcode::G_BITCAST));
2643 return true;
2644 }
2645
2646 if (SrcAS == AMDGPUAS::FLAT_ADDRESS &&
2647 (DestAS == AMDGPUAS::LOCAL_ADDRESS || DestAS == AMDGPUAS::BARRIER ||
2648 DestAS == AMDGPUAS::PRIVATE_ADDRESS || DestAS == AMDGPUAS::VGPR)) {
2649 auto castFlatToLocalOrPrivate = [&](const DstOp &Dst) -> Register {
2650 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2651 ST.hasGloballyAddressableScratch()) {
2652 // flat -> private with globally addressable scratch: subtract
2653 // src_flat_scratch_base_lo.
2654 Register SrcLo = B.buildExtract(I32, Src, 0).getReg(0);
2655 Register FlatScratchBaseLo =
2656 B.buildInstr(AMDGPU::S_MOV_B32, {I32},
2657 {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)})
2658 .getReg(0);
2659 MRI.setRegClass(FlatScratchBaseLo, &AMDGPU::SReg_32RegClass);
2660 Register Sub = B.buildSub(I32, SrcLo, FlatScratchBaseLo).getReg(0);
2661 return B.buildIntToPtr(Dst, Sub).getReg(0);
2662 }
2663
2664 return B.buildExtract(Dst, Src, 0).getReg(0);
2665 };
2666
2667 if (IsNonNull || isKnownNonNull(Src, MRI, TM, SrcAS)) {
2668 castFlatToLocalOrPrivate(Dst);
2669 MI.eraseFromParent();
2670 return true;
2671 }
2672
2673 unsigned NullVal = AMDGPU::getNullPointerValue(DestAS);
2674
2675 auto SegmentNull = B.buildConstant(DstTy, NullVal);
2676 auto FlatNull = B.buildConstant(SrcTy, 0);
2677
2678 // Extract low 32-bits of the pointer.
2679 auto PtrLo32 = castFlatToLocalOrPrivate(DstTy);
2680
2681 auto CmpRes =
2682 B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, FlatNull.getReg(0));
2683 B.buildSelect(Dst, CmpRes, PtrLo32, SegmentNull.getReg(0));
2684
2685 MI.eraseFromParent();
2686 return true;
2687 }
2688
2689 if (DestAS == AMDGPUAS::FLAT_ADDRESS &&
2690 (SrcAS == AMDGPUAS::LOCAL_ADDRESS || SrcAS == AMDGPUAS::BARRIER ||
2691 SrcAS == AMDGPUAS::PRIVATE_ADDRESS || SrcAS == AMDGPUAS::VGPR)) {
2692 auto castLocalOrPrivateToFlat = [&](const DstOp &Dst) -> Register {
2693 // Coerce the type of the low half of the result so we can use
2694 // merge_values.
2695 Register SrcAsInt = B.buildPtrToInt(I32, Src).getReg(0);
2696
2697 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
2698 ST.hasGloballyAddressableScratch()) {
2699 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
2700 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
2701 Register AllOnes = B.buildConstant(I32, -1).getReg(0);
2702 Register ThreadID = B.buildConstant(I32, 0).getReg(0);
2703 ThreadID = B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {I32})
2704 .addUse(AllOnes)
2705 .addUse(ThreadID)
2706 .getReg(0);
2707 if (ST.isWave64()) {
2708 ThreadID = B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {I32})
2709 .addUse(AllOnes)
2710 .addUse(ThreadID)
2711 .getReg(0);
2712 }
2713 Register ShAmt =
2714 B.buildConstant(I32, 57 - 32 - ST.getWavefrontSizeLog2()).getReg(0);
2715 Register SrcHi = B.buildShl(I32, ThreadID, ShAmt).getReg(0);
2716 Register CvtPtr =
2717 B.buildMergeLikeInstr(DstTy, {SrcAsInt, SrcHi}).getReg(0);
2718 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
2719 // 64-bit hi:lo value.
2720 Register FlatScratchBase =
2721 B.buildInstr(AMDGPU::S_MOV_B64, {I64},
2722 {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE)})
2723 .getReg(0);
2724 MRI.setRegClass(FlatScratchBase, &AMDGPU::SReg_64RegClass);
2725 return B.buildPtrAdd(Dst, CvtPtr, FlatScratchBase).getReg(0);
2726 }
2727
2728 Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
2729 if (!ApertureReg.isValid())
2730 return false;
2731
2732 // TODO: Should we allow mismatched types but matching sizes in merges to
2733 // avoid the ptrtoint?
2734 return B.buildMergeLikeInstr(Dst, {SrcAsInt, ApertureReg}).getReg(0);
2735 };
2736
2737 if (IsNonNull || isKnownNonNull(Src, MRI, TM, SrcAS)) {
2738 castLocalOrPrivateToFlat(Dst);
2739 MI.eraseFromParent();
2740 return true;
2741 }
2742
2743 Register BuildPtr = castLocalOrPrivateToFlat(DstTy);
2744
2745 auto SegmentNull =
2746 B.buildConstant(SrcTy, AMDGPU::getNullPointerValue(SrcAS));
2747 auto FlatNull = B.buildConstant(DstTy, AMDGPU::getNullPointerValue(DestAS));
2748
2749 auto CmpRes = B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src,
2750 SegmentNull.getReg(0));
2751
2752 B.buildSelect(Dst, CmpRes, BuildPtr, FlatNull);
2753
2754 MI.eraseFromParent();
2755 return true;
2756 }
2757
2758 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
2759 SrcTy.getSizeInBits() == 64) {
2760 // Truncate.
2761 B.buildExtract(Dst, Src, 0);
2762 MI.eraseFromParent();
2763 return true;
2764 }
2765
2766 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
2767 DstTy.getSizeInBits() == 64) {
2769 uint32_t AddrHiVal = Info->get32BitAddressHighBits();
2770 auto PtrLo = B.buildPtrToInt(I32, Src);
2771 if (AddrHiVal == 0) {
2772 auto Zext = B.buildZExt(I64, PtrLo);
2773 B.buildIntToPtr(Dst, Zext);
2774 } else {
2775 auto HighAddr = B.buildConstant(I32, AddrHiVal);
2776 B.buildMergeLikeInstr(Dst, {PtrLo, HighAddr});
2777 }
2778
2779 MI.eraseFromParent();
2780 return true;
2781 }
2782
2783 // Invalid casts are poison.
2784 // TODO: Should return poison
2785 B.buildUndef(Dst);
2786 MI.eraseFromParent();
2787 return true;
2788}
2789
2792 MachineIRBuilder &B) const {
2793 Register Src = MI.getOperand(1).getReg();
2794 LLT Ty = MRI.getType(Src);
2795 assert(Ty.isScalar() && Ty.getSizeInBits() == 64);
2796
2797 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2798 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2799
2800 auto C1 = B.buildFConstant(Ty, C1Val);
2801 auto CopySign = B.buildFCopysign(Ty, C1, Src);
2802
2803 // TODO: Should this propagate fast-math-flags?
2804 auto Tmp1 = B.buildFAdd(Ty, Src, CopySign);
2805 auto Tmp2 = B.buildFSub(Ty, Tmp1, CopySign);
2806
2807 auto C2 = B.buildFConstant(Ty, C2Val);
2808 auto Fabs = B.buildFAbs(Ty, Src);
2809
2810 auto Cond = B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2);
2811 B.buildSelect(MI.getOperand(0).getReg(), Cond, Src, Tmp2);
2812 MI.eraseFromParent();
2813 return true;
2814}
2815
2818 MachineIRBuilder &B) const {
2819
2820 const LLT S1 = LLT::scalar(1);
2821
2822 Register Src = MI.getOperand(1).getReg();
2823 assert(MRI.getType(Src) == F64);
2824
2825 // result = trunc(src)
2826 // if (src > 0.0 && src != result)
2827 // result += 1.0
2828
2829 auto Trunc = B.buildIntrinsicTrunc(F64, Src);
2830
2831 const auto Zero = B.buildFConstant(F64, 0.0);
2832 const auto One = B.buildFConstant(F64, 1.0);
2833 auto Lt0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, Src, Zero);
2834 auto NeTrunc = B.buildFCmp(CmpInst::FCMP_ONE, S1, Src, Trunc);
2835 auto And = B.buildAnd(S1, Lt0, NeTrunc);
2836 auto Add = B.buildSelect(F64, And, One, Zero);
2837
2838 // TODO: Should this propagate fast-math-flags?
2839 B.buildFAdd(MI.getOperand(0).getReg(), Trunc, Add);
2840 MI.eraseFromParent();
2841 return true;
2842}
2843
2846 MachineIRBuilder &B) const {
2847 Register DstReg = MI.getOperand(0).getReg();
2848 Register Src0Reg = MI.getOperand(1).getReg();
2849 Register Src1Reg = MI.getOperand(2).getReg();
2850 auto Flags = MI.getFlags();
2851 LLT Ty = MRI.getType(DstReg);
2852
2853 auto Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, Flags);
2854 auto Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags);
2855 auto Neg = B.buildFNeg(Ty, Trunc, Flags);
2856 B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags);
2857 MI.eraseFromParent();
2858 return true;
2859}
2860
2863 const unsigned FractBits = 52;
2864 const unsigned ExpBits = 11;
2865 LLT I32 = LLT::integer(32);
2866
2867 auto Const0 = B.buildConstant(I32, FractBits - 32);
2868 auto Const1 = B.buildConstant(I32, ExpBits);
2869
2870 auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {I32})
2871 .addUse(Hi)
2872 .addUse(Const0.getReg(0))
2873 .addUse(Const1.getReg(0));
2874
2875 return B.buildSub(I32, ExpPart, B.buildConstant(I32, 1023));
2876}
2877
2880 MachineIRBuilder &B) const {
2881 const LLT S1 = LLT::scalar(1);
2882 const LLT I32 = LLT::integer(32);
2883 const LLT I64 = LLT::integer(64);
2884
2885 Register Src = MI.getOperand(1).getReg();
2886 assert(MRI.getType(Src) == F64);
2887
2888 auto SrcInt = B.buildBitcast(I64, Src);
2889
2890 // TODO: Should this use extract since the low half is unused?
2891 auto Unmerge = B.buildUnmerge({I32, I32}, SrcInt);
2892 Register Hi = Unmerge.getReg(1);
2893
2894 // Extract the upper half, since this is where we will find the sign and
2895 // exponent.
2896 auto Exp = extractF64Exponent(Hi, B);
2897
2898 const unsigned FractBits = 52;
2899
2900 // Extract the sign bit.
2901 const auto SignBitMask = B.buildConstant(I32, UINT32_C(1) << 31);
2902 auto SignBit = B.buildAnd(I32, Hi, SignBitMask);
2903
2904 const auto FractMask = B.buildConstant(I64, (UINT64_C(1) << FractBits) - 1);
2905
2906 const auto Zero32 = B.buildConstant(I32, 0);
2907
2908 // Extend back to 64-bits.
2909 auto SignBit64 = B.buildMergeLikeInstr(I64, {Zero32, SignBit});
2910
2911 auto Shr = B.buildAShr(I64, FractMask, Exp);
2912 auto Not = B.buildNot(I64, Shr);
2913 auto Tmp0 = B.buildAnd(I64, SrcInt, Not);
2914 auto FiftyOne = B.buildConstant(I32, FractBits - 1);
2915
2916 auto ExpLt0 = B.buildICmp(CmpInst::ICMP_SLT, S1, Exp, Zero32);
2917 auto ExpGt51 = B.buildICmp(CmpInst::ICMP_SGT, S1, Exp, FiftyOne);
2918
2919 auto Tmp1 = B.buildSelect(I64, ExpLt0, SignBit64, Tmp0);
2920 auto Res = B.buildSelect(I64, ExpGt51, SrcInt, Tmp1);
2921 B.buildBitcast(MI.getOperand(0).getReg(), Res);
2922 MI.eraseFromParent();
2923 return true;
2924}
2925
2928 MachineIRBuilder &B, bool Signed) const {
2929
2930 Register Dst = MI.getOperand(0).getReg();
2931 Register Src = MI.getOperand(1).getReg();
2932
2933 const LLT I64 = LLT::integer(64);
2934 const LLT I32 = LLT::integer(32);
2935
2936 assert(MRI.getType(Src) == I64);
2937
2938 auto Unmerge = B.buildUnmerge({I32, I32}, Src);
2939 auto ThirtyTwo = B.buildConstant(I32, 32);
2940
2941 if (MRI.getType(Dst) == F64) {
2942 auto CvtHi = Signed ? B.buildSITOFP(F64, Unmerge.getReg(1))
2943 : B.buildUITOFP(F64, Unmerge.getReg(1));
2944
2945 auto CvtLo = B.buildUITOFP(F64, Unmerge.getReg(0));
2946 auto LdExp = B.buildFLdexp(F64, CvtHi, ThirtyTwo);
2947
2948 // TODO: Should this propagate fast-math-flags?
2949 B.buildFAdd(Dst, LdExp, CvtLo);
2950 MI.eraseFromParent();
2951 return true;
2952 }
2953
2954 assert(MRI.getType(Dst) == F32);
2955
2956 auto One = B.buildConstant(I32, 1);
2957
2958 MachineInstrBuilder ShAmt;
2959 if (Signed) {
2960 auto ThirtyOne = B.buildConstant(I32, 31);
2961 auto X = B.buildXor(I32, Unmerge.getReg(0), Unmerge.getReg(1));
2962 auto OppositeSign = B.buildAShr(I32, X, ThirtyOne);
2963 auto MaxShAmt = B.buildAdd(I32, ThirtyTwo, OppositeSign);
2964 auto LS = B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {I32})
2965 .addUse(Unmerge.getReg(1));
2966 auto LS2 = B.buildSub(I32, LS, One);
2967 ShAmt = B.buildUMin(I32, LS2, MaxShAmt);
2968 } else
2969 ShAmt = B.buildCTLZ(I32, Unmerge.getReg(1));
2970 auto Norm = B.buildShl(I64, Src, ShAmt);
2971 auto Unmerge2 = B.buildUnmerge({I32, I32}, Norm);
2972 auto Adjust = B.buildUMin(I32, One, Unmerge2.getReg(0));
2973 auto Norm2 = B.buildOr(I32, Unmerge2.getReg(1), Adjust);
2974 auto FVal = Signed ? B.buildSITOFP(F32, Norm2) : B.buildUITOFP(F32, Norm2);
2975 auto Scale = B.buildSub(I32, ThirtyTwo, ShAmt);
2976 B.buildFLdexp(Dst, FVal, Scale);
2977 MI.eraseFromParent();
2978 return true;
2979}
2980
2981// TODO: Copied from DAG implementation. Verify logic and document how this
2982// actually works.
2986 bool Signed) const {
2987
2988 Register Dst = MI.getOperand(0).getReg();
2989 Register Src = MI.getOperand(1).getReg();
2990
2991 const LLT I64 = LLT::integer(64);
2992 const LLT I32 = LLT::integer(32);
2993
2994 const LLT SrcLT = MRI.getType(Src);
2995 assert((SrcLT == F32 || SrcLT == F64) && MRI.getType(Dst) == I64);
2996
2997 unsigned Flags = MI.getFlags();
2998
2999 // The basic idea of converting a floating point number into a pair of 32-bit
3000 // integers is illustrated as follows:
3001 //
3002 // tf := trunc(val);
3003 // hif := floor(tf * 2^-32);
3004 // lof := tf - hif * 2^32; // lof is always positive due to floor.
3005 // hi := fptoi(hif);
3006 // lo := fptoi(lof);
3007 //
3008 auto Trunc = B.buildIntrinsicTrunc(SrcLT, Src, Flags);
3010 if (Signed && SrcLT == F32) {
3011 // However, a 32-bit floating point number has only 23 bits mantissa and
3012 // it's not enough to hold all the significant bits of `lof` if val is
3013 // negative. To avoid the loss of precision, We need to take the absolute
3014 // value after truncating and flip the result back based on the original
3015 // signedness.
3016 auto SrcInt = B.buildBitcast(I32, Src);
3017 Sign = B.buildAShr(I32, SrcInt, B.buildConstant(I32, 31));
3018 Trunc = B.buildFAbs(F32, Trunc, Flags);
3019 }
3020 MachineInstrBuilder K0, K1;
3021 if (SrcLT == F64) {
3022 K0 = B.buildFConstant(
3023 F64, llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)));
3024 K1 = B.buildFConstant(
3025 F64, llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)));
3026 } else {
3027 K0 = B.buildFConstant(
3028 F32, llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)));
3029 K1 = B.buildFConstant(
3030 F32, llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)));
3031 }
3032
3033 auto Mul = B.buildFMul(SrcLT, Trunc, K0, Flags);
3034 auto FloorMul = B.buildFFloor(SrcLT, Mul, Flags);
3035 auto Fma = B.buildFMA(SrcLT, FloorMul, K1, Trunc, Flags);
3036
3037 auto Hi = (Signed && SrcLT == F64) ? B.buildFPTOSI(I32, FloorMul)
3038 : B.buildFPTOUI(I32, FloorMul);
3039 auto Lo = B.buildFPTOUI(I32, Fma);
3040
3041 if (Signed && SrcLT == F32) {
3042 // Flip the result based on the signedness, which is either all 0s or 1s.
3043 Sign = B.buildMergeLikeInstr(I64, {Sign, Sign});
3044 // r := xor({lo, hi}, sign) - sign;
3045 B.buildSub(Dst, B.buildXor(I64, B.buildMergeLikeInstr(I64, {Lo, Hi}), Sign),
3046 Sign);
3047 } else
3048 B.buildMergeLikeInstr(Dst, {Lo, Hi});
3049 MI.eraseFromParent();
3050
3051 return true;
3052}
3053
3055 MachineInstr &MI) const {
3056 MachineFunction &MF = Helper.MIRBuilder.getMF();
3058
3059 // With ieee_mode disabled, the instructions have the correct behavior.
3060 if (!MFI->getMode().IEEE)
3061 return true;
3062
3064}
3065
3067 MachineInstr &MI) const {
3068 MachineIRBuilder &B = Helper.MIRBuilder;
3069 MachineRegisterInfo &MRI = *B.getMRI();
3070 Register DstReg = MI.getOperand(0).getReg();
3071 Register SrcReg = MI.getOperand(1).getReg();
3072 uint64_t Offset = MI.getOperand(2).getImm();
3073
3074 // Fall back to generic lowering for offset 0 (trivial trunc) and
3075 // non-32-bit-aligned cases which require shift+trunc sequences
3076 // that generic code handles correctly.
3077 if (Offset == 0 || Offset % 32 != 0)
3078 return Helper.lowerExtract(MI) == LegalizerHelper::Legalized;
3079
3080 const LLT DstTy = MRI.getType(DstReg);
3081 unsigned StartIdx = Offset / 32;
3082 unsigned DstCount = DstTy.getSizeInBits() / 32;
3083 auto Unmerge = B.buildUnmerge(LLT::integer(32), SrcReg);
3084
3085 if (DstCount == 1) {
3086 if (DstTy.isPointer())
3087 B.buildIntToPtr(DstReg, Unmerge.getReg(StartIdx));
3088 else {
3089 Helper.Observer.changingAllUsesOfReg(MRI, DstReg);
3090 MRI.replaceRegWith(DstReg, Unmerge.getReg(StartIdx));
3092 }
3093 } else {
3094 SmallVector<Register, 8> MergeVec;
3095 for (unsigned I = 0; I < DstCount; ++I)
3096 MergeVec.push_back(Unmerge.getReg(StartIdx + I));
3097 B.buildMergeLikeInstr(DstReg, MergeVec);
3098 }
3099
3100 MI.eraseFromParent();
3101 return true;
3102}
3103
3105 MachineInstr &MI) const {
3106 MachineIRBuilder &B = Helper.MIRBuilder;
3107 MachineRegisterInfo &MRI = *B.getMRI();
3108 Register DstReg = MI.getOperand(0).getReg();
3109 Register SrcReg = MI.getOperand(1).getReg();
3110 Register InsertSrc = MI.getOperand(2).getReg();
3111 uint64_t Offset = MI.getOperand(3).getImm();
3112
3113 unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
3114 const LLT InsertTy = MRI.getType(InsertSrc);
3115 unsigned InsertSize = InsertTy.getSizeInBits();
3116
3117 // Fall back to generic lowering for non-32-bit-aligned cases which
3118 // require shift+mask sequences that generic code handles correctly.
3119 if (Offset % 32 != 0 || DstSize % 32 != 0 || InsertSize % 32 != 0)
3120 return Helper.lowerInsert(MI) == LegalizerHelper::Legalized;
3121
3122 const LLT I32 = LLT::integer(32);
3123 unsigned DstCount = DstSize / 32;
3124 unsigned InsertCount = InsertSize / 32;
3125 unsigned StartIdx = Offset / 32;
3126
3127 auto SrcUnmerge = B.buildUnmerge(I32, SrcReg);
3128
3129 SmallVector<Register, 8> MergeVec;
3130 for (unsigned I = 0; I < StartIdx; ++I)
3131 MergeVec.push_back(SrcUnmerge.getReg(I));
3132
3133 if (InsertCount == 1) {
3134 // Merge-like instructions require same source types. Convert pointer
3135 // to scalar when inserting a pointer value into a scalar.
3136 if (InsertTy.isPointer())
3137 InsertSrc = B.buildPtrToInt(I32, InsertSrc).getReg(0);
3138 MergeVec.push_back(InsertSrc);
3139 } else {
3140 auto InsertUnmerge = B.buildUnmerge(I32, InsertSrc);
3141 for (unsigned I = 0; I < InsertCount; ++I)
3142 MergeVec.push_back(InsertUnmerge.getReg(I));
3143 }
3144
3145 for (unsigned I = StartIdx + InsertCount; I < DstCount; ++I)
3146 MergeVec.push_back(SrcUnmerge.getReg(I));
3147
3148 B.buildMergeLikeInstr(DstReg, MergeVec);
3149
3150 MI.eraseFromParent();
3151 return true;
3152}
3153
3156 MachineIRBuilder &B) const {
3157 // TODO: Should move some of this into LegalizerHelper.
3158
3159 // TODO: Promote dynamic indexing of i16/f16 to i32/f32
3160
3161 Register Dst = MI.getOperand(0).getReg();
3162 Register Vec = MI.getOperand(1).getReg();
3163
3164 LLT VecTy = MRI.getType(Vec);
3165 LLT EltTy = VecTy.getElementType();
3166 assert(EltTy == MRI.getType(Dst));
3167
3168 // Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
3169 // but we can't go directly to that logic becasue you can't bitcast a vector
3170 // of pointers to a vector of integers. Therefore, introduce an intermediate
3171 // vector of integers using ptrtoint (and inttoptr on the output) in order to
3172 // drive the legalization forward.
3173 if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
3174 LLT IntTy = LLT::integer(EltTy.getSizeInBits());
3175 LLT IntVecTy = VecTy.changeElementType(IntTy);
3176
3177 auto IntVec = B.buildPtrToInt(IntVecTy, Vec);
3178 auto IntElt = B.buildExtractVectorElement(IntTy, IntVec, MI.getOperand(2));
3179 B.buildIntToPtr(Dst, IntElt);
3180
3181 MI.eraseFromParent();
3182 return true;
3183 }
3184
3185 // FIXME: Artifact combiner probably should have replaced the truncated
3186 // constant before this, so we shouldn't need
3187 // getIConstantVRegValWithLookThrough.
3188 std::optional<ValueAndVReg> MaybeIdxVal =
3189 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
3190 if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
3191 return true;
3192 const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
3193
3194 if (IdxVal < VecTy.getNumElements()) {
3195 auto Unmerge = B.buildUnmerge(EltTy, Vec);
3196 B.buildCopy(Dst, Unmerge.getReg(IdxVal));
3197 } else {
3198 B.buildUndef(Dst);
3199 }
3200
3201 MI.eraseFromParent();
3202 return true;
3203}
3204
3207 MachineIRBuilder &B) const {
3208 // TODO: Should move some of this into LegalizerHelper.
3209
3210 // TODO: Promote dynamic indexing of i16/f16 to i32/f32
3211
3212 Register Dst = MI.getOperand(0).getReg();
3213 Register Vec = MI.getOperand(1).getReg();
3214 Register Ins = MI.getOperand(2).getReg();
3215
3216 LLT VecTy = MRI.getType(Vec);
3217 LLT EltTy = VecTy.getElementType();
3218 assert(EltTy == MRI.getType(Ins));
3219
3220 // Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
3221 // but we can't go directly to that logic becasue you can't bitcast a vector
3222 // of pointers to a vector of integers. Therefore, make the pointer vector
3223 // into an equivalent vector of integers with ptrtoint, insert the ptrtoint'd
3224 // new value, and then inttoptr the result vector back. This will then allow
3225 // the rest of legalization to take over.
3226 if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
3227 LLT IntTy = LLT::integer(EltTy.getSizeInBits());
3228 LLT IntVecTy = VecTy.changeElementType(IntTy);
3229
3230 auto IntVecSource = B.buildPtrToInt(IntVecTy, Vec);
3231 auto IntIns = B.buildPtrToInt(IntTy, Ins);
3232 auto IntVecDest = B.buildInsertVectorElement(IntVecTy, IntVecSource, IntIns,
3233 MI.getOperand(3));
3234 B.buildIntToPtr(Dst, IntVecDest);
3235 MI.eraseFromParent();
3236 return true;
3237 }
3238
3239 // FIXME: Artifact combiner probably should have replaced the truncated
3240 // constant before this, so we shouldn't need
3241 // getIConstantVRegValWithLookThrough.
3242 std::optional<ValueAndVReg> MaybeIdxVal =
3243 getIConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
3244 if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
3245 return true;
3246
3247 const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
3248
3249 unsigned NumElts = VecTy.getNumElements();
3250 if (IdxVal < NumElts) {
3252 for (unsigned i = 0; i < NumElts; ++i)
3253 SrcRegs.push_back(MRI.createGenericVirtualRegister(EltTy));
3254 B.buildUnmerge(SrcRegs, Vec);
3255
3256 SrcRegs[IdxVal] = MI.getOperand(2).getReg();
3257 B.buildMergeLikeInstr(Dst, SrcRegs);
3258 } else {
3259 B.buildUndef(Dst);
3260 }
3261
3262 MI.eraseFromParent();
3263 return true;
3264}
3265
3268 MachineIRBuilder &B) const {
3269
3270 Register DstReg = MI.getOperand(0).getReg();
3271 Register SrcReg = MI.getOperand(1).getReg();
3272 LLT Ty = MRI.getType(DstReg);
3273 unsigned Flags = MI.getFlags();
3274
3275 Register TrigVal;
3276 auto OneOver2Pi = B.buildFConstant(Ty, 0.5 * numbers::inv_pi);
3277 if (ST.hasTrigReducedRange()) {
3278 auto MulVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags);
3279 TrigVal = B.buildIntrinsic(Intrinsic::amdgcn_fract, {Ty})
3280 .addUse(MulVal.getReg(0))
3281 .setMIFlags(Flags)
3282 .getReg(0);
3283 } else
3284 TrigVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags).getReg(0);
3285
3286 Intrinsic::ID TrigIntrin = MI.getOpcode() == AMDGPU::G_FSIN ?
3287 Intrinsic::amdgcn_sin : Intrinsic::amdgcn_cos;
3288 B.buildIntrinsic(TrigIntrin, ArrayRef<Register>(DstReg))
3289 .addUse(TrigVal)
3290 .setMIFlags(Flags);
3291 MI.eraseFromParent();
3292 return true;
3293}
3294
3297 const GlobalValue *GV,
3298 int64_t Offset,
3299 unsigned GAFlags) const {
3300 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
3301 // In order to support pc-relative addressing, SI_PC_ADD_REL_OFFSET is lowered
3302 // to the following code sequence:
3303 //
3304 // For constant address space:
3305 // s_getpc_b64 s[0:1]
3306 // s_add_u32 s0, s0, $symbol
3307 // s_addc_u32 s1, s1, 0
3308 //
3309 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
3310 // a fixup or relocation is emitted to replace $symbol with a literal
3311 // constant, which is a pc-relative offset from the encoding of the $symbol
3312 // operand to the global variable.
3313 //
3314 // For global address space:
3315 // s_getpc_b64 s[0:1]
3316 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
3317 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
3318 //
3319 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
3320 // fixups or relocations are emitted to replace $symbol@*@lo and
3321 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
3322 // which is a 64-bit pc-relative offset from the encoding of the $symbol
3323 // operand to the global variable.
3324
3325 MachineRegisterInfo &MRI = *B.getMRI();
3326 LLT PCRegTy = PtrTy.getSizeInBits() == 32
3328 : PtrTy;
3329 Register PCReg =
3330 MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, PCRegTy});
3331
3332 if (ST.has64BitLiterals()) {
3333 assert(GAFlags != SIInstrInfo::MO_NONE);
3334
3336 B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET64).addDef(PCReg);
3337 MIB.addGlobalAddress(GV, Offset, GAFlags + 2);
3338 } else {
3340 B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET).addDef(PCReg);
3341
3342 MIB.addGlobalAddress(GV, Offset, GAFlags);
3343 if (GAFlags == SIInstrInfo::MO_NONE)
3344 MIB.addImm(0);
3345 else
3346 MIB.addGlobalAddress(GV, Offset, GAFlags + 1);
3347 MIB.setOperandDead(3); // $scc
3348 }
3349
3350 if (PtrTy.getSizeInBits() == 32)
3351 B.buildExtract(DstReg, PCReg, 0);
3352 else
3353 B.buildCopy(DstReg, PCReg);
3354 return true;
3355}
3356
3357// Emit a ABS32_LO / ABS32_HI relocation stub.
3359 Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV,
3360 MachineRegisterInfo &MRI) const {
3361 bool RequiresHighHalf = PtrTy.getSizeInBits() != 32;
3362
3363 if (RequiresHighHalf && ST.has64BitLiterals()) {
3364 Register Addr =
3365 MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, PtrTy});
3366 B.buildInstr(AMDGPU::S_MOV_B64)
3367 .addDef(Addr)
3368 .addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS64);
3369 B.buildCopy(DstReg, Addr);
3370 return;
3371 }
3372
3373 LLT I32 = LLT::integer(32);
3374 Register AddrLo = MRI.createVirtualRegister({&AMDGPU::SReg_32RegClass, I32});
3375
3376 // Write the lower half.
3377 B.buildInstr(AMDGPU::S_MOV_B32)
3378 .addDef(AddrLo)
3379 .addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_LO);
3380
3381 // If required, write the upper half as well.
3382 if (RequiresHighHalf) {
3383 assert(PtrTy.getSizeInBits() == 64 &&
3384 "Must provide a 64-bit pointer type!");
3385
3386 Register AddrHi =
3387 MRI.createVirtualRegister({&AMDGPU::SReg_32RegClass, I32});
3388
3389 B.buildInstr(AMDGPU::S_MOV_B32)
3390 .addDef(AddrHi)
3391 .addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_HI);
3392
3393 Register AddrDst =
3394 MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, LLT::integer(64)});
3395
3396 B.buildMergeValues(AddrDst, {AddrLo, AddrHi});
3397 B.buildCast(DstReg, AddrDst);
3398 } else {
3399 B.buildCast(DstReg, AddrLo);
3400 }
3401}
3402
3405 MachineIRBuilder &B) const {
3406 Register DstReg = MI.getOperand(0).getReg();
3407 LLT Ty = MRI.getType(DstReg);
3408 unsigned AS = Ty.getAddressSpace();
3409
3410 const GlobalValue *GV = MI.getOperand(1).getGlobal();
3411 MachineFunction &MF = B.getMF();
3413
3414 if (AS == AMDGPUAS::BARRIER) {
3415 const GlobalVariable *GVar = cast<GlobalVariable>(GV);
3416 if (!AMDGPU::isNamedBarrier(*GVar)) {
3417 const Function &Fn = MF.getFunction();
3419 Fn, "unsupported use of BARRIER address space", MI.getDebugLoc(),
3420 DS_Error));
3421 B.buildUndef(DstReg);
3422 MI.eraseFromParent();
3423 return true;
3424 }
3425
3426 B.buildConstant(DstReg,
3427 MFI->allocateBarrierGlobal(B.getDataLayout(), *GVar));
3428 MI.eraseFromParent();
3429 return true;
3430 }
3431
3433 if (!MFI->isModuleEntryFunction() &&
3434 GV->getName() != "llvm.amdgcn.module.lds") {
3435 const Function &Fn = MF.getFunction();
3437 Fn, "local memory global used by non-kernel function",
3438 MI.getDebugLoc(), DS_Warning));
3439
3440 // We currently don't have a way to correctly allocate LDS objects that
3441 // aren't directly associated with a kernel. We do force inlining of
3442 // functions that use local objects. However, if these dead functions are
3443 // not eliminated, we don't want a compile time error. Just emit a warning
3444 // and a trap, since there should be no callable path here.
3445 B.buildTrap();
3446 B.buildUndef(DstReg);
3447 MI.eraseFromParent();
3448 return true;
3449 }
3450
3451 // TODO: We could emit code to handle the initialization somewhere.
3452 // We ignore the initializer for now and legalize it to allow selection.
3453 // The initializer will anyway get errored out during assembly emission.
3454 const SITargetLowering *TLI = ST.getTargetLowering();
3455 if (!TLI->shouldUseLDSConstAddress(GV)) {
3456 MI.getOperand(1).setTargetFlags(SIInstrInfo::MO_ABS32_LO);
3457 return true; // Leave in place;
3458 }
3459
3460 const GlobalVariable &GVar = *cast<GlobalVariable>(GV);
3461 if (AS == AMDGPUAS::LOCAL_ADDRESS && GV->hasExternalLinkage()) {
3462 // HIP uses an unsized array `extern __shared__ T s[]` or similar
3463 // zero-sized type in other languages to declare the dynamic shared
3464 // memory which size is not known at the compile time. They will be
3465 // allocated by the runtime and placed directly after the static
3466 // allocated ones. They all share the same offset.
3467 if (GVar.getGlobalSize(GVar.getDataLayout()) == 0) {
3468 // Adjust alignment for that dynamic shared memory array.
3469 MFI->setDynLDSAlign(MF.getFunction(), GVar);
3470 LLT I32 = LLT::integer(32);
3471 auto Sz = B.buildIntrinsic(Intrinsic::amdgcn_groupstaticsize, {I32});
3472 B.buildIntToPtr(DstReg, Sz);
3473 MI.eraseFromParent();
3474 return true;
3475 }
3476 }
3477
3478 B.buildConstant(DstReg, MFI->allocateLDSGlobal(B.getDataLayout(), GVar));
3479 MI.eraseFromParent();
3480 return true;
3481 }
3482
3483 if (ST.isAmdPalOS() || ST.isMesa3DOS()) {
3484 buildAbsGlobalAddress(DstReg, Ty, B, GV, MRI);
3485 MI.eraseFromParent();
3486 return true;
3487 }
3488
3489 const SITargetLowering *TLI = ST.getTargetLowering();
3490
3491 if (TLI->shouldEmitFixup(GV)) {
3492 buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0);
3493 MI.eraseFromParent();
3494 return true;
3495 }
3496
3497 if (TLI->shouldEmitPCReloc(GV)) {
3498 buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0, SIInstrInfo::MO_REL32);
3499 MI.eraseFromParent();
3500 return true;
3501 }
3502
3504 Register GOTAddr = MRI.createGenericVirtualRegister(PtrTy);
3505
3506 LLT LoadTy = Ty.getSizeInBits() == 32 ? PtrTy : Ty;
3511 LoadTy, Align(8));
3512
3513 buildPCRelGlobalAddress(GOTAddr, PtrTy, B, GV, 0, SIInstrInfo::MO_GOTPCREL32);
3514
3515 if (Ty.getSizeInBits() == 32) {
3516 // Truncate if this is a 32-bit constant address.
3517 auto Load = B.buildLoad(PtrTy, GOTAddr, *GOTMMO);
3518 B.buildExtract(DstReg, Load, 0);
3519 } else
3520 B.buildLoad(DstReg, GOTAddr, *GOTMMO);
3521
3522 MI.eraseFromParent();
3523 return true;
3524}
3525
3527 if (Ty.isVector())
3528 return Ty.changeElementCount(
3529 ElementCount::getFixed(PowerOf2Ceil(Ty.getNumElements())));
3530 return Ty.changeElementSize(PowerOf2Ceil(Ty.getSizeInBits()));
3531}
3532
3533// An alloca in the VGPR ("as memory") address space has no registers to live
3534// in. Parallels SITargetLowering::lowerFrameIndex.
3536 MachineIRBuilder &B) const {
3537 MachineFunction &MF = B.getMF();
3538 int FI = MI.getOperand(1).getIndex();
3539 const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(FI);
3540 assert(Alloca && Alloca->getAddressSpace() == AMDGPUAS::VGPR);
3541
3543 MF.getFunction(),
3544 "unsupported alloca in VGPR 'as memory' address space (13)",
3545 Alloca->getDebugLoc()));
3546 B.buildUndef(MI.getOperand(0));
3547 MI.eraseFromParent();
3548 return true;
3549}
3550
3551// The widest part of a VGPR access of type \p Ty that a VGPR tuple holds, in
3552// whole elements if \p Ty is a vector; failing that, one element of whole
3553// dwords. No type if there is neither.
3555 const unsigned EltSize = Ty.isVector() ? Ty.getScalarSizeInBits() : 32;
3556 for (unsigned Size = std::min<unsigned>(Ty.getSizeInBits(), MaxRegisterSize);
3557 Size >= 32; Size -= 32) {
3558 if (Size % EltSize != 0 ||
3560 continue;
3561 if (!Ty.isVector())
3562 return Ty.changeElementSize(Size);
3564 Ty.getElementType());
3565 }
3566 if (EltSize % 32 == 0)
3567 return Ty.getElementType();
3568 return LLT();
3569}
3570
3571/// Lower a whole-dword load or store on AMDGPUAS::VGPR into
3572/// G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE. Parallels
3573/// SITargetLowering::lowerLoadStoreVGPR.
3574static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, GLoadStore &LdSt) {
3575 MachineIRBuilder &B = Helper.MIRBuilder;
3576 MachineRegisterInfo &MRI = *B.getMRI();
3577 MachineMemOperand &MMO = LdSt.getMMO();
3578
3579 const bool IsStore = isa<GStore>(LdSt);
3580 Register ValReg = LdSt.getReg(0);
3581 Register PtrReg = LdSt.getPointerReg();
3582
3583 const LLT ValTy = MRI.getType(ValReg);
3584 const unsigned ValSize = ValTy.getSizeInBits();
3585 const LLT I32 = LLT::integer(32);
3586
3587 // The combiner turns a masked or sign-extended whole-dword load into an
3588 // extending load of its low bits. Load the dword back and extend it in
3589 // registers; a load that really is sub-dword gets the checks below.
3590 auto *ExtLoad = dyn_cast<GExtLoad>(&LdSt);
3591 if (ExtLoad && MMO.getMemoryType().isScalar()) {
3592 const unsigned MemSize = MMO.getMemoryType().getSizeInBits();
3593 LLT LoadTy = LLT::integer(MemSize);
3594 if (MemSize < 32 && LdSt.getAlign() >= Align(4))
3595 LoadTy = I32;
3596 Register Loaded = B.buildLoadFromOffset(LoadTy, PtrReg, MMO, 0).getReg(0);
3597 const bool IsSigned = isa<GSExtLoad>(ExtLoad);
3598 if (LoadTy.getSizeInBits() != MemSize) {
3599 Loaded = IsSigned ? B.buildSExtInReg(LoadTy, Loaded, MemSize).getReg(0)
3600 : B.buildZExtInReg(LoadTy, Loaded, MemSize).getReg(0);
3601 }
3602 if (IsSigned)
3603 B.buildSExtOrTrunc(ValReg, Loaded);
3604 else
3605 B.buildZExtOrTrunc(ValReg, Loaded);
3606 LdSt.eraseFromParent();
3607 return true;
3608 }
3609
3610 // The index is the pointer >> 2, so an under-aligned access would silently
3611 // reach the containing dword.
3612 const unsigned MemSize = MMO.getMemoryType().getSizeInBits();
3613 if (MemSize != ValSize || MemSize % 32 != 0 || LdSt.getAlign() < Align(4)) {
3614 const Function &F = B.getMF().getFunction();
3615 F.getContext().diagnose(DiagnosticInfoUnsupported(
3616 F,
3617 "unsupported access of VGPR 'as memory' address space (13); only "
3618 "dword-aligned whole-dword loads and stores are implemented",
3619 LdSt.getDebugLoc()));
3620 if (!IsStore)
3621 B.buildUndef(ValReg);
3622 LdSt.eraseFromParent();
3623 return true;
3624 }
3625
3626 // A width no VGPR tuple has is split, the widest part first, and each part is
3627 // legalized again. When there is no part, the access is first made one of
3628 // <N x i32>.
3629 if (!isVGPRLoadStoreSizeSupported(MemSize, ValSize)) {
3630 LLT PartTy = getVGPRLoadStorePartType(ValTy);
3632 PartTy.isValid()
3633 ? Helper.reduceLoadStoreWidth(LdSt, 0, PartTy)
3634 : Helper.bitcast(LdSt, 0, LLT::fixed_vector(ValSize / 32, I32));
3635 return Res == LegalizerHelper::Legalized;
3636 }
3637
3638 const MachineInstrBuilder PtrAsInt = B.buildPtrToInt(I32, PtrReg);
3639 const MachineInstrBuilder Index =
3640 B.buildLShr(I32, PtrAsInt, B.buildConstant(I32, 2));
3641
3642 // Pointers go through integers: a scalar pointer cannot be bitcast, and not
3643 // every pointer type has a selection pattern.
3644 LLT IntTy = ValTy;
3645 if (ValTy.isPointerOrPointerVector())
3646 IntTy = ValTy.changeElementType(LLT::integer(ValTy.getScalarSizeInBits()));
3647
3648 // Normalize to i32 / <N x i32> so a selection pattern exists (e.g. v4i8).
3649 LLT RegTy = IntTy;
3650 if (IntTy.getScalarSizeInBits() != 32) {
3651 unsigned NumDwords = ValSize / 32;
3652 RegTy = NumDwords == 1 ? I32 : LLT::fixed_vector(NumDwords, I32);
3653 }
3654
3655 if (IsStore) {
3656 Register Value = ValReg;
3657 if (IntTy != ValTy)
3658 Value = B.buildPtrToInt(IntTy, Value).getReg(0);
3659 if (RegTy != IntTy)
3660 Value = B.buildBitcast(RegTy, Value).getReg(0);
3661 B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE, {}, {Value, Index.getReg(0)})
3662 .addMemOperand(&MMO);
3663 } else if (RegTy == ValTy) {
3664 B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {ValReg}, {Index.getReg(0)})
3665 .addMemOperand(&MMO);
3666 } else {
3667 Register Result =
3668 B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {RegTy}, {Index.getReg(0)})
3669 .addMemOperand(&MMO)
3670 .getReg(0);
3671 if (IntTy == ValTy) {
3672 B.buildBitcast(ValReg, Result);
3673 } else {
3674 if (RegTy != IntTy)
3675 Result = B.buildBitcast(IntTy, Result).getReg(0);
3676 B.buildIntToPtr(ValReg, Result);
3677 }
3678 }
3679
3680 LdSt.eraseFromParent();
3681 return true;
3682}
3683
3685 MachineInstr &MI) const {
3686 MachineIRBuilder &B = Helper.MIRBuilder;
3687 MachineRegisterInfo &MRI = *B.getMRI();
3688 GISelChangeObserver &Observer = Helper.Observer;
3689
3690 unsigned PtrIdx = MI.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3691 Register PtrReg = MI.getOperand(PtrIdx).getReg();
3692 LLT PtrTy = MRI.getType(PtrReg);
3693 unsigned AddrSpace = PtrTy.getAddressSpace();
3694
3695 if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
3697 auto Cast = B.buildAddrSpaceCast(ConstPtr, PtrReg);
3698 Observer.changingInstr(MI);
3699 MI.getOperand(PtrIdx).setReg(Cast.getReg(0));
3700 Observer.changedInstr(MI);
3701 return true;
3702 }
3703
3704 if (AddrSpace == AMDGPUAS::VGPR && isa<GExtLoad>(MI))
3705 return lowerLoadStoreVGPR(Helper, cast<GLoadStore>(MI));
3706
3707 if (MI.getOpcode() != AMDGPU::G_LOAD)
3708 return false;
3709
3710 Register ValReg = MI.getOperand(0).getReg();
3711 LLT ValTy = MRI.getType(ValReg);
3712
3713 if (hasBufferRsrcWorkaround(ValTy)) {
3714 Observer.changingInstr(MI);
3716 Observer.changedInstr(MI);
3717 return true;
3718 }
3719
3720 if (AddrSpace == AMDGPUAS::VGPR)
3721 return lowerLoadStoreVGPR(Helper, cast<GLoadStore>(MI));
3722
3723 MachineMemOperand *MMO = *MI.memoperands_begin();
3724 const unsigned ValSize = ValTy.getSizeInBits();
3725 const LLT MemTy = MMO->getMemoryType();
3726 const Align MemAlign = MMO->getAlign();
3727 const unsigned MemSize = MemTy.getSizeInBits();
3728 const uint64_t AlignInBits = 8 * MemAlign.value();
3729
3730 // Widen non-power-of-2 loads to the alignment if needed
3731 if (shouldWidenLoad(ST, MemTy, AlignInBits, AddrSpace, MI.getOpcode())) {
3732 const unsigned WideMemSize = PowerOf2Ceil(MemSize);
3733
3734 // This was already the correct extending load result type, so just adjust
3735 // the memory type.
3736 if (WideMemSize == ValSize) {
3737 MachineFunction &MF = B.getMF();
3738
3739 MachineMemOperand *WideMMO =
3740 MF.getMachineMemOperand(MMO, 0, WideMemSize / 8);
3741 Observer.changingInstr(MI);
3742 MI.setMemRefs(MF, {WideMMO});
3743 Observer.changedInstr(MI);
3744 return true;
3745 }
3746
3747 // Don't bother handling edge case that should probably never be produced.
3748 if (ValSize > WideMemSize)
3749 return false;
3750
3751 LLT WideTy = widenToNextPowerOf2(ValTy);
3752
3753 Register WideLoad;
3754 if (!WideTy.isVector()) {
3755 WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
3756 B.buildTrunc(ValReg, WideLoad).getReg(0);
3757 } else {
3758 // Extract the subvector.
3759
3760 if (isRegisterType(ST, ValTy)) {
3761 // If this a case where G_EXTRACT is legal, use it.
3762 // (e.g. <3 x i32> -> <4 x i32>)
3763 WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
3764 B.buildExtract(ValReg, WideLoad, 0);
3765 } else {
3766 // For cases where the widened type isn't a nice register value, unmerge
3767 // from a widened register (e.g. <3 x i16> -> <4 x i16>)
3768 WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
3769 B.buildDeleteTrailingVectorElements(ValReg, WideLoad);
3770 }
3771 }
3772
3773 MI.eraseFromParent();
3774 return true;
3775 }
3776
3777 return false;
3778}
3779
3781 MachineInstr &MI) const {
3782 MachineIRBuilder &B = Helper.MIRBuilder;
3783 MachineRegisterInfo &MRI = *B.getMRI();
3784 GISelChangeObserver &Observer = Helper.Observer;
3785
3786 Register DataReg = MI.getOperand(0).getReg();
3787 LLT DataTy = MRI.getType(DataReg);
3788
3789 if (hasBufferRsrcWorkaround(DataTy)) {
3790 Observer.changingInstr(MI);
3792 Observer.changedInstr(MI);
3793 return true;
3794 }
3795
3796 if (MRI.getType(MI.getOperand(1).getReg()).getAddressSpace() ==
3798 return lowerLoadStoreVGPR(Helper, cast<GLoadStore>(MI));
3799 return false;
3800}
3801
3804 MachineIRBuilder &B) const {
3805 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3806 assert(Ty.isScalar());
3807
3808 MachineFunction &MF = B.getMF();
3810
3811 // TODO: Always legal with future ftz flag.
3812 // TODO: Type is expected to be LLT::float32()/LLT::float16()
3813 // FIXME: Do we need just output?
3814 if (Ty == F32 &&
3816 return true;
3817 if (Ty == F16 &&
3819 return true;
3820
3821 MachineIRBuilder HelperBuilder(MI);
3822 GISelObserverWrapper DummyObserver;
3823 LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
3824 return Helper.lowerFMad(MI) == LegalizerHelper::Legalized;
3825}
3826
3829 Register DstReg = MI.getOperand(0).getReg();
3830 Register PtrReg = MI.getOperand(1).getReg();
3831 Register CmpVal = MI.getOperand(2).getReg();
3832 Register NewVal = MI.getOperand(3).getReg();
3833
3834 assert(AMDGPU::isFlatGlobalAddrSpace(MRI.getType(PtrReg).getAddressSpace()) &&
3835 "this should not have been custom lowered");
3836
3837 LLT ValTy = MRI.getType(CmpVal);
3838 LLT VecTy = LLT::fixed_vector(2, ValTy);
3839
3840 Register PackedVal = B.buildBuildVector(VecTy, { NewVal, CmpVal }).getReg(0);
3841
3842 B.buildInstr(AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG)
3843 .addDef(DstReg)
3844 .addUse(PtrReg)
3845 .addUse(PackedVal)
3846 .setMemRefs(MI.memoperands());
3847
3848 MI.eraseFromParent();
3849 return true;
3850}
3851
3852/// Return true if it's known that \p Src can never be an f32 denormal value.
3854 Register Src) {
3855 const MachineInstr *DefMI = MRI.getVRegDef(Src);
3856 switch (DefMI->getOpcode()) {
3857 case TargetOpcode::G_INTRINSIC: {
3859 case Intrinsic::amdgcn_frexp_mant:
3860 case Intrinsic::amdgcn_log:
3861 case Intrinsic::amdgcn_log_clamp:
3862 case Intrinsic::amdgcn_exp2:
3863 case Intrinsic::amdgcn_sqrt:
3864 return true;
3865 default:
3866 break;
3867 }
3868
3869 break;
3870 }
3871 case TargetOpcode::G_FSQRT:
3872 return true;
3873 case TargetOpcode::G_FFREXP: {
3874 if (DefMI->getOperand(0).getReg() == Src)
3875 return true;
3876 break;
3877 }
3878 case TargetOpcode::G_FPEXT: {
3879 return MRI.getType(DefMI->getOperand(1).getReg()) == F16;
3880 }
3881 default:
3882 return false;
3883 }
3884
3885 return false;
3886}
3887
3888static bool allowApproxFunc(const MachineFunction &MF, unsigned Flags) {
3889 return Flags & MachineInstr::FmAfn;
3890}
3891
3893 unsigned Flags) {
3894 return !valueIsKnownNeverF32Denorm(MF.getRegInfo(), Src) &&
3897}
3898
3899std::pair<Register, Register>
3901 unsigned Flags) const {
3902 if (!needsDenormHandlingF32(B.getMF(), Src, Flags))
3903 return {};
3904
3905 auto SmallestNormal = B.buildFConstant(
3907 auto IsLtSmallestNormal =
3908 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src, SmallestNormal);
3909
3910 auto Scale32 = B.buildFConstant(F32, 0x1.0p+32);
3911 auto One = B.buildFConstant(F32, 1.0);
3912 auto ScaleFactor =
3913 B.buildSelect(F32, IsLtSmallestNormal, Scale32, One, Flags);
3914 auto ScaledInput = B.buildFMul(F32, Src, ScaleFactor, Flags);
3915
3916 return {ScaledInput.getReg(0), IsLtSmallestNormal.getReg(0)};
3917}
3918
3920 MachineIRBuilder &B) const {
3921 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
3922 // If we have to handle denormals, scale up the input and adjust the result.
3923
3924 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
3925 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
3926
3927 Register Dst = MI.getOperand(0).getReg();
3928 Register Src = MI.getOperand(1).getReg();
3929 LLT Ty = B.getMRI()->getType(Dst);
3930 unsigned Flags = MI.getFlags();
3931
3932 if (Ty == F16) {
3933 // Nothing in half is a denormal when promoted to f32.
3934 auto Ext = B.buildFPExt(F32, Src, Flags);
3935 auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_log, {F32})
3936 .addUse(Ext.getReg(0))
3937 .setMIFlags(Flags);
3938 B.buildFPTrunc(Dst, Log2, Flags);
3939 MI.eraseFromParent();
3940 return true;
3941 }
3942
3943 assert(Ty == F32);
3944
3945 auto [ScaledInput, IsLtSmallestNormal] = getScaledLogInput(B, Src, Flags);
3946 if (!ScaledInput) {
3947 B.buildIntrinsic(Intrinsic::amdgcn_log, {MI.getOperand(0)})
3948 .addUse(Src)
3949 .setMIFlags(Flags);
3950 MI.eraseFromParent();
3951 return true;
3952 }
3953
3954 auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
3955 .addUse(ScaledInput)
3956 .setMIFlags(Flags);
3957
3958 auto ThirtyTwo = B.buildFConstant(Ty, 32.0);
3959 auto Zero = B.buildFConstant(Ty, 0.0);
3960 auto ResultOffset =
3961 B.buildSelect(Ty, IsLtSmallestNormal, ThirtyTwo, Zero, Flags);
3962 B.buildFSub(Dst, Log2, ResultOffset, Flags);
3963
3964 MI.eraseFromParent();
3965 return true;
3966}
3967
3969 Register Z, unsigned Flags) {
3970 auto FMul = B.buildFMul(Ty, X, Y, Flags);
3971 return B.buildFAdd(Ty, FMul, Z, Flags).getReg(0);
3972}
3973
3975 MachineIRBuilder &B) const {
3976 const bool IsLog10 = MI.getOpcode() == TargetOpcode::G_FLOG10;
3977 assert(IsLog10 || MI.getOpcode() == TargetOpcode::G_FLOG);
3978
3979 MachineRegisterInfo &MRI = *B.getMRI();
3980 Register Dst = MI.getOperand(0).getReg();
3981 Register X = MI.getOperand(1).getReg();
3982 unsigned Flags = MI.getFlags();
3983 const LLT Ty = MRI.getType(X);
3984
3985 if (Ty == F16 || MI.getFlag(MachineInstr::FmAfn)) {
3986 // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
3987 // depending on !fpmath metadata.
3988 bool PromoteToF32 =
3989 Ty == F16 && (!MI.getFlag(MachineInstr::FmAfn) || !ST.has16BitInsts());
3990 if (PromoteToF32) {
3991 Register LogVal = MRI.createGenericVirtualRegister(F32);
3992 auto PromoteSrc = B.buildFPExt(F32, X, Flags);
3993 legalizeFlogUnsafe(B, LogVal, PromoteSrc.getReg(0), IsLog10, Flags);
3994 B.buildFPTrunc(Dst, LogVal, Flags);
3995 } else {
3996 legalizeFlogUnsafe(B, Dst, X, IsLog10, Flags);
3997 }
3998
3999 MI.eraseFromParent();
4000 return true;
4001 }
4002
4003 auto [ScaledInput, IsScaled] = getScaledLogInput(B, X, Flags);
4004 if (ScaledInput)
4005 X = ScaledInput;
4006
4007 auto Y =
4008 B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty}).addUse(X).setMIFlags(Flags);
4009
4010 Register R;
4011 if (ST.hasFastFMAF32()) {
4012 // c+cc are ln(2)/ln(10) to more than 49 bits
4013 const float c_log10 = 0x1.344134p-2f;
4014 const float cc_log10 = 0x1.09f79ep-26f;
4015
4016 // c + cc is ln(2) to more than 49 bits
4017 const float c_log = 0x1.62e42ep-1f;
4018 const float cc_log = 0x1.efa39ep-25f;
4019
4020 auto C = B.buildFConstant(Ty, IsLog10 ? c_log10 : c_log);
4021 auto CC = B.buildFConstant(Ty, IsLog10 ? cc_log10 : cc_log);
4022 // This adds correction terms for which contraction may lead to an increase
4023 // in the error of the approximation, so disable it.
4024 auto NewFlags = Flags & ~(MachineInstr::FmContract);
4025 R = B.buildFMul(Ty, Y, C, NewFlags).getReg(0);
4026 auto NegR = B.buildFNeg(Ty, R, NewFlags);
4027 auto FMA0 = B.buildFMA(Ty, Y, C, NegR, NewFlags);
4028 auto FMA1 = B.buildFMA(Ty, Y, CC, FMA0, NewFlags);
4029 R = B.buildFAdd(Ty, R, FMA1, NewFlags).getReg(0);
4030 } else {
4031 // ch+ct is ln(2)/ln(10) to more than 36 bits
4032 const float ch_log10 = 0x1.344000p-2f;
4033 const float ct_log10 = 0x1.3509f6p-18f;
4034
4035 // ch + ct is ln(2) to more than 36 bits
4036 const float ch_log = 0x1.62e000p-1f;
4037 const float ct_log = 0x1.0bfbe8p-15f;
4038
4039 auto CH = B.buildFConstant(Ty, IsLog10 ? ch_log10 : ch_log);
4040 auto CT = B.buildFConstant(Ty, IsLog10 ? ct_log10 : ct_log);
4041
4042 const LLT I32 = LLT::integer(32);
4043 auto YInt = B.buildBitcast(I32, Y);
4044 auto MaskConst = B.buildConstant(I32, 0xfffff000);
4045 auto YH = B.buildBitcast(Ty, B.buildAnd(I32, YInt, MaskConst));
4046 auto YT = B.buildFSub(Ty, Y, YH, Flags);
4047 // This adds correction terms for which contraction may lead to an increase
4048 // in the error of the approximation, so disable it.
4049 auto NewFlags = Flags & ~(MachineInstr::FmContract);
4050 auto YTCT = B.buildFMul(Ty, YT, CT, NewFlags);
4051
4052 Register Mad0 =
4053 getMad(B, Ty, YH.getReg(0), CT.getReg(0), YTCT.getReg(0), NewFlags);
4054 Register Mad1 = getMad(B, Ty, YT.getReg(0), CH.getReg(0), Mad0, NewFlags);
4055 R = getMad(B, Ty, YH.getReg(0), CH.getReg(0), Mad1, NewFlags);
4056 }
4057
4058 const bool IsFiniteOnly =
4060
4061 if (!IsFiniteOnly) {
4062 // Expand isfinite(x) => fabs(x) < inf
4063 auto Inf = B.buildFConstant(Ty, APFloat::getInf(APFloat::IEEEsingle()));
4064 auto Fabs = B.buildFAbs(Ty, Y);
4065 auto IsFinite =
4066 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags);
4067 R = B.buildSelect(Ty, IsFinite, R, Y, Flags).getReg(0);
4068 }
4069
4070 if (ScaledInput) {
4071 auto Zero = B.buildFConstant(Ty, 0.0);
4072 auto ShiftK =
4073 B.buildFConstant(Ty, IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f);
4074 auto Shift = B.buildSelect(Ty, IsScaled, ShiftK, Zero, Flags);
4075 B.buildFSub(Dst, R, Shift, Flags);
4076 } else {
4077 B.buildCopy(Dst, R);
4078 }
4079
4080 MI.eraseFromParent();
4081 return true;
4082}
4083
4085 Register Src, bool IsLog10,
4086 unsigned Flags) const {
4087 const double Log2BaseInverted =
4089
4090 LLT Ty = B.getMRI()->getType(Dst);
4091
4092 if (Ty == F32) {
4093 auto [ScaledInput, IsScaled] = getScaledLogInput(B, Src, Flags);
4094 if (ScaledInput) {
4095 auto LogSrc = B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
4096 .addUse(ScaledInput)
4097 .setMIFlags(Flags);
4098 auto ScaledResultOffset = B.buildFConstant(Ty, -32.0 * Log2BaseInverted);
4099 auto Zero = B.buildFConstant(Ty, 0.0);
4100 auto ResultOffset =
4101 B.buildSelect(Ty, IsScaled, ScaledResultOffset, Zero, Flags);
4102 auto Log2Inv = B.buildFConstant(Ty, Log2BaseInverted);
4103
4104 if (ST.hasFastFMAF32())
4105 B.buildFMA(Dst, LogSrc, Log2Inv, ResultOffset, Flags);
4106 else {
4107 auto Mul = B.buildFMul(Ty, LogSrc, Log2Inv, Flags);
4108 B.buildFAdd(Dst, Mul, ResultOffset, Flags);
4109 }
4110
4111 return true;
4112 }
4113 }
4114
4115 auto Log2Operand = Ty == F16 ? B.buildFLog2(Ty, Src, Flags)
4116 : B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
4117 .addUse(Src)
4118 .setMIFlags(Flags);
4119 auto Log2BaseInvertedOperand = B.buildFConstant(Ty, Log2BaseInverted);
4120 B.buildFMul(Dst, Log2Operand, Log2BaseInvertedOperand, Flags);
4121 return true;
4122}
4123
4125 MachineIRBuilder &B) const {
4126 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
4127 // If we have to handle denormals, scale up the input and adjust the result.
4128
4129 Register Dst = MI.getOperand(0).getReg();
4130 Register Src = MI.getOperand(1).getReg();
4131 unsigned Flags = MI.getFlags();
4132 LLT Ty = B.getMRI()->getType(Dst);
4133
4134 if (Ty == F64)
4135 return legalizeFEXPF64(MI, B);
4136
4137 if (Ty == F16) {
4138 // Nothing in half is a denormal when promoted to f32.
4139 auto Ext = B.buildFPExt(F32, Src, Flags);
4140 auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {F32})
4141 .addUse(Ext.getReg(0))
4142 .setMIFlags(Flags);
4143 B.buildFPTrunc(Dst, Log2, Flags);
4144 MI.eraseFromParent();
4145 return true;
4146 }
4147
4148 assert(Ty == F32);
4149
4150 if (!needsDenormHandlingF32(B.getMF(), Src, Flags)) {
4151 B.buildIntrinsic(Intrinsic::amdgcn_exp2, ArrayRef<Register>{Dst})
4152 .addUse(Src)
4153 .setMIFlags(Flags);
4154 MI.eraseFromParent();
4155 return true;
4156 }
4157
4158 // bool needs_scaling = x < -0x1.f80000p+6f;
4159 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
4160
4161 // -nextafter(128.0, -1)
4162 auto RangeCheckConst = B.buildFConstant(Ty, -0x1.f80000p+6f);
4163 auto NeedsScaling = B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src,
4164 RangeCheckConst, Flags);
4165
4166 auto SixtyFour = B.buildFConstant(Ty, 0x1.0p+6f);
4167 auto Zero = B.buildFConstant(Ty, 0.0);
4168 auto AddOffset = B.buildSelect(F32, NeedsScaling, SixtyFour, Zero, Flags);
4169 auto AddInput = B.buildFAdd(F32, Src, AddOffset, Flags);
4170
4171 auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
4172 .addUse(AddInput.getReg(0))
4173 .setMIFlags(Flags);
4174
4175 auto TwoExpNeg64 = B.buildFConstant(Ty, 0x1.0p-64f);
4176 auto One = B.buildFConstant(Ty, 1.0);
4177 auto ResultScale = B.buildSelect(F32, NeedsScaling, TwoExpNeg64, One, Flags);
4178 B.buildFMul(Dst, Exp2, ResultScale, Flags);
4179 MI.eraseFromParent();
4180 return true;
4181}
4182
4184 const SrcOp &Src, unsigned Flags) {
4185 LLT Ty = Dst.getLLTTy(*B.getMRI());
4186
4187 if (Ty == F32) {
4188 return B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Dst})
4189 .addUse(Src.getReg())
4190 .setMIFlags(Flags);
4191 }
4192 return B.buildFExp2(Dst, Src, Flags);
4193}
4194
4196 Register Dst, Register X,
4197 unsigned Flags,
4198 bool IsExp10) const {
4199 LLT Ty = B.getMRI()->getType(X);
4200
4201 // exp(x) -> exp2(M_LOG2E_F * x);
4202 // exp10(x) -> exp2(log2(10) * x);
4203 auto Const = B.buildFConstant(Ty, IsExp10 ? 0x1.a934f0p+1f : numbers::log2e);
4204 auto Mul = B.buildFMul(Ty, X, Const, Flags);
4205 buildExp(B, Dst, Mul, Flags);
4206 return true;
4207}
4208
4210 Register X, unsigned Flags) const {
4211 LLT Ty = B.getMRI()->getType(Dst);
4212
4213 if (Ty != F32 || !needsDenormHandlingF32(B.getMF(), X, Flags)) {
4214 return legalizeFExpUnsafeImpl(B, Dst, X, Flags, /*IsExp10=*/false);
4215 }
4216
4217 auto Threshold = B.buildFConstant(Ty, -0x1.5d58a0p+6f);
4218 auto NeedsScaling =
4219 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, Threshold, Flags);
4220 auto ScaleOffset = B.buildFConstant(Ty, 0x1.0p+6f);
4221 auto ScaledX = B.buildFAdd(Ty, X, ScaleOffset, Flags);
4222 auto AdjustedX = B.buildSelect(Ty, NeedsScaling, ScaledX, X, Flags);
4223
4224 auto Log2E = B.buildFConstant(Ty, numbers::log2e);
4225 auto ExpInput = B.buildFMul(Ty, AdjustedX, Log2E, Flags);
4226
4227 auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
4228 .addUse(ExpInput.getReg(0))
4229 .setMIFlags(Flags);
4230
4231 auto ResultScaleFactor = B.buildFConstant(Ty, 0x1.969d48p-93f);
4232 auto AdjustedResult = B.buildFMul(Ty, Exp2, ResultScaleFactor, Flags);
4233 B.buildSelect(Dst, NeedsScaling, AdjustedResult, Exp2, Flags);
4234 return true;
4235}
4236
4238 Register Dst, Register X,
4239 unsigned Flags) const {
4240 LLT Ty = B.getMRI()->getType(Dst);
4241
4242 if (Ty != F32 || !needsDenormHandlingF32(B.getMF(), X, Flags)) {
4243 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
4244 auto K0 = B.buildFConstant(Ty, 0x1.a92000p+1f);
4245 auto K1 = B.buildFConstant(Ty, 0x1.4f0978p-11f);
4246
4247 auto Mul1 = B.buildFMul(Ty, X, K1, Flags);
4248 auto Exp2_1 = buildExp(B, Ty, Mul1, Flags);
4249 auto Mul0 = B.buildFMul(Ty, X, K0, Flags);
4250 auto Exp2_0 = buildExp(B, Ty, Mul0, Flags);
4251 B.buildFMul(Dst, Exp2_0, Exp2_1, Flags);
4252 return true;
4253 }
4254
4255 // bool s = x < -0x1.2f7030p+5f;
4256 // x += s ? 0x1.0p+5f : 0.0f;
4257 // exp10 = exp2(x * 0x1.a92000p+1f) *
4258 // exp2(x * 0x1.4f0978p-11f) *
4259 // (s ? 0x1.9f623ep-107f : 1.0f);
4260
4261 auto Threshold = B.buildFConstant(Ty, -0x1.2f7030p+5f);
4262 auto NeedsScaling =
4263 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, Threshold);
4264
4265 auto ScaleOffset = B.buildFConstant(Ty, 0x1.0p+5f);
4266 auto ScaledX = B.buildFAdd(Ty, X, ScaleOffset, Flags);
4267 auto AdjustedX = B.buildSelect(Ty, NeedsScaling, ScaledX, X);
4268
4269 auto K0 = B.buildFConstant(Ty, 0x1.a92000p+1f);
4270 auto K1 = B.buildFConstant(Ty, 0x1.4f0978p-11f);
4271
4272 auto Mul1 = B.buildFMul(Ty, AdjustedX, K1, Flags);
4273 auto Exp2_1 = buildExp(B, Ty, Mul1, Flags);
4274 auto Mul0 = B.buildFMul(Ty, AdjustedX, K0, Flags);
4275 auto Exp2_0 = buildExp(B, Ty, Mul0, Flags);
4276
4277 auto MulExps = B.buildFMul(Ty, Exp2_0, Exp2_1, Flags);
4278 auto ResultScaleFactor = B.buildFConstant(Ty, 0x1.9f623ep-107f);
4279 auto AdjustedResult = B.buildFMul(Ty, MulExps, ResultScaleFactor, Flags);
4280
4281 B.buildSelect(Dst, NeedsScaling, AdjustedResult, MulExps);
4282 return true;
4283}
4284
4285// This expansion gives a result slightly better than 1ulp.
4287 MachineIRBuilder &B) const {
4288
4289 Register X = MI.getOperand(1).getReg();
4290 LLT I32 = LLT::integer(32);
4291 LLT S1 = LLT::scalar(1);
4292
4293 // TODO: Check if reassoc is safe. There is an output change in exp2 and
4294 // exp10, which slightly increases ulp.
4295 unsigned Flags = MI.getFlags() & ~MachineInstr::FmReassoc;
4296
4297 Register Dn, F, T;
4298
4299 if (MI.getOpcode() == TargetOpcode::G_FEXP2) {
4300 // Dn = rint(X)
4301 Dn = B.buildFRint(F64, X, Flags).getReg(0);
4302 // F = X - Dn
4303 F = B.buildFSub(F64, X, Dn, Flags).getReg(0);
4304 // T = F*C1 + F*C2
4305 auto C1 = B.buildFConstant(F64, APFloat(0x1.62e42fefa39efp-1));
4306 auto C2 = B.buildFConstant(F64, APFloat(0x1.abc9e3b39803fp-56));
4307 auto Mul2 = B.buildFMul(F64, F, C2, Flags).getReg(0);
4308 T = B.buildFMA(F64, F, C1, Mul2, Flags).getReg(0);
4309
4310 } else if (MI.getOpcode() == TargetOpcode::G_FEXP10) {
4311 auto C1 = B.buildFConstant(F64, APFloat(0x1.a934f0979a371p+1));
4312 auto Mul = B.buildFMul(F64, X, C1, Flags).getReg(0);
4313 Dn = B.buildFRint(F64, Mul, Flags).getReg(0);
4314
4315 auto NegDn = B.buildFNeg(F64, Dn, Flags).getReg(0);
4316 auto C2 = B.buildFConstant(F64, APFloat(-0x1.9dc1da994fd21p-59));
4317 auto C3 = B.buildFConstant(F64, APFloat(0x1.34413509f79ffp-2));
4318 auto Inner = B.buildFMA(F64, NegDn, C3, X, Flags).getReg(0);
4319 F = B.buildFMA(F64, NegDn, C2, Inner, Flags).getReg(0);
4320
4321 auto C4 = B.buildFConstant(F64, APFloat(0x1.26bb1bbb55516p+1));
4322 auto C5 = B.buildFConstant(F64, APFloat(-0x1.f48ad494ea3e9p-53));
4323 auto MulF = B.buildFMul(F64, F, C5, Flags).getReg(0);
4324 T = B.buildFMA(F64, F, C4, MulF, Flags).getReg(0);
4325
4326 } else { // G_FEXP
4327 auto C1 = B.buildFConstant(F64, APFloat(0x1.71547652b82fep+0));
4328 auto Mul = B.buildFMul(F64, X, C1, Flags).getReg(0);
4329 Dn = B.buildFRint(F64, Mul, Flags).getReg(0);
4330
4331 auto NegDn = B.buildFNeg(F64, Dn, Flags).getReg(0);
4332 auto C2 = B.buildFConstant(F64, APFloat(0x1.abc9e3b39803fp-56));
4333 auto C3 = B.buildFConstant(F64, APFloat(0x1.62e42fefa39efp-1));
4334 auto Inner = B.buildFMA(F64, NegDn, C3, X, Flags).getReg(0);
4335 T = B.buildFMA(F64, NegDn, C2, Inner, Flags).getReg(0);
4336 }
4337
4338 // Polynomial chain for P
4339 auto P = B.buildFConstant(F64, 0x1.ade156a5dcb37p-26);
4340 P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.28af3fca7ab0cp-22),
4341 Flags);
4342 P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.71dee623fde64p-19),
4343 Flags);
4344 P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.a01997c89e6b0p-16),
4345 Flags);
4346 P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.a01a014761f6ep-13),
4347 Flags);
4348 P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.6c16c1852b7b0p-10),
4349 Flags);
4350 P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.1111111122322p-7), Flags);
4351 P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.55555555502a1p-5), Flags);
4352 P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.5555555555511p-3), Flags);
4353 P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.000000000000bp-1), Flags);
4354
4355 auto One = B.buildFConstant(F64, 1.0);
4356 P = B.buildFMA(F64, T, P, One, Flags);
4357 P = B.buildFMA(F64, T, P, One, Flags);
4358
4359 // Z = FLDEXP(P, (int)Dn)
4360 auto DnInt = B.buildFPTOSI(I32, Dn);
4361 auto Z = B.buildFLdexp(F64, P, DnInt, Flags);
4362
4363 if (!(Flags & MachineInstr::FmNoInfs)) {
4364 // Overflow guard: if X <= 1024.0 then Z else +inf
4365 auto CondHi = B.buildFCmp(CmpInst::FCMP_ULE, S1, X,
4366 B.buildFConstant(F64, APFloat(1024.0)));
4367 auto PInf = B.buildFConstant(F64, APFloat::getInf(APFloat::IEEEdouble()));
4368 Z = B.buildSelect(F64, CondHi, Z, PInf, Flags);
4369 }
4370
4371 // Underflow guard: if X >= -1075.0 then Z else 0.0
4372 auto CondLo = B.buildFCmp(CmpInst::FCMP_UGE, S1, X,
4373 B.buildFConstant(F64, APFloat(-1075.0)));
4374 auto Zero = B.buildFConstant(F64, APFloat(0.0));
4375 B.buildSelect(MI.getOperand(0).getReg(), CondLo, Z, Zero, Flags);
4376
4377 MI.eraseFromParent();
4378 return true;
4379}
4380
4382 MachineIRBuilder &B) const {
4383 Register Dst = MI.getOperand(0).getReg();
4384 Register X = MI.getOperand(1).getReg();
4385 const unsigned Flags = MI.getFlags();
4386 MachineFunction &MF = B.getMF();
4387 MachineRegisterInfo &MRI = *B.getMRI();
4388 LLT Ty = MRI.getType(Dst);
4389
4390 if (Ty == F64)
4391 return legalizeFEXPF64(MI, B);
4392
4393 const bool IsExp10 = MI.getOpcode() == TargetOpcode::G_FEXP10;
4394
4395 if (Ty == F16) {
4396 // v_exp_f16 (fmul x, log2e)
4397 if (allowApproxFunc(MF, Flags)) {
4398 // TODO: Does this really require fast?
4399 IsExp10 ? legalizeFExp10Unsafe(B, Dst, X, Flags)
4400 : legalizeFExpUnsafe(B, Dst, X, Flags);
4401 MI.eraseFromParent();
4402 return true;
4403 }
4404
4405 // Nothing in half is a denormal when promoted to f32.
4406 //
4407 // exp(f16 x) ->
4408 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
4409 //
4410 // exp10(f16 x) ->
4411 // fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
4412 auto Ext = B.buildFPExt(F32, X, Flags);
4413 Register Lowered = MRI.createGenericVirtualRegister(F32);
4414 legalizeFExpUnsafeImpl(B, Lowered, Ext.getReg(0), Flags, IsExp10);
4415 B.buildFPTrunc(Dst, Lowered, Flags);
4416 MI.eraseFromParent();
4417 return true;
4418 }
4419
4420 assert(Ty == F32);
4421
4422 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
4423 // library behavior. Also, is known-not-daz source sufficient?
4424 if (allowApproxFunc(MF, Flags)) {
4425 IsExp10 ? legalizeFExp10Unsafe(B, Dst, X, Flags)
4426 : legalizeFExpUnsafe(B, Dst, X, Flags);
4427 MI.eraseFromParent();
4428 return true;
4429 }
4430
4431 // Algorithm:
4432 //
4433 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
4434 //
4435 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
4436 // n = 64*m + j, 0 <= j < 64
4437 //
4438 // e^x = 2^((64*m + j + f)/64)
4439 // = (2^m) * (2^(j/64)) * 2^(f/64)
4440 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
4441 //
4442 // f = x*(64/ln(2)) - n
4443 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
4444 //
4445 // e^x = (2^m) * (2^(j/64)) * e^r
4446 //
4447 // (2^(j/64)) is precomputed
4448 //
4449 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
4450 // e^r = 1 + q
4451 //
4452 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
4453 //
4454 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
4455 const unsigned FlagsNoContract = Flags & ~MachineInstr::FmContract;
4456 Register PH, PL;
4457
4458 if (ST.hasFastFMAF32()) {
4459 const float c_exp = numbers::log2ef;
4460 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
4461 const float c_exp10 = 0x1.a934f0p+1f;
4462 const float cc_exp10 = 0x1.2f346ep-24f;
4463
4464 auto C = B.buildFConstant(Ty, IsExp10 ? c_exp10 : c_exp);
4465 PH = B.buildFMul(Ty, X, C, Flags).getReg(0);
4466 auto NegPH = B.buildFNeg(Ty, PH, Flags);
4467 auto FMA0 = B.buildFMA(Ty, X, C, NegPH, Flags);
4468
4469 auto CC = B.buildFConstant(Ty, IsExp10 ? cc_exp10 : cc_exp);
4470 PL = B.buildFMA(Ty, X, CC, FMA0, Flags).getReg(0);
4471 } else {
4472 const float ch_exp = 0x1.714000p+0f;
4473 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
4474
4475 const float ch_exp10 = 0x1.a92000p+1f;
4476 const float cl_exp10 = 0x1.4f0978p-11f;
4477
4478 const LLT I32 = LLT::integer(32);
4479 auto XInt = B.buildBitcast(I32, X);
4480 auto MaskConst = B.buildConstant(I32, 0xfffff000);
4481 auto XH = B.buildBitcast(Ty, B.buildAnd(I32, XInt, MaskConst));
4482 auto XL = B.buildFSub(Ty, X, XH, Flags);
4483
4484 auto CH = B.buildFConstant(Ty, IsExp10 ? ch_exp10 : ch_exp);
4485 PH = B.buildFMul(Ty, XH, CH, Flags).getReg(0);
4486
4487 auto CL = B.buildFConstant(Ty, IsExp10 ? cl_exp10 : cl_exp);
4488 auto XLCL = B.buildFMul(Ty, XL, CL, Flags);
4489
4490 Register Mad0 =
4491 getMad(B, Ty, XL.getReg(0), CH.getReg(0), XLCL.getReg(0), Flags);
4492 PL = getMad(B, Ty, XH.getReg(0), CL.getReg(0), Mad0, Flags);
4493 }
4494
4495 auto E = B.buildIntrinsicRoundeven(Ty, PH, Flags);
4496
4497 // It is unsafe to contract this fsub into the PH multiply.
4498 auto PHSubE = B.buildFSub(Ty, PH, E, FlagsNoContract);
4499 auto A = B.buildFAdd(Ty, PHSubE, PL, Flags);
4500 const LLT I32 = LLT::integer(32);
4501 auto IntE = B.buildFPTOSI(I32, E);
4502
4503 auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
4504 .addUse(A.getReg(0))
4505 .setMIFlags(Flags);
4506 auto R = B.buildFLdexp(Ty, Exp2, IntE, Flags);
4507
4508 auto UnderflowCheckConst =
4509 B.buildFConstant(Ty, IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f);
4510 auto Zero = B.buildFConstant(Ty, 0.0);
4511 auto Underflow =
4512 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, UnderflowCheckConst);
4513
4514 R = B.buildSelect(Ty, Underflow, Zero, R);
4515
4516 if (!(Flags & MachineInstr::FmNoInfs)) {
4517 auto OverflowCheckConst =
4518 B.buildFConstant(Ty, IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f);
4519
4520 auto Overflow =
4521 B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), X, OverflowCheckConst);
4522 auto Inf = B.buildFConstant(Ty, APFloat::getInf(APFloat::IEEEsingle()));
4523 R = B.buildSelect(Ty, Overflow, Inf, R, Flags);
4524 }
4525
4526 B.buildCopy(Dst, R);
4527 MI.eraseFromParent();
4528 return true;
4529}
4530
4531// Keep in sync with AMDGPUTargetLowering::lowerFPOW, which documents this.
4533 MachineInstr &MI) const {
4534 MachineIRBuilder &B = Helper.MIRBuilder;
4535 Register Dst = MI.getOperand(0).getReg();
4536 Register X = MI.getOperand(1).getReg();
4537 Register Y = MI.getOperand(2).getReg();
4538 unsigned Flags = MI.getFlags();
4539 assert(B.getMRI()->getType(Dst) == F32);
4540
4541 // log2(0) is -inf, which exp2 turns back into a finite result, so the core
4542 // goes infinite for inputs a ninf fpow still asserts about, like pow(0, 2).
4543 unsigned CoreFlags = Flags & ~MachineInstr::FmNoInfs;
4544
4545 // Fast expansion: ignores denormals, NaN for a negative base.
4546 if (allowApproxFunc(B.getMF(), Flags)) {
4547 auto Log = B.buildIntrinsic(Intrinsic::amdgcn_log, {F32})
4548 .addUse(X)
4549 .setMIFlags(CoreFlags);
4550 auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {F32})
4551 .addUse(Y)
4552 .addUse(Log.getReg(0))
4553 .setMIFlags(CoreFlags);
4554 buildExp(B, Dst, Mul.getReg(0), CoreFlags);
4555 MI.eraseFromParent();
4556 return true;
4557 }
4558
4559 auto Abs = B.buildFAbs(F32, X, Flags);
4560 auto Log = B.buildFLog2(F32, Abs, CoreFlags);
4561 auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {F32})
4562 .addUse(Y)
4563 .addUse(Log.getReg(0))
4564 .setMIFlags(CoreFlags);
4565
4566 // A base that is never negative needs neither the sign fixup nor the NaN.
4567 if (Helper.getValueTracking()
4568 ->computeKnownFPClass(X, Flags, fcNegative, 0)
4569 .signBitIsZeroOrNaN()) {
4570 B.buildFExp2(Dst, Mul, CoreFlags);
4571 MI.eraseFromParent();
4572 return true;
4573 }
4574
4575 Register R = B.buildFExp2(F32, Mul, CoreFlags).getReg(0);
4576
4577 auto YTrunc = B.buildIntrinsicTrunc(F32, Y);
4578 auto YIsInt = B.buildFCmp(CmpInst::FCMP_OEQ, S1, YTrunc, Y);
4579 auto YHalf = B.buildFMul(F32, Y, B.buildFConstant(F32, 0.5));
4580 auto YHalfTrunc = B.buildIntrinsicTrunc(F32, YHalf);
4581 auto YIsOdd = B.buildAnd(
4582 S1, YIsInt, B.buildFCmp(CmpInst::FCMP_ONE, S1, YHalfTrunc, YHalf));
4583
4584 // pow(-x, odd y) = -pow(x, y). Selecting the copysign lets even y fold it.
4585 auto Neg = B.buildFCopysign(F32, R, X);
4586 if (Flags & MachineInstr::FmNoNans) {
4587 B.buildSelect(Dst, YIsOdd, Neg, R);
4588 MI.eraseFromParent();
4589 return true;
4590 }
4591 R = B.buildSelect(F32, YIsOdd, Neg, R).getReg(0);
4592
4593 // A negative finite base to a non-integral power is NaN. -inf is excluded:
4594 // the core already gives pow(+inf, y). So are subnormals when flushed.
4595 FPClassTest NegFiniteMask = fcNegNormal;
4596 if (!B.getMF().getDenormalMode(APFloat::IEEEsingle()).inputsAreZero())
4597 NegFiniteMask |= fcNegSubnormal;
4598 auto XNegFinite = B.buildIsFPClass(S1, X, NegFiniteMask);
4599 // Not an ONE compare: pow(-1, NaN) needs the NaN-true behavior of !OEQ.
4600 auto NegNonInt = B.buildAnd(S1, XNegFinite, B.buildNot(S1, YIsInt));
4601 auto NaN = B.buildFConstant(F32, APFloat::getQNaN(APFloat::IEEEsingle()));
4602 B.buildSelect(Dst, NegNonInt, NaN, R);
4603
4604 MI.eraseFromParent();
4605 return true;
4606}
4607
4608// Find a source register, ignoring any possible source modifiers.
4610 Register ModSrc = OrigSrc;
4611 if (MachineInstr *SrcFNeg = getOpcodeDef(AMDGPU::G_FNEG, ModSrc, MRI)) {
4612 ModSrc = SrcFNeg->getOperand(1).getReg();
4613 if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
4614 ModSrc = SrcFAbs->getOperand(1).getReg();
4615 } else if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
4616 ModSrc = SrcFAbs->getOperand(1).getReg();
4617 return ModSrc;
4618}
4619
4622 MachineIRBuilder &B) const {
4623
4624 const LLT S1 = LLT::scalar(1);
4625 Register Dst = MI.getOperand(0).getReg();
4626 Register OrigSrc = MI.getOperand(1).getReg();
4627 unsigned Flags = MI.getFlags();
4628 assert(ST.hasFractBug() && MRI.getType(Dst) == F64 &&
4629 "this should not have been custom lowered");
4630
4631 // V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x))
4632 // is used instead. However, SI doesn't have V_FLOOR_F64, so the most
4633 // efficient way to implement it is using V_FRACT_F64. The workaround for the
4634 // V_FRACT bug is:
4635 // fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999)
4636 //
4637 // Convert floor(x) to (x - fract(x))
4638
4639 auto Fract = B.buildIntrinsic(Intrinsic::amdgcn_fract, {F64})
4640 .addUse(OrigSrc)
4641 .setMIFlags(Flags);
4642
4643 // Give source modifier matching some assistance before obscuring a foldable
4644 // pattern.
4645
4646 // TODO: We can avoid the neg on the fract? The input sign to fract
4647 // shouldn't matter?
4648 Register ModSrc = stripAnySourceMods(OrigSrc, MRI);
4649
4650 auto Const =
4651 B.buildFConstant(F64, llvm::bit_cast<double>(0x3fefffffffffffff));
4652
4653 Register Min = MRI.createGenericVirtualRegister(F64);
4654
4655 // We don't need to concern ourselves with the snan handling difference, so
4656 // use the one which will directly select.
4657 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
4658 if (MFI->getMode().IEEE && !ST.hasIEEEMinimumMaximumInsts())
4659 B.buildFMinNumIEEE(Min, Fract, Const, Flags);
4660 else
4661 B.buildFMinNum(Min, Fract, Const, Flags);
4662
4663 Register CorrectedFract = Min;
4664 if (!MI.getFlag(MachineInstr::FmNoNans)) {
4665 auto IsNan = B.buildFCmp(CmpInst::FCMP_ORD, S1, ModSrc, ModSrc, Flags);
4666 CorrectedFract = B.buildSelect(F64, IsNan, ModSrc, Min, Flags).getReg(0);
4667 }
4668
4669 auto NegFract = B.buildFNeg(F64, CorrectedFract, Flags);
4670 B.buildFAdd(Dst, OrigSrc, NegFract, Flags);
4671
4672 MI.eraseFromParent();
4673 return true;
4674}
4675
4676// Turn an illegal packed v2i16/v2f16 build vector into bit operations.
4677// TODO: This should probably be a bitcast action in LegalizerHelper.
4680 Register Dst = MI.getOperand(0).getReg();
4681 const LLT I32 = LLT::integer(32);
4682 const LLT I16 = LLT::integer(16);
4683 assert(MRI.getType(Dst).isVector() &&
4684 MRI.getType(Dst).getNumElements() == 2 &&
4685 MRI.getType(Dst).getScalarSizeInBits() == 16);
4686
4687 Register Src0 = MI.getOperand(1).getReg();
4688 Register Src1 = MI.getOperand(2).getReg();
4689
4690 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC) {
4691 assert(MRI.getType(Src0) == I32);
4692 Src0 = B.buildTrunc(I16, MI.getOperand(1).getReg()).getReg(0);
4693 Src1 = B.buildTrunc(I16, MI.getOperand(2).getReg()).getReg(0);
4694 }
4695
4696 auto Merge = B.buildMergeLikeInstr(I32, {Src0, Src1});
4697 B.buildBitcast(Dst, Merge);
4698
4699 MI.eraseFromParent();
4700 return true;
4701}
4702
4703// Build a big integer multiply or multiply-add using MAD_64_32 instructions.
4704//
4705// Source and accumulation registers must all be 32-bits.
4706//
4707// TODO: When the multiply is uniform, we should produce a code sequence
4708// that is better suited to instruction selection on the SALU. Instead of
4709// the outer loop going over parts of the result, the outer loop should go
4710// over parts of one of the factors. This should result in instruction
4711// selection that makes full use of S_ADDC_U32 instructions.
4714 ArrayRef<Register> Src0,
4715 ArrayRef<Register> Src1,
4716 bool UsePartialMad64_32,
4717 bool SeparateOddAlignedProducts) const {
4718 // Use (possibly empty) vectors of S1 registers to represent the set of
4719 // carries from one pair of positions to the next.
4720 using Carry = SmallVector<Register, 2>;
4721
4722 MachineIRBuilder &B = Helper.MIRBuilder;
4723 GISelValueTracking &VT = *Helper.getValueTracking();
4724
4725 const LLT S1 = LLT::scalar(1);
4726 const LLT I32 = LLT::integer(32);
4727 const LLT I64 = LLT::integer(64);
4728
4729 Register Zero32;
4730 Register Zero64;
4731
4732 auto getZero32 = [&]() -> Register {
4733 if (!Zero32)
4734 Zero32 = B.buildConstant(I32, 0).getReg(0);
4735 return Zero32;
4736 };
4737 auto getZero64 = [&]() -> Register {
4738 if (!Zero64)
4739 Zero64 = B.buildConstant(I64, 0).getReg(0);
4740 return Zero64;
4741 };
4742
4743 SmallVector<bool, 2> Src0KnownZeros, Src1KnownZeros;
4744 for (unsigned i = 0; i < Src0.size(); ++i) {
4745 Src0KnownZeros.push_back(VT.getKnownBits(Src0[i]).isZero());
4746 Src1KnownZeros.push_back(VT.getKnownBits(Src1[i]).isZero());
4747 }
4748
4749 // Merge the given carries into the 32-bit LocalAccum, which is modified
4750 // in-place.
4751 //
4752 // Returns the carry-out, which is a single S1 register or null.
4753 auto mergeCarry =
4754 [&](Register &LocalAccum, const Carry &CarryIn) -> Register {
4755 if (CarryIn.empty())
4756 return Register();
4757
4758 bool HaveCarryOut = true;
4759 Register CarryAccum;
4760 if (CarryIn.size() == 1) {
4761 if (!LocalAccum) {
4762 LocalAccum = B.buildZExt(I32, CarryIn[0]).getReg(0);
4763 return Register();
4764 }
4765
4766 CarryAccum = getZero32();
4767 } else {
4768 CarryAccum = B.buildZExt(I32, CarryIn[0]).getReg(0);
4769 for (unsigned i = 1; i + 1 < CarryIn.size(); ++i) {
4770 CarryAccum =
4771 B.buildUAdde(I32, S1, CarryAccum, getZero32(), CarryIn[i])
4772 .getReg(0);
4773 }
4774
4775 if (!LocalAccum) {
4776 LocalAccum = getZero32();
4777 HaveCarryOut = false;
4778 }
4779 }
4780
4781 auto Add =
4782 B.buildUAdde(I32, S1, CarryAccum, LocalAccum, CarryIn.back());
4783 LocalAccum = Add.getReg(0);
4784 return HaveCarryOut ? Add.getReg(1) : Register();
4785 };
4786
4787 // Build a multiply-add chain to compute
4788 //
4789 // LocalAccum + (partial products at DstIndex)
4790 // + (opportunistic subset of CarryIn)
4791 //
4792 // LocalAccum is an array of one or two 32-bit registers that are updated
4793 // in-place. The incoming registers may be null.
4794 //
4795 // In some edge cases, carry-ins can be consumed "for free". In that case,
4796 // the consumed carry bits are removed from CarryIn in-place.
4797 auto buildMadChain =
4798 [&](MutableArrayRef<Register> LocalAccum, unsigned DstIndex, Carry &CarryIn)
4799 -> Carry {
4800 assert((DstIndex + 1 < Accum.size() && LocalAccum.size() == 2) ||
4801 (DstIndex + 1 >= Accum.size() && LocalAccum.size() == 1));
4802
4803 Carry CarryOut;
4804 unsigned j0 = 0;
4805
4806 // Use plain 32-bit multiplication for the most significant part of the
4807 // result by default.
4808 if (LocalAccum.size() == 1 &&
4809 (!UsePartialMad64_32 || !CarryIn.empty())) {
4810 do {
4811 // Skip multiplication if one of the operands is 0
4812 unsigned j1 = DstIndex - j0;
4813 if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
4814 ++j0;
4815 continue;
4816 }
4817 auto Mul = B.buildMul(I32, Src0[j0], Src1[j1]);
4818 if (!LocalAccum[0] || VT.getKnownBits(LocalAccum[0]).isZero()) {
4819 LocalAccum[0] = Mul.getReg(0);
4820 } else {
4821 if (CarryIn.empty()) {
4822 LocalAccum[0] = B.buildAdd(I32, LocalAccum[0], Mul).getReg(0);
4823 } else {
4824 LocalAccum[0] =
4825 B.buildUAdde(I32, S1, LocalAccum[0], Mul, CarryIn.back())
4826 .getReg(0);
4827 CarryIn.pop_back();
4828 }
4829 }
4830 ++j0;
4831 } while (j0 <= DstIndex && (!UsePartialMad64_32 || !CarryIn.empty()));
4832 }
4833
4834 // Build full 64-bit multiplies.
4835 if (j0 <= DstIndex) {
4836 bool HaveSmallAccum = false;
4837 Register Tmp;
4838
4839 if (LocalAccum[0]) {
4840 if (LocalAccum.size() == 1) {
4841 Tmp = B.buildAnyExt(I64, LocalAccum[0]).getReg(0);
4842 HaveSmallAccum = true;
4843 } else if (LocalAccum[1]) {
4844 Tmp = B.buildMergeLikeInstr(I64, LocalAccum).getReg(0);
4845 HaveSmallAccum = false;
4846 } else {
4847 Tmp = B.buildZExt(I64, LocalAccum[0]).getReg(0);
4848 HaveSmallAccum = true;
4849 }
4850 } else {
4851 assert(LocalAccum.size() == 1 || !LocalAccum[1]);
4852 Tmp = getZero64();
4853 HaveSmallAccum = true;
4854 }
4855
4856 do {
4857 unsigned j1 = DstIndex - j0;
4858 if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
4859 ++j0;
4860 continue;
4861 }
4862 auto Mad = B.buildInstr(AMDGPU::G_AMDGPU_MAD_U64_U32, {I64, S1},
4863 {Src0[j0], Src1[j1], Tmp});
4864 Tmp = Mad.getReg(0);
4865 if (!HaveSmallAccum)
4866 CarryOut.push_back(Mad.getReg(1));
4867 HaveSmallAccum = false;
4868
4869 ++j0;
4870 } while (j0 <= DstIndex);
4871
4872 auto Unmerge = B.buildUnmerge(I32, Tmp);
4873 LocalAccum[0] = Unmerge.getReg(0);
4874 if (LocalAccum.size() > 1)
4875 LocalAccum[1] = Unmerge.getReg(1);
4876 }
4877
4878 // Every partial product contributing to this destination index was
4879 // skipped because an operand half is known zero, so nothing has been
4880 // accumulated and the result is zero.
4881 if (!LocalAccum[0])
4882 LocalAccum[0] = getZero32();
4883
4884 // A second element is only ever requested when the full 64-bit multiply
4885 // block above runs, which always writes it.
4886 assert((LocalAccum.size() == 1 || LocalAccum[1]) &&
4887 "Uninitialized accumulator part");
4888
4889 return CarryOut;
4890 };
4891
4892 // Outer multiply loop, iterating over destination parts from least
4893 // significant to most significant parts.
4894 //
4895 // The columns of the following diagram correspond to the destination parts
4896 // affected by one iteration of the outer loop (ignoring boundary
4897 // conditions).
4898 //
4899 // Dest index relative to 2 * i: 1 0 -1
4900 // ------
4901 // Carries from previous iteration: e o
4902 // Even-aligned partial product sum: E E .
4903 // Odd-aligned partial product sum: O O
4904 //
4905 // 'o' is OddCarry, 'e' is EvenCarry.
4906 // EE and OO are computed from partial products via buildMadChain and use
4907 // accumulation where possible and appropriate.
4908 //
4909 Register SeparateOddCarry;
4910 Carry EvenCarry;
4911 Carry OddCarry;
4912
4913 for (unsigned i = 0; i <= Accum.size() / 2; ++i) {
4914 Carry OddCarryIn = std::move(OddCarry);
4915 Carry EvenCarryIn = std::move(EvenCarry);
4916 OddCarry.clear();
4917 EvenCarry.clear();
4918
4919 // Partial products at offset 2 * i.
4920 if (2 * i < Accum.size()) {
4921 auto LocalAccum = Accum.drop_front(2 * i).take_front(2);
4922 EvenCarry = buildMadChain(LocalAccum, 2 * i, EvenCarryIn);
4923 }
4924
4925 // Partial products at offset 2 * i - 1.
4926 if (i > 0) {
4927 if (!SeparateOddAlignedProducts) {
4928 auto LocalAccum = Accum.drop_front(2 * i - 1).take_front(2);
4929 OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
4930 } else {
4931 bool IsHighest = 2 * i >= Accum.size();
4932 Register SeparateOddOut[2];
4933 auto LocalAccum = MutableArrayRef(SeparateOddOut)
4934 .take_front(IsHighest ? 1 : 2);
4935 OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
4936
4938
4939 if (i == 1) {
4940 if (!IsHighest)
4941 Lo = B.buildUAddo(I32, S1, Accum[2 * i - 1], SeparateOddOut[0]);
4942 else
4943 Lo = B.buildAdd(I32, Accum[2 * i - 1], SeparateOddOut[0]);
4944 } else {
4945 Lo = B.buildUAdde(I32, S1, Accum[2 * i - 1], SeparateOddOut[0],
4946 SeparateOddCarry);
4947 }
4948 Accum[2 * i - 1] = Lo->getOperand(0).getReg();
4949
4950 if (!IsHighest) {
4951 auto Hi = B.buildUAdde(I32, S1, Accum[2 * i], SeparateOddOut[1],
4952 Lo->getOperand(1).getReg());
4953 Accum[2 * i] = Hi.getReg(0);
4954 SeparateOddCarry = Hi.getReg(1);
4955 }
4956 }
4957 }
4958
4959 // Add in the carries from the previous iteration
4960 if (i > 0) {
4961 if (Register CarryOut = mergeCarry(Accum[2 * i - 1], OddCarryIn))
4962 EvenCarryIn.push_back(CarryOut);
4963
4964 if (2 * i < Accum.size()) {
4965 if (Register CarryOut = mergeCarry(Accum[2 * i], EvenCarryIn))
4966 OddCarry.push_back(CarryOut);
4967 }
4968 }
4969 }
4970}
4971
4972// Custom narrowing of wide multiplies using wide multiply-add instructions.
4973//
4974// TODO: If the multiply is followed by an addition, we should attempt to
4975// integrate it to make better use of V_MAD_U64_U32's multiply-add capabilities.
4977 MachineInstr &MI) const {
4978 assert(ST.hasMad64_32());
4979 assert(MI.getOpcode() == TargetOpcode::G_MUL);
4980
4981 MachineIRBuilder &B = Helper.MIRBuilder;
4982 MachineRegisterInfo &MRI = *B.getMRI();
4983
4984 Register DstReg = MI.getOperand(0).getReg();
4985 Register Src0 = MI.getOperand(1).getReg();
4986 Register Src1 = MI.getOperand(2).getReg();
4987
4988 LLT Ty = MRI.getType(DstReg);
4989 assert(Ty.isScalar());
4990
4991 unsigned Size = Ty.getSizeInBits();
4992 if (ST.useVMulU64Inst() && Size == 64)
4993 return true;
4994
4995 unsigned NumParts = Size / 32;
4996 assert((Size % 32) == 0);
4997 assert(NumParts >= 2);
4998
4999 // Whether to use MAD_64_32 for partial products whose high half is
5000 // discarded. This avoids some ADD instructions but risks false dependency
5001 // stalls on some subtargets in some cases.
5002 const bool UsePartialMad64_32 = ST.getGeneration() < AMDGPUSubtarget::GFX10;
5003
5004 // Whether to compute odd-aligned partial products separately. This is
5005 // advisable on subtargets where the accumulator of MAD_64_32 must be placed
5006 // in an even-aligned VGPR.
5007 const bool SeparateOddAlignedProducts = ST.hasFullRate64Ops();
5008
5009 LLT I32 = LLT::integer(32);
5010 SmallVector<Register, 2> Src0Parts, Src1Parts;
5011 for (unsigned i = 0; i < NumParts; ++i) {
5012 Src0Parts.push_back(MRI.createGenericVirtualRegister(I32));
5013 Src1Parts.push_back(MRI.createGenericVirtualRegister(I32));
5014 }
5015 B.buildUnmerge(Src0Parts, Src0);
5016 B.buildUnmerge(Src1Parts, Src1);
5017
5019 buildMultiply(Helper, AccumRegs, Src0Parts, Src1Parts, UsePartialMad64_32,
5020 SeparateOddAlignedProducts);
5021
5022 B.buildMergeLikeInstr(DstReg, AccumRegs);
5023 MI.eraseFromParent();
5024 return true;
5025}
5026
5027// Legalize ctlz/cttz to ffbh/ffbl instead of the default legalization to
5028// ctlz/cttz_zero_poison. This allows us to fix up the result for the zero input
5029// case with a single min instruction instead of a compare+select.
5032 MachineIRBuilder &B) const {
5033 Register Dst = MI.getOperand(0).getReg();
5034 Register Src = MI.getOperand(1).getReg();
5035 LLT DstTy = MRI.getType(Dst);
5036 LLT SrcTy = MRI.getType(Src);
5037
5038 unsigned NewOpc = MI.getOpcode() == AMDGPU::G_CTLZ
5039 ? AMDGPU::G_AMDGPU_FFBH_U32
5040 : AMDGPU::G_AMDGPU_FFBL_B32;
5041 auto Tmp = B.buildInstr(NewOpc, {DstTy}, {Src});
5042 B.buildUMin(Dst, Tmp, B.buildConstant(DstTy, SrcTy.getSizeInBits()));
5043
5044 MI.eraseFromParent();
5045 return true;
5046}
5047
5050 MachineIRBuilder &B) const {
5051 Register Dst = MI.getOperand(0).getReg();
5052 Register Src = MI.getOperand(1).getReg();
5053 LLT SrcTy = MRI.getType(Src);
5054 TypeSize NumBits = SrcTy.getSizeInBits();
5055
5056 assert(NumBits < 32u);
5057
5058 const LLT I32 = LLT::integer(32);
5059 auto ShiftAmt = B.buildConstant(I32, 32u - NumBits);
5060 auto Extend = B.buildAnyExt(I32, {Src}).getReg(0u);
5061 auto Shift = B.buildShl(I32, Extend, ShiftAmt);
5062 auto Ctlz = B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {I32}, {Shift});
5063 B.buildTrunc(Dst, Ctlz);
5064 MI.eraseFromParent();
5065 return true;
5066}
5067
5070 MachineIRBuilder &B) const {
5071 Register Dst = MI.getOperand(0).getReg();
5072 Register Src = MI.getOperand(1).getReg();
5073 LLT SrcTy = MRI.getType(Src);
5074 const LLT I32 = LLT::integer(32);
5075 assert(SrcTy == I32 && "legalizeCTLS only supports i32");
5076 unsigned BitWidth = SrcTy.getSizeInBits();
5077
5078 auto Sffbh = B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {I32}).addUse(Src);
5079 auto Clamped = B.buildUMin(I32, Sffbh, B.buildConstant(I32, BitWidth));
5080 B.buildSub(Dst, Clamped, B.buildConstant(I32, 1));
5081 MI.eraseFromParent();
5082 return true;
5083}
5084
5085// Check that this is a G_XOR x, -1
5086static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI) {
5087 if (MI.getOpcode() != TargetOpcode::G_XOR)
5088 return false;
5089 auto ConstVal = getIConstantVRegSExtVal(MI.getOperand(2).getReg(), MRI);
5090 return ConstVal == -1;
5091}
5092
5093// Return the use branch instruction, otherwise null if the usage is invalid.
5094static MachineInstr *
5096 MachineBasicBlock *&UncondBrTarget, bool &Negated) {
5097 Register CondDef = MI.getOperand(0).getReg();
5098 if (!MRI.hasOneNonDBGUse(CondDef))
5099 return nullptr;
5100
5101 MachineBasicBlock *Parent = MI.getParent();
5102 MachineInstr *UseMI = &*MRI.use_instr_nodbg_begin(CondDef);
5103
5104 if (isNot(MRI, *UseMI)) {
5105 Register NegatedCond = UseMI->getOperand(0).getReg();
5106 if (!MRI.hasOneNonDBGUse(NegatedCond))
5107 return nullptr;
5108
5109 // We're deleting the def of this value, so we need to remove it.
5110 eraseInstr(*UseMI, MRI);
5111
5112 UseMI = &*MRI.use_instr_nodbg_begin(NegatedCond);
5113 Negated = true;
5114 }
5115
5116 if (UseMI->getParent() != Parent || UseMI->getOpcode() != AMDGPU::G_BRCOND)
5117 return nullptr;
5118
5119 // Make sure the cond br is followed by a G_BR, or is the last instruction.
5120 MachineBasicBlock::iterator Next = std::next(UseMI->getIterator());
5121 if (Next == Parent->end()) {
5122 MachineFunction::iterator NextMBB = std::next(Parent->getIterator());
5123 if (NextMBB == Parent->getParent()->end()) // Illegal intrinsic use.
5124 return nullptr;
5125 UncondBrTarget = &*NextMBB;
5126 } else {
5127 if (Next->getOpcode() != AMDGPU::G_BR)
5128 return nullptr;
5129 Br = &*Next;
5130 UncondBrTarget = Br->getOperand(0).getMBB();
5131 }
5132
5133 return UseMI;
5134}
5135
5138 const ArgDescriptor *Arg,
5139 const TargetRegisterClass *ArgRC,
5140 LLT ArgTy) const {
5141 MCRegister SrcReg = Arg->getRegister();
5142 assert(SrcReg.isPhysical() && "Physical register expected");
5143 assert(DstReg.isVirtual() && "Virtual register expected");
5144
5145 Register LiveIn = getFunctionLiveInPhysReg(B.getMF(), B.getTII(), SrcReg,
5146 *ArgRC, B.getDebugLoc(), ArgTy);
5147 if (Arg->isMasked()) {
5148 // TODO: Should we try to emit this once in the entry block?
5149 const LLT I32 = LLT::integer(32);
5150 const unsigned Mask = Arg->getMask();
5151 const unsigned Shift = llvm::countr_zero<unsigned>(Mask);
5152
5153 Register AndMaskSrc = LiveIn;
5154
5155 // TODO: Avoid clearing the high bits if we know workitem id y/z are always
5156 // 0.
5157 if (Shift != 0) {
5158 auto ShiftAmt = B.buildConstant(I32, Shift);
5159 AndMaskSrc = B.buildLShr(I32, LiveIn, ShiftAmt).getReg(0);
5160 }
5161
5162 B.buildAnd(DstReg, AndMaskSrc, B.buildConstant(I32, Mask >> Shift));
5163 } else {
5164 B.buildCopy(DstReg, LiveIn);
5165 }
5166}
5167
5172 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
5173 Register DstReg = MI.getOperand(0).getReg();
5174 if (!ST.hasClusters()) {
5175 if (!loadInputValue(DstReg, B, WorkGroupIdPV))
5176 return false;
5177 MI.eraseFromParent();
5178 return true;
5179 }
5180
5181 // Clusters are supported. Return the global position in the grid. If clusters
5182 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
5183
5184 // WorkGroupIdXYZ = ClusterId == 0 ?
5185 // ClusterIdXYZ :
5186 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
5187 MachineRegisterInfo &MRI = *B.getMRI();
5188 const LLT I32 = LLT::integer(32);
5189 Register ClusterIdXYZ = MRI.createGenericVirtualRegister(I32);
5190 Register ClusterMaxIdXYZ = MRI.createGenericVirtualRegister(I32);
5191 Register ClusterWorkGroupIdXYZ = MRI.createGenericVirtualRegister(I32);
5192 if (!loadInputValue(ClusterIdXYZ, B, WorkGroupIdPV) ||
5193 !loadInputValue(ClusterWorkGroupIdXYZ, B, ClusterWorkGroupIdPV) ||
5194 !loadInputValue(ClusterMaxIdXYZ, B, ClusterMaxIdPV))
5195 return false;
5196
5197 auto One = B.buildConstant(I32, 1);
5198 auto ClusterSizeXYZ = B.buildAdd(I32, ClusterMaxIdXYZ, One);
5199 auto GlobalIdXYZ = B.buildAdd(I32, ClusterWorkGroupIdXYZ,
5200 B.buildMul(I32, ClusterIdXYZ, ClusterSizeXYZ));
5201
5202 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
5203
5204 switch (MFI->getClusterDims().getKind()) {
5207 B.buildCopy(DstReg, GlobalIdXYZ);
5208 MI.eraseFromParent();
5209 return true;
5210 }
5212 B.buildCopy(DstReg, ClusterIdXYZ);
5213 MI.eraseFromParent();
5214 return true;
5215 }
5217 using namespace AMDGPU::Hwreg;
5218 unsigned ClusterIdField = HwregEncoding::encode(ID_IB_STS2, 6, 4);
5219 Register ClusterId = MRI.createGenericVirtualRegister(I32);
5220 MRI.setRegClass(ClusterId, &AMDGPU::SReg_32RegClass);
5221 B.buildInstr(AMDGPU::S_GETREG_B32_const)
5222 .addDef(ClusterId)
5223 .addImm(ClusterIdField);
5224 auto Zero = B.buildConstant(I32, 0);
5225 auto NoClusters =
5226 B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), ClusterId, Zero);
5227 B.buildSelect(DstReg, NoClusters, ClusterIdXYZ, GlobalIdXYZ);
5228 MI.eraseFromParent();
5229 return true;
5230 }
5231 }
5232
5233 llvm_unreachable("nothing should reach here");
5234}
5235
5237 Register DstReg, MachineIRBuilder &B,
5239 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
5240 const ArgDescriptor *Arg = nullptr;
5241 const TargetRegisterClass *ArgRC = nullptr;
5242 LLT ArgTy;
5243
5244 CallingConv::ID CC = B.getMF().getFunction().getCallingConv();
5245 const ArgDescriptor WorkGroupIDX =
5246 ArgDescriptor::createRegister(AMDGPU::TTMP9);
5247 // If GridZ is not programmed in an entry function then the hardware will set
5248 // it to all zeros, so there is no need to mask the GridY value in the low
5249 // order bits.
5250 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
5251 AMDGPU::TTMP7,
5252 AMDGPU::isEntryFunctionCC(CC) && !MFI->hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
5253 const ArgDescriptor WorkGroupIDZ =
5254 ArgDescriptor::createRegister(AMDGPU::TTMP7, 0xFFFF0000u);
5255 const ArgDescriptor ClusterWorkGroupIDX =
5256 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000000Fu);
5257 const ArgDescriptor ClusterWorkGroupIDY =
5258 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000000F0u);
5259 const ArgDescriptor ClusterWorkGroupIDZ =
5260 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00000F00u);
5261 const ArgDescriptor ClusterWorkGroupMaxIDX =
5262 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000F000u);
5263 const ArgDescriptor ClusterWorkGroupMaxIDY =
5264 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000F0000u);
5265 const ArgDescriptor ClusterWorkGroupMaxIDZ =
5266 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00F00000u);
5267 const ArgDescriptor ClusterWorkGroupMaxFlatID =
5268 ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0F000000u);
5269
5270 auto LoadConstant = [&](unsigned N) {
5271 B.buildConstant(DstReg, N);
5272 return true;
5273 };
5274
5275 if (ST.hasArchitectedSGPRs() &&
5278 AMDGPU::ClusterDimsAttr ClusterDims = MFI->getClusterDims();
5279 bool HasFixedDims = ClusterDims.isFixedDims();
5280
5281 switch (ArgType) {
5283 Arg = &WorkGroupIDX;
5284 ArgRC = &AMDGPU::SReg_32RegClass;
5285 ArgTy = LLT::integer(32);
5286 break;
5288 Arg = &WorkGroupIDY;
5289 ArgRC = &AMDGPU::SReg_32RegClass;
5290 ArgTy = LLT::integer(32);
5291 break;
5293 Arg = &WorkGroupIDZ;
5294 ArgRC = &AMDGPU::SReg_32RegClass;
5295 ArgTy = LLT::integer(32);
5296 break;
5298 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
5299 return LoadConstant(0);
5300 Arg = &ClusterWorkGroupIDX;
5301 ArgRC = &AMDGPU::SReg_32RegClass;
5302 ArgTy = LLT::integer(32);
5303 break;
5305 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
5306 return LoadConstant(0);
5307 Arg = &ClusterWorkGroupIDY;
5308 ArgRC = &AMDGPU::SReg_32RegClass;
5309 ArgTy = LLT::integer(32);
5310 break;
5312 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
5313 return LoadConstant(0);
5314 Arg = &ClusterWorkGroupIDZ;
5315 ArgRC = &AMDGPU::SReg_32RegClass;
5316 ArgTy = LLT::integer(32);
5317 break;
5319 if (HasFixedDims)
5320 return LoadConstant(ClusterDims.getDims()[0] - 1);
5321 Arg = &ClusterWorkGroupMaxIDX;
5322 ArgRC = &AMDGPU::SReg_32RegClass;
5323 ArgTy = LLT::integer(32);
5324 break;
5326 if (HasFixedDims)
5327 return LoadConstant(ClusterDims.getDims()[1] - 1);
5328 Arg = &ClusterWorkGroupMaxIDY;
5329 ArgRC = &AMDGPU::SReg_32RegClass;
5330 ArgTy = LLT::integer(32);
5331 break;
5333 if (HasFixedDims)
5334 return LoadConstant(ClusterDims.getDims()[2] - 1);
5335 Arg = &ClusterWorkGroupMaxIDZ;
5336 ArgRC = &AMDGPU::SReg_32RegClass;
5337 ArgTy = LLT::integer(32);
5338 break;
5340 Arg = &ClusterWorkGroupMaxFlatID;
5341 ArgRC = &AMDGPU::SReg_32RegClass;
5342 ArgTy = LLT::integer(32);
5343 break;
5344 default:
5345 break;
5346 }
5347 }
5348
5349 if (!Arg)
5350 std::tie(Arg, ArgRC, ArgTy) = MFI->getPreloadedValue(ArgType);
5351
5352 if (!Arg) {
5354 // The intrinsic may appear when we have a 0 sized kernarg segment, in
5355 // which case the pointer argument may be missing and we use null.
5356 return LoadConstant(0);
5357 }
5358
5359 // It's undefined behavior if a function marked with the amdgpu-no-*
5360 // attributes uses the corresponding intrinsic.
5361 B.buildUndef(DstReg);
5362 return true;
5363 }
5364
5365 if (!Arg->isRegister() || !Arg->getRegister().isValid())
5366 return false; // TODO: Handle these
5367 buildLoadInputValue(DstReg, B, Arg, ArgRC, ArgTy);
5368 return true;
5369}
5370
5374 if (!loadInputValue(MI.getOperand(0).getReg(), B, ArgType))
5375 return false;
5376
5377 MI.eraseFromParent();
5378 return true;
5379}
5380
5382 int64_t C) {
5383 B.buildConstant(MI.getOperand(0).getReg(), C);
5384 MI.eraseFromParent();
5385 return true;
5386}
5387
5390 unsigned Dim, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
5391 unsigned MaxID = ST.getMaxWorkitemID(B.getMF().getFunction(), Dim);
5392 if (MaxID == 0)
5393 return replaceWithConstant(B, MI, 0);
5394
5395 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
5396 const ArgDescriptor *Arg;
5397 const TargetRegisterClass *ArgRC;
5398 LLT ArgTy;
5399 std::tie(Arg, ArgRC, ArgTy) = MFI->getPreloadedValue(ArgType);
5400
5401 Register DstReg = MI.getOperand(0).getReg();
5402 if (!Arg) {
5403 // It's undefined behavior if a function marked with the amdgpu-no-*
5404 // attributes uses the corresponding intrinsic.
5405 B.buildUndef(DstReg);
5406 MI.eraseFromParent();
5407 return true;
5408 }
5409
5410 if (Arg->isMasked()) {
5411 // Don't bother inserting AssertZext for packed IDs since we're emitting the
5412 // masking operations anyway.
5413 //
5414 // TODO: We could assert the top bit is 0 for the source copy.
5415 if (!loadInputValue(DstReg, B, ArgType))
5416 return false;
5417 } else {
5418 Register TmpReg = MRI.createGenericVirtualRegister(LLT::integer(32));
5419 if (!loadInputValue(TmpReg, B, ArgType))
5420 return false;
5421 B.buildAssertZExt(DstReg, TmpReg, llvm::bit_width(MaxID));
5422 }
5423
5424 MI.eraseFromParent();
5425 return true;
5426}
5427
5430 // This isn't really a constant pool but close enough.
5433 return PtrInfo;
5434}
5435
5437 int64_t Offset) const {
5439 Register KernArgReg = B.getMRI()->createGenericVirtualRegister(PtrTy);
5440
5441 // TODO: If we passed in the base kernel offset we could have a better
5442 // alignment than 4, but we don't really need it.
5443 if (!loadInputValue(KernArgReg, B,
5445 llvm_unreachable("failed to find kernarg segment ptr");
5446
5447 auto COffset = B.buildConstant(LLT::integer(64), Offset);
5448 return B.buildObjectPtrOffset(PtrTy, KernArgReg, COffset).getReg(0);
5449}
5450
5451/// Legalize a value that's loaded from kernel arguments. This is only used by
5452/// legacy intrinsics.
5455 uint64_t Offset,
5456 Align Alignment) const {
5457 Register DstReg = MI.getOperand(0).getReg();
5458
5459 assert(B.getMRI()->getType(DstReg) == LLT::integer(32) &&
5460 "unexpected kernarg parameter type");
5461
5464 B.buildLoad(DstReg, Ptr, PtrInfo.getWithOffset(Offset), Align(4),
5467 MI.eraseFromParent();
5468 return true;
5469}
5470
5473 MachineIRBuilder &B) const {
5474 Register Dst = MI.getOperand(0).getReg();
5475 LLT DstTy = MRI.getType(Dst);
5476
5477 if (DstTy == F16)
5478 return legalizeFDIV16(MI, MRI, B);
5479 if (DstTy == F32)
5480 return legalizeFDIV32(MI, MRI, B);
5481 if (DstTy == F64)
5482 return legalizeFDIV64(MI, MRI, B);
5483
5484 return false;
5485}
5486
5488 Register DstDivReg,
5489 Register DstRemReg,
5490 Register X,
5491 Register Y) const {
5492 const LLT S1 = LLT::scalar(1);
5493 const LLT I32 = LLT::integer(32);
5494
5495 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
5496 // algorithm used here.
5497
5498 // Initial estimate of inv(y).
5499 auto FloatY = B.buildUITOFP(F32, Y);
5500 auto RcpIFlag = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {F32}, {FloatY});
5501 auto Scale = B.buildFConstant(F32, llvm::bit_cast<float>(0x4f7ffffe));
5502 auto ScaledY = B.buildFMul(F32, RcpIFlag, Scale);
5503 auto Z = B.buildFPTOUI(I32, ScaledY);
5504
5505 // One round of UNR.
5506 auto NegY = B.buildSub(I32, B.buildConstant(I32, 0), Y);
5507 auto NegYZ = B.buildMul(I32, NegY, Z);
5508 Z = B.buildAdd(I32, Z, B.buildUMulH(I32, Z, NegYZ));
5509
5510 // Quotient/remainder estimate.
5511 auto Q = B.buildUMulH(I32, X, Z);
5512 auto R = B.buildSub(I32, X, B.buildMul(I32, Q, Y));
5513
5514 // First quotient/remainder refinement.
5515 auto One = B.buildConstant(I32, 1);
5516 auto Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
5517 if (DstDivReg)
5518 Q = B.buildSelect(I32, Cond, B.buildAdd(I32, Q, One), Q);
5519 R = B.buildSelect(I32, Cond, B.buildSub(I32, R, Y), R);
5520
5521 // Second quotient/remainder refinement.
5522 Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
5523 if (DstDivReg)
5524 B.buildSelect(DstDivReg, Cond, B.buildAdd(I32, Q, One), Q);
5525
5526 if (DstRemReg)
5527 B.buildSelect(DstRemReg, Cond, B.buildSub(I32, R, Y), R);
5528}
5529
5530// Build integer reciprocal sequence around V_RCP_IFLAG_F32
5531//
5532// Return lo, hi of result
5533//
5534// %cvt.lo = G_UITOFP Val.lo
5535// %cvt.hi = G_UITOFP Val.hi
5536// %mad = G_FMAD %cvt.hi, 2**32, %cvt.lo
5537// %rcp = G_AMDGPU_RCP_IFLAG %mad
5538// %mul1 = G_FMUL %rcp, 0x5f7ffffc
5539// %mul2 = G_FMUL %mul1, 2**(-32)
5540// %trunc = G_INTRINSIC_TRUNC %mul2
5541// %mad2 = G_FMAD %trunc, -(2**32), %mul1
5542// return {G_FPTOUI %mad2, G_FPTOUI %trunc}
5543static std::pair<Register, Register> emitReciprocalU64(MachineIRBuilder &B,
5544 Register Val) {
5545 const LLT I32 = LLT::integer(32);
5546 auto Unmerge = B.buildUnmerge(I32, Val);
5547
5548 auto CvtLo = B.buildUITOFP(F32, Unmerge.getReg(0));
5549 auto CvtHi = B.buildUITOFP(F32, Unmerge.getReg(1));
5550
5551 auto Mad = B.buildFMAD(
5552 F32, CvtHi, // 2**32
5553 B.buildFConstant(F32, llvm::bit_cast<float>(0x4f800000)), CvtLo);
5554
5555 auto Rcp = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {F32}, {Mad});
5556 auto Mul1 = B.buildFMul(
5557 F32, Rcp, B.buildFConstant(F32, llvm::bit_cast<float>(0x5f7ffffc)));
5558
5559 // 2**(-32)
5560 auto Mul2 = B.buildFMul(
5561 F32, Mul1, B.buildFConstant(F32, llvm::bit_cast<float>(0x2f800000)));
5562 auto Trunc = B.buildIntrinsicTrunc(F32, Mul2);
5563
5564 // -(2**32)
5565 auto Mad2 = B.buildFMAD(
5566 F32, Trunc, B.buildFConstant(F32, llvm::bit_cast<float>(0xcf800000)),
5567 Mul1);
5568
5569 auto ResultLo = B.buildFPTOUI(I32, Mad2);
5570 auto ResultHi = B.buildFPTOUI(I32, Trunc);
5571
5572 return {ResultLo.getReg(0), ResultHi.getReg(0)};
5573}
5574
5576 Register DstDivReg,
5577 Register DstRemReg,
5578 Register Numer,
5579 Register Denom) const {
5580 const LLT I32 = LLT::integer(32);
5581 const LLT I64 = LLT::integer(64);
5582 const LLT S1 = LLT::scalar(1);
5583 Register RcpLo, RcpHi;
5584
5585 std::tie(RcpLo, RcpHi) = emitReciprocalU64(B, Denom);
5586
5587 auto Rcp = B.buildMergeLikeInstr(I64, {RcpLo, RcpHi});
5588
5589 auto Zero64 = B.buildConstant(I64, 0);
5590 auto NegDenom = B.buildSub(I64, Zero64, Denom);
5591
5592 auto MulLo1 = B.buildMul(I64, NegDenom, Rcp);
5593 auto MulHi1 = B.buildUMulH(I64, Rcp, MulLo1);
5594
5595 auto UnmergeMulHi1 = B.buildUnmerge(I32, MulHi1);
5596 Register MulHi1_Lo = UnmergeMulHi1.getReg(0);
5597 Register MulHi1_Hi = UnmergeMulHi1.getReg(1);
5598
5599 auto Add1_Lo = B.buildUAddo(I32, S1, RcpLo, MulHi1_Lo);
5600 auto Add1_Hi = B.buildUAdde(I32, S1, RcpHi, MulHi1_Hi, Add1_Lo.getReg(1));
5601 auto Add1 = B.buildMergeLikeInstr(I64, {Add1_Lo, Add1_Hi});
5602
5603 auto MulLo2 = B.buildMul(I64, NegDenom, Add1);
5604 auto MulHi2 = B.buildUMulH(I64, Add1, MulLo2);
5605 auto UnmergeMulHi2 = B.buildUnmerge(I32, MulHi2);
5606 Register MulHi2_Lo = UnmergeMulHi2.getReg(0);
5607 Register MulHi2_Hi = UnmergeMulHi2.getReg(1);
5608
5609 auto Zero32 = B.buildConstant(I32, 0);
5610 auto Add2_Lo = B.buildUAddo(I32, S1, Add1_Lo, MulHi2_Lo);
5611 auto Add2_Hi = B.buildUAdde(I32, S1, Add1_Hi, MulHi2_Hi, Add2_Lo.getReg(1));
5612 auto Add2 = B.buildMergeLikeInstr(I64, {Add2_Lo, Add2_Hi});
5613
5614 auto UnmergeNumer = B.buildUnmerge(I32, Numer);
5615 Register NumerLo = UnmergeNumer.getReg(0);
5616 Register NumerHi = UnmergeNumer.getReg(1);
5617
5618 auto MulHi3 = B.buildUMulH(I64, Numer, Add2);
5619 auto Mul3 = B.buildMul(I64, Denom, MulHi3);
5620 auto UnmergeMul3 = B.buildUnmerge(I32, Mul3);
5621 Register Mul3_Lo = UnmergeMul3.getReg(0);
5622 Register Mul3_Hi = UnmergeMul3.getReg(1);
5623 auto Sub1_Lo = B.buildUSubo(I32, S1, NumerLo, Mul3_Lo);
5624 auto Sub1_Hi = B.buildUSube(I32, S1, NumerHi, Mul3_Hi, Sub1_Lo.getReg(1));
5625 auto Sub1_Mi = B.buildSub(I32, NumerHi, Mul3_Hi);
5626 auto Sub1 = B.buildMergeLikeInstr(I64, {Sub1_Lo, Sub1_Hi});
5627
5628 auto UnmergeDenom = B.buildUnmerge(I32, Denom);
5629 Register DenomLo = UnmergeDenom.getReg(0);
5630 Register DenomHi = UnmergeDenom.getReg(1);
5631
5632 auto CmpHi = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Hi, DenomHi);
5633 auto C1 = B.buildSExt(I32, CmpHi);
5634
5635 auto CmpLo = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Lo, DenomLo);
5636 auto C2 = B.buildSExt(I32, CmpLo);
5637
5638 auto CmpEq = B.buildICmp(CmpInst::ICMP_EQ, S1, Sub1_Hi, DenomHi);
5639 auto C3 = B.buildSelect(I32, CmpEq, C2, C1);
5640
5641 // TODO: Here and below portions of the code can be enclosed into if/endif.
5642 // Currently control flow is unconditional and we have 4 selects after
5643 // potential endif to substitute PHIs.
5644
5645 // if C3 != 0 ...
5646 auto Sub2_Lo = B.buildUSubo(I32, S1, Sub1_Lo, DenomLo);
5647 auto Sub2_Mi = B.buildUSube(I32, S1, Sub1_Mi, DenomHi, Sub1_Lo.getReg(1));
5648 auto Sub2_Hi = B.buildUSube(I32, S1, Sub2_Mi, Zero32, Sub2_Lo.getReg(1));
5649 auto Sub2 = B.buildMergeLikeInstr(I64, {Sub2_Lo, Sub2_Hi});
5650
5651 auto One64 = B.buildConstant(I64, 1);
5652 auto Add3 = B.buildAdd(I64, MulHi3, One64);
5653
5654 auto C4 =
5655 B.buildSExt(I32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Hi, DenomHi));
5656 auto C5 =
5657 B.buildSExt(I32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Lo, DenomLo));
5658 auto C6 = B.buildSelect(
5659 I32, B.buildICmp(CmpInst::ICMP_EQ, S1, Sub2_Hi, DenomHi), C5, C4);
5660
5661 // if (C6 != 0)
5662 auto Add4 = B.buildAdd(I64, Add3, One64);
5663 auto Sub3_Lo = B.buildUSubo(I32, S1, Sub2_Lo, DenomLo);
5664
5665 auto Sub3_Mi = B.buildUSube(I32, S1, Sub2_Mi, DenomHi, Sub2_Lo.getReg(1));
5666 auto Sub3_Hi = B.buildUSube(I32, S1, Sub3_Mi, Zero32, Sub3_Lo.getReg(1));
5667 auto Sub3 = B.buildMergeLikeInstr(I64, {Sub3_Lo, Sub3_Hi});
5668
5669 // endif C6
5670 // endif C3
5671
5672 if (DstDivReg) {
5673 auto Sel1 = B.buildSelect(
5674 I64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Add4, Add3);
5675 B.buildSelect(DstDivReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
5676 Sel1, MulHi3);
5677 }
5678
5679 if (DstRemReg) {
5680 auto Sel2 = B.buildSelect(
5681 I64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Sub3, Sub2);
5682 B.buildSelect(DstRemReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
5683 Sel2, Sub1);
5684 }
5685}
5686
5689 MachineIRBuilder &B) const {
5690 Register DstDivReg, DstRemReg;
5691 switch (MI.getOpcode()) {
5692 default:
5693 llvm_unreachable("Unexpected opcode!");
5694 case AMDGPU::G_UDIV: {
5695 DstDivReg = MI.getOperand(0).getReg();
5696 break;
5697 }
5698 case AMDGPU::G_UREM: {
5699 DstRemReg = MI.getOperand(0).getReg();
5700 break;
5701 }
5702 case AMDGPU::G_UDIVREM: {
5703 DstDivReg = MI.getOperand(0).getReg();
5704 DstRemReg = MI.getOperand(1).getReg();
5705 break;
5706 }
5707 }
5708
5709 const LLT I64 = LLT::integer(64);
5710 const LLT I32 = LLT::integer(32);
5711 const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
5712 Register Num = MI.getOperand(FirstSrcOpIdx).getReg();
5713 Register Den = MI.getOperand(FirstSrcOpIdx + 1).getReg();
5714 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
5715
5716 if (Ty == I32)
5717 legalizeUnsignedDIV_REM32Impl(B, DstDivReg, DstRemReg, Num, Den);
5718 else if (Ty == I64)
5719 legalizeUnsignedDIV_REM64Impl(B, DstDivReg, DstRemReg, Num, Den);
5720 else
5721 return false;
5722
5723 MI.eraseFromParent();
5724 return true;
5725}
5726
5729 MachineIRBuilder &B) const {
5730 const LLT I64 = LLT::integer(64);
5731 const LLT I32 = LLT::integer(32);
5732
5733 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
5734 if (Ty != I32 && Ty != I64)
5735 return false;
5736
5737 const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
5738 Register LHS = MI.getOperand(FirstSrcOpIdx).getReg();
5739 Register RHS = MI.getOperand(FirstSrcOpIdx + 1).getReg();
5740
5741 auto SignBitOffset = B.buildConstant(I32, Ty.getSizeInBits() - 1);
5742 auto LHSign = B.buildAShr(Ty, LHS, SignBitOffset);
5743 auto RHSign = B.buildAShr(Ty, RHS, SignBitOffset);
5744
5745 LHS = B.buildAdd(Ty, LHS, LHSign).getReg(0);
5746 RHS = B.buildAdd(Ty, RHS, RHSign).getReg(0);
5747
5748 LHS = B.buildXor(Ty, LHS, LHSign).getReg(0);
5749 RHS = B.buildXor(Ty, RHS, RHSign).getReg(0);
5750
5751 Register DstDivReg, DstRemReg, TmpDivReg, TmpRemReg;
5752 switch (MI.getOpcode()) {
5753 default:
5754 llvm_unreachable("Unexpected opcode!");
5755 case AMDGPU::G_SDIV: {
5756 DstDivReg = MI.getOperand(0).getReg();
5757 TmpDivReg = MRI.createGenericVirtualRegister(Ty);
5758 break;
5759 }
5760 case AMDGPU::G_SREM: {
5761 DstRemReg = MI.getOperand(0).getReg();
5762 TmpRemReg = MRI.createGenericVirtualRegister(Ty);
5763 break;
5764 }
5765 case AMDGPU::G_SDIVREM: {
5766 DstDivReg = MI.getOperand(0).getReg();
5767 DstRemReg = MI.getOperand(1).getReg();
5768 TmpDivReg = MRI.createGenericVirtualRegister(Ty);
5769 TmpRemReg = MRI.createGenericVirtualRegister(Ty);
5770 break;
5771 }
5772 }
5773
5774 if (Ty == I32)
5775 legalizeUnsignedDIV_REM32Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);
5776 else
5777 legalizeUnsignedDIV_REM64Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);
5778
5779 if (DstDivReg) {
5780 auto Sign = B.buildXor(Ty, LHSign, RHSign).getReg(0);
5781 auto SignXor = B.buildXor(Ty, TmpDivReg, Sign).getReg(0);
5782 B.buildSub(DstDivReg, SignXor, Sign);
5783 }
5784
5785 if (DstRemReg) {
5786 auto Sign = LHSign.getReg(0); // Remainder sign is the same as LHS
5787 auto SignXor = B.buildXor(Ty, TmpRemReg, Sign).getReg(0);
5788 B.buildSub(DstRemReg, SignXor, Sign);
5789 }
5790
5791 MI.eraseFromParent();
5792 return true;
5793}
5794
5797 MachineIRBuilder &B) const {
5798 Register Res = MI.getOperand(0).getReg();
5799 Register LHS = MI.getOperand(1).getReg();
5800 Register RHS = MI.getOperand(2).getReg();
5801 uint16_t Flags = MI.getFlags();
5802 LLT ResTy = MRI.getType(Res);
5803
5804 bool AllowInaccurateRcp = MI.getFlag(MachineInstr::FmAfn);
5805
5806 if (const auto *CLHS = getConstantFPVRegVal(LHS, MRI)) {
5807 if (!AllowInaccurateRcp && ResTy != F16)
5808 return false;
5809
5810 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
5811 // the CI documentation has a worst case error of 1 ulp.
5812 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
5813 // use it as long as we aren't trying to use denormals.
5814 //
5815 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
5816
5817 // 1 / x -> RCP(x)
5818 if (CLHS->isOne()) {
5819 B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res)
5820 .addUse(RHS)
5821 .setMIFlags(Flags);
5822
5823 MI.eraseFromParent();
5824 return true;
5825 }
5826
5827 // -1 / x -> FNEG( RCP(x) )
5828 if (CLHS->isMinusOne()) {
5829 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
5830 .addUse(RHS)
5831 .setMIFlags(Flags);
5832 B.buildFNeg(Res, RCP, Flags);
5833
5834 MI.eraseFromParent();
5835 return true;
5836 }
5837 }
5838
5839 // For f16 require afn or arcp.
5840 // For f32 require afn.
5841 if (!AllowInaccurateRcp &&
5842 (ResTy != F16 || !MI.getFlag(MachineInstr::FmArcp)))
5843 return false;
5844
5845 // x / y -> x * (1.0 / y)
5846 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
5847 .addUse(RHS)
5848 .setMIFlags(Flags);
5849 B.buildFMul(Res, LHS, RCP, Flags);
5850
5851 MI.eraseFromParent();
5852 return true;
5853}
5854
5857 MachineIRBuilder &B) const {
5858 Register Res = MI.getOperand(0).getReg();
5859 Register X = MI.getOperand(1).getReg();
5860 Register Y = MI.getOperand(2).getReg();
5861 uint16_t Flags = MI.getFlags();
5862 LLT ResTy = MRI.getType(Res);
5863
5864 bool AllowInaccurateRcp = MI.getFlag(MachineInstr::FmAfn);
5865
5866 if (!AllowInaccurateRcp)
5867 return false;
5868
5869 const ConstantFP *CLHS = getConstantFPVRegVal(X, MRI);
5870 bool IsNegRcp = CLHS && CLHS->isMinusOne();
5871
5872 // Pull out the negation so it folds for free into the source modifiers.
5873 if (IsNegRcp)
5874 X = B.buildFConstant(ResTy, 1.0).getReg(0);
5875
5876 Register NegY = IsNegRcp ? Y : B.buildFNeg(ResTy, Y).getReg(0);
5877 auto One = B.buildFConstant(ResTy, 1.0);
5878
5879 auto R = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
5880 .addUse(Y)
5881 .setMIFlags(Flags);
5882 if (IsNegRcp)
5883 R = B.buildFNeg(ResTy, R);
5884
5885 auto Tmp0 = B.buildFMA(ResTy, NegY, R, One);
5886 R = B.buildFMA(ResTy, Tmp0, R, R);
5887
5888 auto Tmp1 = B.buildFMA(ResTy, NegY, R, One);
5889 R = B.buildFMA(ResTy, Tmp1, R, R);
5890
5891 // Skip the last 2 correction terms for reciprocal.
5892 if (IsNegRcp || (CLHS && CLHS->isOne())) {
5893 B.buildCopy(Res, R);
5894 MI.eraseFromParent();
5895 return true;
5896 }
5897
5898 auto Ret = B.buildFMul(ResTy, X, R);
5899 auto Tmp2 = B.buildFMA(ResTy, NegY, Ret, X);
5900
5901 B.buildFMA(Res, Tmp2, R, Ret);
5902 MI.eraseFromParent();
5903 return true;
5904}
5905
5908 MachineIRBuilder &B) const {
5910 return true;
5911
5912 Register Res = MI.getOperand(0).getReg();
5913 Register LHS = MI.getOperand(1).getReg();
5914 Register RHS = MI.getOperand(2).getReg();
5915
5916 uint16_t Flags = MI.getFlags();
5917
5918 LLT I32 = LLT::integer(32);
5919
5920 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
5921 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
5922 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
5923 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
5924 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
5925 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
5926 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
5927 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
5928 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
5929 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
5930 // q16.u = opx(V_CVT_F16_F32, q32.u);
5931 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
5932
5933 auto LHSExt = B.buildFPExt(F32, LHS, Flags);
5934 auto RHSExt = B.buildFPExt(F32, RHS, Flags);
5935 auto NegRHSExt = B.buildFNeg(F32, RHSExt);
5936 auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F32})
5937 .addUse(RHSExt.getReg(0))
5938 .setMIFlags(Flags);
5939 auto Quot = B.buildFMul(F32, LHSExt, Rcp, Flags);
5941 if (ST.hasMadMacF32Insts()) {
5942 Err = B.buildFMAD(F32, NegRHSExt, Quot, LHSExt, Flags);
5943 Quot = B.buildFMAD(F32, Err, Rcp, Quot, Flags);
5944 Err = B.buildFMAD(F32, NegRHSExt, Quot, LHSExt, Flags);
5945 } else {
5946 Err = B.buildFMA(F32, NegRHSExt, Quot, LHSExt, Flags);
5947 Quot = B.buildFMA(F32, Err, Rcp, Quot, Flags);
5948 Err = B.buildFMA(F32, NegRHSExt, Quot, LHSExt, Flags);
5949 }
5950 auto Tmp = B.buildFMul(F32, Err, Rcp, Flags);
5951 auto TmpInt = B.buildBitcast(I32, Tmp);
5952 auto MaskedInt = B.buildAnd(I32, TmpInt, B.buildConstant(I32, 0xff800000));
5953 auto Masked = B.buildBitcast(F32, MaskedInt);
5954 Quot = B.buildFAdd(F32, Masked, Quot, Flags);
5955 auto RDst = B.buildFPTrunc(F16, Quot, Flags);
5956 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
5957 .addUse(RDst.getReg(0))
5958 .addUse(RHS)
5959 .addUse(LHS)
5960 .setMIFlags(Flags);
5961
5962 MI.eraseFromParent();
5963 return true;
5964}
5965
5966static constexpr unsigned SPDenormModeBitField =
5968
5969// Enable or disable FP32 denorm mode. When 'Enable' is true, emit instructions
5970// to enable denorm mode. When 'Enable' is false, disable denorm mode.
5972 const GCNSubtarget &ST,
5974 // Set SP denorm mode to this value.
5975 unsigned SPDenormMode =
5976 Enable ? FP_DENORM_FLUSH_NONE : Mode.fpDenormModeSPValue();
5977
5978 if (ST.hasDenormModeInst()) {
5979 // Preserve default FP64FP16 denorm mode while updating FP32 mode.
5980 uint32_t DPDenormModeDefault = Mode.fpDenormModeDPValue();
5981
5982 uint32_t NewDenormModeValue = SPDenormMode | (DPDenormModeDefault << 2);
5983 B.buildInstr(AMDGPU::S_DENORM_MODE)
5984 .addImm(NewDenormModeValue);
5985
5986 } else {
5987 B.buildInstr(AMDGPU::S_SETREG_IMM32_B32)
5988 .addImm(SPDenormMode)
5989 .addImm(SPDenormModeBitField);
5990 }
5991}
5992
5995 MachineIRBuilder &B) const {
5997 return true;
5998
5999 Register Res = MI.getOperand(0).getReg();
6000 Register LHS = MI.getOperand(1).getReg();
6001 Register RHS = MI.getOperand(2).getReg();
6002 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
6003 SIModeRegisterDefaults Mode = MFI->getMode();
6004
6005 uint16_t Flags = MI.getFlags();
6006
6007 LLT S1 = LLT::scalar(1);
6008
6009 auto One = B.buildFConstant(F32, 1.0f);
6010
6011 auto DenominatorScaled =
6012 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F32, S1})
6013 .addUse(LHS)
6014 .addUse(RHS)
6015 .addImm(0)
6016 .setMIFlags(Flags);
6017 auto NumeratorScaled =
6018 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F32, S1})
6019 .addUse(LHS)
6020 .addUse(RHS)
6021 .addImm(1)
6022 .setMIFlags(Flags);
6023
6024 auto ApproxRcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F32})
6025 .addUse(DenominatorScaled.getReg(0))
6026 .setMIFlags(Flags);
6027 auto NegDivScale0 = B.buildFNeg(F32, DenominatorScaled, Flags);
6028
6029 const bool PreservesDenormals = Mode.FP32Denormals == DenormalMode::getIEEE();
6030 const bool HasDynamicDenormals =
6031 (Mode.FP32Denormals.Input == DenormalMode::Dynamic) ||
6032 (Mode.FP32Denormals.Output == DenormalMode::Dynamic);
6033
6034 Register SavedSPDenormMode;
6035 if (!PreservesDenormals) {
6036 if (HasDynamicDenormals) {
6037 SavedSPDenormMode = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6038 B.buildInstr(AMDGPU::S_GETREG_B32)
6039 .addDef(SavedSPDenormMode)
6040 .addImm(SPDenormModeBitField);
6041 }
6042 toggleSPDenormMode(true, B, ST, Mode);
6043 }
6044
6045 auto Fma0 = B.buildFMA(F32, NegDivScale0, ApproxRcp, One, Flags);
6046 auto Fma1 = B.buildFMA(F32, Fma0, ApproxRcp, ApproxRcp, Flags);
6047 auto Mul = B.buildFMul(F32, NumeratorScaled, Fma1, Flags);
6048 auto Fma2 = B.buildFMA(F32, NegDivScale0, Mul, NumeratorScaled, Flags);
6049 auto Fma3 = B.buildFMA(F32, Fma2, Fma1, Mul, Flags);
6050 auto Fma4 = B.buildFMA(F32, NegDivScale0, Fma3, NumeratorScaled, Flags);
6051
6052 if (!PreservesDenormals) {
6053 if (HasDynamicDenormals) {
6054 assert(SavedSPDenormMode);
6055 B.buildInstr(AMDGPU::S_SETREG_B32)
6056 .addReg(SavedSPDenormMode)
6057 .addImm(SPDenormModeBitField);
6058 } else
6059 toggleSPDenormMode(false, B, ST, Mode);
6060 }
6061
6062 auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {F32})
6063 .addUse(Fma4.getReg(0))
6064 .addUse(Fma1.getReg(0))
6065 .addUse(Fma3.getReg(0))
6066 .addUse(NumeratorScaled.getReg(1))
6067 .setMIFlags(Flags);
6068
6069 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
6070 .addUse(Fmas.getReg(0))
6071 .addUse(RHS)
6072 .addUse(LHS)
6073 .setMIFlags(Flags);
6074
6075 MI.eraseFromParent();
6076 return true;
6077}
6078
6081 MachineIRBuilder &B) const {
6083 return true;
6084
6085 Register Res = MI.getOperand(0).getReg();
6086 Register LHS = MI.getOperand(1).getReg();
6087 Register RHS = MI.getOperand(2).getReg();
6088
6089 uint16_t Flags = MI.getFlags();
6090
6091 LLT S1 = LLT::scalar(1);
6092
6093 auto One = B.buildFConstant(F64, 1.0);
6094
6095 auto DivScale0 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F64, S1})
6096 .addUse(LHS)
6097 .addUse(RHS)
6098 .addImm(0)
6099 .setMIFlags(Flags);
6100
6101 auto NegDivScale0 = B.buildFNeg(F64, DivScale0.getReg(0), Flags);
6102
6103 auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F64})
6104 .addUse(DivScale0.getReg(0))
6105 .setMIFlags(Flags);
6106
6107 auto Fma0 = B.buildFMA(F64, NegDivScale0, Rcp, One, Flags);
6108 auto Fma1 = B.buildFMA(F64, Rcp, Fma0, Rcp, Flags);
6109 auto Fma2 = B.buildFMA(F64, NegDivScale0, Fma1, One, Flags);
6110
6111 auto DivScale1 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F64, S1})
6112 .addUse(LHS)
6113 .addUse(RHS)
6114 .addImm(1)
6115 .setMIFlags(Flags);
6116
6117 auto Fma3 = B.buildFMA(F64, Fma1, Fma2, Fma1, Flags);
6118 auto Mul = B.buildFMul(F64, DivScale1.getReg(0), Fma3, Flags);
6119 auto Fma4 = B.buildFMA(F64, NegDivScale0, Mul, DivScale1.getReg(0), Flags);
6120
6121 Register Scale;
6122 if (!ST.hasUsableDivScaleConditionOutput()) {
6123 // Workaround a hardware bug on SI where the condition output from div_scale
6124 // is not usable.
6125
6126 LLT I32 = LLT::integer(32);
6127 LLT I64 = LLT::integer(64);
6128
6129 auto NumUnmerge = B.buildUnmerge(I32, B.buildBitcast(I64, LHS));
6130 auto DenUnmerge = B.buildUnmerge(I32, B.buildBitcast(I64, RHS));
6131 auto Scale0Unmerge = B.buildUnmerge(I32, B.buildBitcast(I64, DivScale0));
6132 auto Scale1Unmerge = B.buildUnmerge(I32, B.buildBitcast(I64, DivScale1));
6133
6134 auto CmpNum = B.buildICmp(ICmpInst::ICMP_EQ, S1, NumUnmerge.getReg(1),
6135 Scale1Unmerge.getReg(1));
6136 auto CmpDen = B.buildICmp(ICmpInst::ICMP_EQ, S1, DenUnmerge.getReg(1),
6137 Scale0Unmerge.getReg(1));
6138 Scale = B.buildXor(S1, CmpNum, CmpDen).getReg(0);
6139 } else {
6140 Scale = DivScale1.getReg(1);
6141 }
6142
6143 auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {F64})
6144 .addUse(Fma4.getReg(0))
6145 .addUse(Fma3.getReg(0))
6146 .addUse(Mul.getReg(0))
6147 .addUse(Scale)
6148 .setMIFlags(Flags);
6149
6150 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, ArrayRef(Res))
6151 .addUse(Fmas.getReg(0))
6152 .addUse(RHS)
6153 .addUse(LHS)
6154 .setMIFlags(Flags);
6155
6156 MI.eraseFromParent();
6157 return true;
6158}
6159
6162 MachineIRBuilder &B) const {
6163 Register Res0 = MI.getOperand(0).getReg();
6164 Register Res1 = MI.getOperand(1).getReg();
6165 Register Val = MI.getOperand(2).getReg();
6166 uint16_t Flags = MI.getFlags();
6167
6168 LLT Ty = MRI.getType(Res0);
6169 LLT InstrExpTy = Ty == F16 ? LLT::integer(16) : LLT::integer(32);
6170
6171 auto Mant = B.buildIntrinsic(Intrinsic::amdgcn_frexp_mant, {Ty})
6172 .addUse(Val)
6173 .setMIFlags(Flags);
6174 auto Exp = B.buildIntrinsic(Intrinsic::amdgcn_frexp_exp, {InstrExpTy})
6175 .addUse(Val)
6176 .setMIFlags(Flags);
6177
6178 if (ST.hasFractBug()) {
6179 auto Fabs = B.buildFAbs(Ty, Val);
6180 auto Inf = B.buildFConstant(Ty, APFloat::getInf(getFltSemanticForLLT(Ty)));
6181 auto IsFinite =
6182 B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags);
6183 auto Zero = B.buildConstant(InstrExpTy, 0);
6184 Exp = B.buildSelect(InstrExpTy, IsFinite, Exp, Zero);
6185 Mant = B.buildSelect(Ty, IsFinite, Mant, Val);
6186 }
6187
6188 B.buildCopy(Res0, Mant);
6189 B.buildSExtOrTrunc(Res1, Exp);
6190
6191 MI.eraseFromParent();
6192 return true;
6193}
6194
6197 MachineIRBuilder &B) const {
6198 Register Res = MI.getOperand(0).getReg();
6199 Register LHS = MI.getOperand(2).getReg();
6200 Register RHS = MI.getOperand(3).getReg();
6201 uint16_t Flags = MI.getFlags();
6202
6203 LLT S1 = LLT::scalar(1);
6204
6205 auto Abs = B.buildFAbs(F32, RHS, Flags);
6206
6207 auto C0 = B.buildFConstant(F32, 0x1p+96f);
6208 auto C1 = B.buildFConstant(F32, 0x1p-32f);
6209 auto C2 = B.buildFConstant(F32, 1.0f);
6210
6211 auto CmpRes = B.buildFCmp(CmpInst::FCMP_OGT, S1, Abs, C0, Flags);
6212 auto Sel = B.buildSelect(F32, CmpRes, C1, C2, Flags);
6213
6214 auto Mul0 = B.buildFMul(F32, RHS, Sel, Flags);
6215
6216 auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F32})
6217 .addUse(Mul0.getReg(0))
6218 .setMIFlags(Flags);
6219
6220 auto Mul1 = B.buildFMul(F32, LHS, RCP, Flags);
6221
6222 B.buildFMul(Res, Sel, Mul1, Flags);
6223
6224 MI.eraseFromParent();
6225 return true;
6226}
6227
6230 MachineIRBuilder &B) const {
6231 // Bypass the correct expansion a standard promotion through G_FSQRT would
6232 // get. The f32 op is accurate enough for the f16 cas.
6233 unsigned Flags = MI.getFlags();
6234 assert(!ST.has16BitInsts());
6235 auto Ext = B.buildFPExt(F32, MI.getOperand(1), Flags);
6236 auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_sqrt, {F32})
6237 .addUse(Ext.getReg(0))
6238 .setMIFlags(Flags);
6239 B.buildFPTrunc(MI.getOperand(0), Log2, Flags);
6240 MI.eraseFromParent();
6241 return true;
6242}
6243
6246 MachineIRBuilder &B) const {
6247 MachineFunction &MF = B.getMF();
6248 Register Dst = MI.getOperand(0).getReg();
6249 Register X = MI.getOperand(1).getReg();
6250 const unsigned Flags = MI.getFlags();
6251 const LLT I1 = LLT::integer(1);
6252 const LLT I32 = LLT::integer(32);
6253
6254 if (allowApproxFunc(MF, Flags)) {
6255 B.buildIntrinsic(Intrinsic::amdgcn_sqrt, ArrayRef<Register>({Dst}))
6256 .addUse(X)
6257 .setMIFlags(Flags);
6258 MI.eraseFromParent();
6259 return true;
6260 }
6261
6262 auto ScaleThreshold = B.buildFConstant(F32, 0x1.0p-96f);
6263 auto NeedScale = B.buildFCmp(CmpInst::FCMP_OGT, I1, ScaleThreshold, X, Flags);
6264 auto ScaleUpFactor = B.buildFConstant(F32, 0x1.0p+32f);
6265 auto ScaledX = B.buildFMul(F32, X, ScaleUpFactor, Flags);
6266 auto SqrtX = B.buildSelect(F32, NeedScale, ScaledX, X, Flags);
6267
6268 Register SqrtS = MRI.createGenericVirtualRegister(F32);
6269 if (needsDenormHandlingF32(MF, X, Flags)) {
6270 B.buildIntrinsic(Intrinsic::amdgcn_sqrt, ArrayRef<Register>({SqrtS}))
6271 .addUse(SqrtX.getReg(0))
6272 .setMIFlags(Flags);
6273
6274 auto SqrtSInt = B.buildBitcast(I32, SqrtS);
6275 auto NegOne = B.buildConstant(I32, -1);
6276 auto SqrtSNextDown = B.buildBitcast(F32, B.buildAdd(I32, SqrtSInt, NegOne));
6277
6278 auto NegSqrtSNextDown = B.buildFNeg(F32, SqrtSNextDown, Flags);
6279 auto SqrtVP = B.buildFMA(F32, NegSqrtSNextDown, SqrtS, SqrtX, Flags);
6280
6281 auto PosOne = B.buildConstant(I32, 1);
6282 auto SqrtSNextUp = B.buildBitcast(F32, B.buildAdd(I32, SqrtSInt, PosOne));
6283
6284 auto NegSqrtSNextUp = B.buildFNeg(F32, SqrtSNextUp, Flags);
6285 auto SqrtVS = B.buildFMA(F32, NegSqrtSNextUp, SqrtS, SqrtX, Flags);
6286
6287 auto Zero = B.buildFConstant(F32, 0.0f);
6288 auto SqrtVPLE0 = B.buildFCmp(CmpInst::FCMP_OLE, I1, SqrtVP, Zero, Flags);
6289
6290 SqrtS =
6291 B.buildSelect(F32, SqrtVPLE0, SqrtSNextDown, SqrtS, Flags).getReg(0);
6292
6293 auto SqrtVPVSGT0 = B.buildFCmp(CmpInst::FCMP_OGT, I1, SqrtVS, Zero, Flags);
6294 SqrtS =
6295 B.buildSelect(F32, SqrtVPVSGT0, SqrtSNextUp, SqrtS, Flags).getReg(0);
6296 } else {
6297 auto SqrtR =
6298 B.buildIntrinsic(Intrinsic::amdgcn_rsq, {F32}).addReg(SqrtX.getReg(0));
6299 B.buildFMul(SqrtS, SqrtX, SqrtR, Flags);
6300
6301 auto Half = B.buildFConstant(F32, 0.5f);
6302 auto SqrtH = B.buildFMul(F32, SqrtR, Half, Flags);
6303 auto NegSqrtH = B.buildFNeg(F32, SqrtH, Flags);
6304 auto SqrtE = B.buildFMA(F32, NegSqrtH, SqrtS, Half, Flags);
6305 SqrtH = B.buildFMA(F32, SqrtH, SqrtE, SqrtH, Flags);
6306 SqrtS = B.buildFMA(F32, SqrtS, SqrtE, SqrtS, Flags).getReg(0);
6307 auto NegSqrtS = B.buildFNeg(F32, SqrtS, Flags);
6308 auto SqrtD = B.buildFMA(F32, NegSqrtS, SqrtS, SqrtX, Flags);
6309 SqrtS = B.buildFMA(F32, SqrtD, SqrtH, SqrtS, Flags).getReg(0);
6310 }
6311
6312 auto ScaleDownFactor = B.buildFConstant(F32, 0x1.0p-16f);
6313
6314 auto ScaledDown = B.buildFMul(F32, SqrtS, ScaleDownFactor, Flags);
6315
6316 SqrtS = B.buildSelect(F32, NeedScale, ScaledDown, SqrtS, Flags).getReg(0);
6317
6318 auto IsZeroOrInf = B.buildIsFPClass(I1, SqrtX, fcZero | fcPosInf);
6319 B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtS, Flags);
6320
6321 MI.eraseFromParent();
6322 return true;
6323}
6324
6327 MachineIRBuilder &B) const {
6328 // For double type, the SQRT and RSQ instructions don't have required
6329 // precision, we apply Goldschmidt's algorithm to improve the result:
6330 //
6331 // y0 = rsq(x)
6332 // g0 = x * y0
6333 // h0 = 0.5 * y0
6334 //
6335 // r0 = 0.5 - h0 * g0
6336 // g1 = g0 * r0 + g0
6337 // h1 = h0 * r0 + h0
6338 //
6339 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
6340 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
6341 // h2 = h1 * r1 + h1
6342 //
6343 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
6344 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
6345 //
6346 // sqrt(x) = g3
6347
6348 const LLT I1 = LLT::integer(1);
6349 const LLT I32 = LLT::integer(32);
6350
6351 Register Dst = MI.getOperand(0).getReg();
6352 assert(MRI.getType(Dst) == F64 && "only expect to lower f64 sqrt");
6353
6354 Register X = MI.getOperand(1).getReg();
6355 unsigned Flags = MI.getFlags();
6356
6357 Register SqrtX = X;
6358 Register Scaling, ZeroInt;
6359 if (!MI.getFlag(MachineInstr::FmAfn)) {
6360 auto ScaleConstant = B.buildFConstant(F64, 0x1.0p-767);
6361
6362 ZeroInt = B.buildConstant(I32, 0).getReg(0);
6363 Scaling = B.buildFCmp(FCmpInst::FCMP_OLT, I1, X, ScaleConstant).getReg(0);
6364
6365 // Scale up input if it is too small.
6366 auto ScaleUpFactor = B.buildConstant(I32, 256);
6367 auto ScaleUp = B.buildSelect(I32, Scaling, ScaleUpFactor, ZeroInt);
6368 SqrtX = B.buildFLdexp(F64, X, ScaleUp, Flags).getReg(0);
6369 }
6370
6371 auto SqrtY = B.buildIntrinsic(Intrinsic::amdgcn_rsq, {F64}).addReg(SqrtX);
6372
6373 auto Half = B.buildFConstant(F64, 0.5);
6374 auto SqrtH0 = B.buildFMul(F64, SqrtY, Half);
6375 auto SqrtS0 = B.buildFMul(F64, SqrtX, SqrtY);
6376
6377 auto NegSqrtH0 = B.buildFNeg(F64, SqrtH0);
6378 auto SqrtR0 = B.buildFMA(F64, NegSqrtH0, SqrtS0, Half);
6379
6380 auto SqrtS1 = B.buildFMA(F64, SqrtS0, SqrtR0, SqrtS0);
6381 auto SqrtH1 = B.buildFMA(F64, SqrtH0, SqrtR0, SqrtH0);
6382
6383 auto NegSqrtS1 = B.buildFNeg(F64, SqrtS1);
6384 auto SqrtD0 = B.buildFMA(F64, NegSqrtS1, SqrtS1, SqrtX);
6385
6386 auto SqrtS2 = B.buildFMA(F64, SqrtD0, SqrtH1, SqrtS1);
6387
6388 Register SqrtRet = SqrtS2.getReg(0);
6389 if (!MI.getFlag(MachineInstr::FmAfn)) {
6390 auto NegSqrtS2 = B.buildFNeg(F64, SqrtS2);
6391 auto SqrtD1 = B.buildFMA(F64, NegSqrtS2, SqrtS2, SqrtX);
6392 auto SqrtD2 = B.buildFMA(F64, SqrtD1, SqrtH1, SqrtS2);
6393
6394 // Scale down the result.
6395 auto ScaleDownFactor = B.buildConstant(I32, -128);
6396 auto ScaleDown = B.buildSelect(I32, Scaling, ScaleDownFactor, ZeroInt);
6397 SqrtRet = B.buildFLdexp(F64, SqrtD2, ScaleDown, Flags).getReg(0);
6398 }
6399
6400 Register IsZeroOrInf;
6401 if (MI.getFlag(MachineInstr::FmNoInfs)) {
6402 auto ZeroFP = B.buildFConstant(F64, 0.0);
6403 IsZeroOrInf = B.buildFCmp(FCmpInst::FCMP_OEQ, I1, SqrtX, ZeroFP).getReg(0);
6404 } else {
6405 IsZeroOrInf = B.buildIsFPClass(I1, SqrtX, fcZero | fcPosInf).getReg(0);
6406 }
6407
6408 // TODO: Check for DAZ and expand to subnormals
6409
6410 // If x is +INF, +0, or -0, use its original value
6411 B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtRet, Flags);
6412
6413 MI.eraseFromParent();
6414 return true;
6415}
6416
6419 MachineIRBuilder &B) const {
6420 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
6421 if (Ty == F32)
6422 return legalizeFSQRTF32(MI, MRI, B);
6423 if (Ty == F64)
6424 return legalizeFSQRTF64(MI, MRI, B);
6425 if (Ty == F16)
6426 return legalizeFSQRTF16(MI, MRI, B);
6427 return false;
6428}
6429
6430// Expand llvm.amdgcn.rsq.clamp on targets that don't support the instruction.
6431// FIXME: Why do we handle this one but not other removed instructions?
6432//
6433// Reciprocal square root. The clamp prevents infinite results, clamping
6434// infinities to max_float. D.f = 1.0 / sqrt(S0.f), result clamped to
6435// +-max_float.
6438 MachineIRBuilder &B) const {
6439 if (ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
6440 return true;
6441
6442 Register Dst = MI.getOperand(0).getReg();
6443 Register Src = MI.getOperand(2).getReg();
6444 auto Flags = MI.getFlags();
6445
6446 LLT Ty = MRI.getType(Dst);
6447
6448 const fltSemantics *FltSemantics;
6449 if (Ty == F32)
6450 FltSemantics = &APFloat::IEEEsingle();
6451 else if (Ty == F64)
6452 FltSemantics = &APFloat::IEEEdouble();
6453 else
6454 return false;
6455
6456 auto Rsq = B.buildIntrinsic(Intrinsic::amdgcn_rsq, {Ty})
6457 .addUse(Src)
6458 .setMIFlags(Flags);
6459
6460 // We don't need to concern ourselves with the snan handling difference, since
6461 // the rsq quieted (or not) so use the one which will directly select.
6462 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
6463 const bool UseIEEE = MFI->getMode().IEEE;
6464
6465 auto MaxFlt = B.buildFConstant(Ty, APFloat::getLargest(*FltSemantics));
6466 auto ClampMax = UseIEEE && !ST.hasIEEEMinimumMaximumInsts()
6467 ? B.buildFMinNumIEEE(Ty, Rsq, MaxFlt, Flags)
6468 : B.buildFMinNum(Ty, Rsq, MaxFlt, Flags);
6469
6470 auto MinFlt = B.buildFConstant(Ty, APFloat::getLargest(*FltSemantics, true));
6471
6472 if (UseIEEE && !ST.hasIEEEMinimumMaximumInsts())
6473 B.buildFMaxNumIEEE(Dst, ClampMax, MinFlt, Flags);
6474 else
6475 B.buildFMaxNum(Dst, ClampMax, MinFlt, Flags);
6476 MI.eraseFromParent();
6477 return true;
6478}
6479
6480// TODO: Fix pointer type handling
6483 Intrinsic::ID IID) const {
6484
6485 MachineIRBuilder &B = Helper.MIRBuilder;
6486 MachineRegisterInfo &MRI = *B.getMRI();
6487
6488 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
6489 IID == Intrinsic::amdgcn_permlanex16;
6490 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
6491 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
6492 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
6493 IID == Intrinsic::amdgcn_permlane_up ||
6494 IID == Intrinsic::amdgcn_permlane_down ||
6495 IID == Intrinsic::amdgcn_permlane_xor;
6496
6497 auto createLaneOp = [&IID, &B, &MI](Register Src0, Register Src1,
6498 Register Src2, LLT VT) -> Register {
6499 auto LaneOp = B.buildIntrinsic(IID, {VT}).addUse(Src0);
6500 switch (IID) {
6501 case Intrinsic::amdgcn_readfirstlane:
6502 case Intrinsic::amdgcn_permlane64:
6503 return LaneOp.getReg(0);
6504 case Intrinsic::amdgcn_readlane:
6505 case Intrinsic::amdgcn_set_inactive:
6506 case Intrinsic::amdgcn_set_inactive_chain_arg:
6507 return LaneOp.addUse(Src1).getReg(0);
6508 case Intrinsic::amdgcn_writelane:
6509 case Intrinsic::amdgcn_permlane_bcast:
6510 case Intrinsic::amdgcn_permlane_up:
6511 case Intrinsic::amdgcn_permlane_down:
6512 case Intrinsic::amdgcn_permlane_xor:
6513 return LaneOp.addUse(Src1).addUse(Src2).getReg(0);
6514 case Intrinsic::amdgcn_permlane16:
6515 case Intrinsic::amdgcn_permlanex16: {
6516 Register Src3 = MI.getOperand(5).getReg();
6517 int64_t Src4 = MI.getOperand(6).getImm();
6518 int64_t Src5 = MI.getOperand(7).getImm();
6519 return LaneOp.addUse(Src1)
6520 .addUse(Src2)
6521 .addUse(Src3)
6522 .addImm(Src4)
6523 .addImm(Src5)
6524 .getReg(0);
6525 }
6526 case Intrinsic::amdgcn_mov_dpp8:
6527 return LaneOp.addImm(MI.getOperand(3).getImm()).getReg(0);
6528 case Intrinsic::amdgcn_update_dpp:
6529 return LaneOp.addUse(Src1)
6530 .addImm(MI.getOperand(4).getImm())
6531 .addImm(MI.getOperand(5).getImm())
6532 .addImm(MI.getOperand(6).getImm())
6533 .addImm(MI.getOperand(7).getImm())
6534 .getReg(0);
6535 default:
6536 llvm_unreachable("unhandled lane op");
6537 }
6538 };
6539
6540 Register DstReg = MI.getOperand(0).getReg();
6541 Register Src0 = MI.getOperand(2).getReg();
6542 Register Src1, Src2;
6543 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
6544 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
6545 IsPermlaneShuffle) {
6546 Src1 = MI.getOperand(3).getReg();
6547 if (IID == Intrinsic::amdgcn_writelane || IsPermLane16 ||
6548 IsPermlaneShuffle) {
6549 Src2 = MI.getOperand(4).getReg();
6550 }
6551 }
6552
6553 LLT Ty = MRI.getType(DstReg);
6554 unsigned Size = Ty.getSizeInBits();
6555
6556 unsigned SplitSize = 32;
6557 if (IID == Intrinsic::amdgcn_update_dpp && (Size % 64 == 0) &&
6558 ST.hasDPALU_DPP() &&
6559 AMDGPU::isLegalDPALU_DPPControl(ST, MI.getOperand(4).getImm()))
6560 SplitSize = 64;
6561
6562 if (Size == SplitSize) {
6563 // Already legal
6564 return true;
6565 }
6566
6567 const LLT I32 = LLT::integer(32);
6568
6569 bool IsFloat = Ty.getScalarType().isFloat();
6570
6571 LLT IntTy = IsFloat ? LLT::integer(Size) : Ty;
6572 if (IsFloat) {
6573 Src0 = B.buildBitcast(IntTy, Src0).getReg(0);
6574 if (Src1 && MRI.getType(Src1).getScalarType().isFloat())
6575 Src1 = B.buildBitcast(IntTy, Src1).getReg(0);
6576 if (Src2 && MRI.getType(Src2).getScalarType().isFloat())
6577 Src2 = B.buildBitcast(IntTy, Src2).getReg(0);
6578 }
6579
6580 if (Size < 32) {
6581 Src0 = B.buildAnyExt(I32, Src0).getReg(0);
6582
6583 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6584 Src1 = B.buildAnyExt(I32, Src1).getReg(0);
6585
6586 if (IID == Intrinsic::amdgcn_writelane)
6587 Src2 = B.buildAnyExt(I32, Src2).getReg(0);
6588
6589 Register LaneOpDst = createLaneOp(Src0, Src1, Src2, I32);
6590 if (IsFloat)
6591 B.buildBitcast(DstReg, B.buildTrunc(IntTy, LaneOpDst));
6592 else
6593 B.buildTrunc(DstReg, LaneOpDst);
6594 MI.eraseFromParent();
6595 return true;
6596 }
6597
6598 if (Size % SplitSize != 0)
6599 return false;
6600
6601 LLT PartialResTy = LLT::integer(SplitSize);
6602 bool NeedsBitcast = false;
6603 if (IntTy.isVector()) {
6604 LLT EltTy = IntTy.getElementType();
6605 unsigned EltSize = EltTy.getSizeInBits();
6606 if (EltSize == SplitSize) {
6607 PartialResTy = EltTy;
6608 } else if (EltSize == 16 || EltSize == 32) {
6609 unsigned NElem = SplitSize / EltSize;
6610 PartialResTy = IntTy.changeElementCount(ElementCount::getFixed(NElem));
6611 } else {
6612 NeedsBitcast = true;
6613 }
6614 }
6615
6616 SmallVector<Register, 4> PartialRes;
6617 unsigned NumParts = Size / SplitSize;
6618 MachineInstrBuilder Src0Parts = B.buildUnmerge(PartialResTy, Src0);
6619 MachineInstrBuilder Src1Parts, Src2Parts;
6620
6621 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6622 Src1Parts = B.buildUnmerge(PartialResTy, Src1);
6623
6624 if (IID == Intrinsic::amdgcn_writelane)
6625 Src2Parts = B.buildUnmerge(PartialResTy, Src2);
6626
6627 for (unsigned i = 0; i < NumParts; ++i) {
6628 Src0 = Src0Parts.getReg(i);
6629
6630 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6631 Src1 = Src1Parts.getReg(i);
6632
6633 if (IID == Intrinsic::amdgcn_writelane)
6634 Src2 = Src2Parts.getReg(i);
6635
6636 PartialRes.push_back(createLaneOp(Src0, Src1, Src2, PartialResTy));
6637 }
6638
6639 if (NeedsBitcast || IsFloat)
6640 B.buildBitcast(
6641 DstReg,
6642 B.buildMergeLikeInstr(LLT::integer(IntTy.getSizeInBits()), PartialRes));
6643 else
6644 B.buildMergeLikeInstr(DstReg, PartialRes);
6645
6646 MI.eraseFromParent();
6647 return true;
6648}
6649
6652 MachineIRBuilder &B) const {
6653 uint64_t Offset =
6654 ST.getTargetLowering()->getImplicitParameterOffset(
6656 LLT DstTy = MRI.getType(DstReg);
6657 LLT IdxTy = LLT::integer(DstTy.getSizeInBits());
6658
6659 Register KernargPtrReg = MRI.createGenericVirtualRegister(DstTy);
6660 if (!loadInputValue(KernargPtrReg, B,
6662 return false;
6663
6664 B.buildObjectPtrOffset(DstReg, KernargPtrReg,
6665 B.buildConstant(IdxTy, Offset).getReg(0));
6666 return true;
6667}
6668
6669/// To create a buffer resource from a 64-bit pointer, mask off the upper 32
6670/// bits of the pointer and replace them with the stride argument, then
6671/// merge_values everything together. In the common case of a raw buffer (the
6672/// stride component is 0), we can just AND off the upper half.
6675 Register Result = MI.getOperand(0).getReg();
6676 Register Pointer = MI.getOperand(2).getReg();
6677 Register Stride = MI.getOperand(3).getReg();
6678 Register NumRecords = MI.getOperand(4).getReg();
6679 Register Flags = MI.getOperand(5).getReg();
6680
6681 LLT I32 = LLT::integer(32);
6682 LLT I64 = LLT::integer(64);
6683
6684 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
6685
6686 auto ExtStride = B.buildAnyExt(I32, Stride);
6687
6688 if (ST.getBufferResourceNumRecordsWidth() == 45) {
6689 NumRecords = B.buildZExtOrTrunc(I64, NumRecords).getReg(0);
6690 NumRecords =
6691 B.buildAnd(I64, NumRecords, B.buildConstant(I64, (1ULL << 45) - 1))
6692 .getReg(0);
6693 Register Zero = B.buildConstant(I32, 0).getReg(0);
6694 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
6695 // num_records.
6696 LLT PtrIntTy = LLT::integer(MRI.getType(Pointer).getSizeInBits());
6697 auto PointerInt = B.buildPtrToInt(PtrIntTy, Pointer);
6698 auto ExtPointer = B.buildAnyExtOrTrunc(I64, PointerInt);
6699 auto NumRecordsLHS = B.buildShl(I64, NumRecords, B.buildConstant(I32, 57));
6700 Register LowHalf = B.buildOr(I64, ExtPointer, NumRecordsLHS).getReg(0);
6701
6702 // Build the higher 64-bit value, which has the higher 38-bit num_records,
6703 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
6704 auto NumRecordsRHS = B.buildLShr(I64, NumRecords, B.buildConstant(I32, 7));
6705 auto ShiftedStride = B.buildShl(I32, ExtStride, B.buildConstant(I32, 12));
6706 auto ExtShiftedStride =
6707 B.buildMergeValues(I64, {Zero, ShiftedStride.getReg(0)});
6708 auto ShiftedFlags = B.buildShl(I32, Flags, B.buildConstant(I32, 28));
6709 auto ExtShiftedFlags =
6710 B.buildMergeValues(I64, {Zero, ShiftedFlags.getReg(0)});
6711 auto CombinedFields = B.buildOr(I64, NumRecordsRHS, ExtShiftedStride);
6712 Register HighHalf =
6713 B.buildOr(I64, CombinedFields, ExtShiftedFlags).getReg(0);
6714 B.buildMergeValues(Result, {LowHalf, HighHalf});
6715 } else {
6716 NumRecords = B.buildZExtOrTrunc(I32, NumRecords).getReg(0);
6717 auto Unmerge = B.buildUnmerge(I32, Pointer);
6718 auto LowHalf = Unmerge.getReg(0);
6719 auto HighHalf = Unmerge.getReg(1);
6720
6721 auto AndMask = B.buildConstant(I32, 0x0000ffff);
6722 auto Masked = B.buildAnd(I32, HighHalf, AndMask);
6723 auto ShiftConst = B.buildConstant(I32, 16);
6724 auto ShiftedStride = B.buildShl(I32, ExtStride, ShiftConst);
6725 auto NewHighHalf = B.buildOr(I32, Masked, ShiftedStride);
6726 Register NewHighHalfReg = NewHighHalf.getReg(0);
6727 B.buildMergeValues(Result, {LowHalf, NewHighHalfReg, NumRecords, Flags});
6728 }
6729
6730 MI.eraseFromParent();
6731 return true;
6732}
6733
6736 MachineIRBuilder &B) const {
6737 const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
6738 if (!MFI->isEntryFunction()) {
6741 }
6742
6743 Register DstReg = MI.getOperand(0).getReg();
6744 if (!getImplicitArgPtr(DstReg, MRI, B))
6745 return false;
6746
6747 MI.eraseFromParent();
6748 return true;
6749}
6750
6754 unsigned AddrSpace) const {
6755 const LLT I32 = LLT::integer(32);
6756 auto Unmerge = B.buildUnmerge(I32, MI.getOperand(2).getReg());
6757 Register Hi32 = Unmerge.getReg(1);
6758
6759 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS &&
6760 ST.hasGloballyAddressableScratch()) {
6761 Register FlatScratchBaseHi =
6762 B.buildInstr(AMDGPU::S_MOV_B32, {I32},
6763 {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)})
6764 .getReg(0);
6765 MRI.setRegClass(FlatScratchBaseHi, &AMDGPU::SReg_32RegClass);
6766 // Test bits 63..58 against the aperture address.
6767 Register XOR = B.buildXor(I32, Hi32, FlatScratchBaseHi).getReg(0);
6768 B.buildICmp(ICmpInst::ICMP_ULT, MI.getOperand(0), XOR,
6769 B.buildConstant(I32, 1u << 26));
6770 } else {
6771 Register ApertureReg = getSegmentAperture(AddrSpace, MRI, B);
6772 B.buildICmp(ICmpInst::ICMP_EQ, MI.getOperand(0), Hi32, ApertureReg);
6773 }
6774 MI.eraseFromParent();
6775 return true;
6776}
6777
6778// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
6779// offset (the offset that is included in bounds checking and swizzling, to be
6780// split between the instruction's voffset and immoffset fields) and soffset
6781// (the offset that is excluded from bounds checking and swizzling, to go in
6782// the instruction's soffset field). This function takes the first kind of
6783// offset and figures out how to split it between voffset and immoffset.
6784std::pair<Register, unsigned>
6786 Register OrigOffset) const {
6787 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST);
6788 Register BaseReg;
6789 unsigned ImmOffset;
6790 const LLT I32 = LLT::integer(32);
6791 MachineRegisterInfo &MRI = *B.getMRI();
6792
6793 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
6794 // being added, so we can only safely match a 32-bit addition with no unsigned
6795 // overflow.
6796 bool CheckNUW = ST.hasGFX1250Insts();
6797 std::tie(BaseReg, ImmOffset) = AMDGPU::getBaseWithConstantOffset(
6798 MRI, OrigOffset, /*KnownBits=*/nullptr, CheckNUW);
6799
6800 // If BaseReg is a pointer, convert it to int.
6801 if (MRI.getType(BaseReg).isPointer())
6802 BaseReg = B.buildPtrToInt(MRI.getType(OrigOffset), BaseReg).getReg(0);
6803
6804 // If the immediate value is too big for the immoffset field, put only bits
6805 // that would normally fit in the immoffset field. The remaining value that
6806 // is copied/added for the voffset field is a large power of 2, and it
6807 // stands more chance of being CSEd with the copy/add for another similar
6808 // load/store.
6809 // However, do not do that rounding down if that is a negative
6810 // number, as it appears to be illegal to have a negative offset in the
6811 // vgpr, even if adding the immediate offset makes it positive.
6812 unsigned Overflow = ImmOffset & ~MaxImm;
6813 ImmOffset -= Overflow;
6814 if ((int32_t)Overflow < 0) {
6815 Overflow += ImmOffset;
6816 ImmOffset = 0;
6817 }
6818
6819 if (Overflow != 0) {
6820 if (!BaseReg) {
6821 BaseReg = B.buildConstant(I32, Overflow).getReg(0);
6822 } else {
6823 auto OverflowVal = B.buildConstant(I32, Overflow);
6824 BaseReg = B.buildAdd(I32, BaseReg, OverflowVal).getReg(0);
6825 }
6826 }
6827
6828 if (!BaseReg)
6829 BaseReg = B.buildConstant(I32, 0).getReg(0);
6830
6831 return std::pair(BaseReg, ImmOffset);
6832}
6833
6834/// Handle register layout difference for f16 images for some subtargets.
6837 Register Reg,
6838 bool ImageStore) const {
6839 const LLT I16 = LLT::integer(16);
6840 const LLT I32 = LLT::integer(32);
6841 LLT StoreVT = MRI.getType(Reg);
6842 assert(StoreVT.isVector() && StoreVT.getElementType().getSizeInBits() == 16);
6843
6844 LLT I16Vec = StoreVT.changeElementType(I16);
6845 Register RegI16 =
6846 StoreVT == I16Vec ? Reg : B.buildBitcast(I16Vec, Reg).getReg(0);
6847
6848 if (ST.hasUnpackedD16VMem()) {
6849 auto Unmerge = B.buildUnmerge(I16, RegI16);
6850
6851 SmallVector<Register, 4> WideRegs;
6852 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
6853 WideRegs.push_back(B.buildAnyExt(I32, Unmerge.getReg(I)).getReg(0));
6854
6855 int NumElts = StoreVT.getNumElements();
6856
6857 return B.buildBuildVector(LLT::fixed_vector(NumElts, I32), WideRegs)
6858 .getReg(0);
6859 }
6860
6861 if (ImageStore && ST.hasImageStoreD16Bug()) {
6862 if (StoreVT.getNumElements() == 2) {
6863 SmallVector<Register, 4> PackedRegs;
6864 Reg = B.buildBitcast(I32, RegI16).getReg(0);
6865 PackedRegs.push_back(Reg);
6866 PackedRegs.resize(2, B.buildUndef(I32).getReg(0));
6867 return B.buildBuildVector(LLT::fixed_vector(2, I32), PackedRegs)
6868 .getReg(0);
6869 }
6870
6871 if (StoreVT.getNumElements() == 3) {
6872 SmallVector<Register, 4> PackedRegs;
6873 auto Unmerge = B.buildUnmerge(I16, RegI16);
6874 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
6875 PackedRegs.push_back(Unmerge.getReg(I));
6876 PackedRegs.resize(6, B.buildUndef(I16).getReg(0));
6877 Reg = B.buildBuildVector(LLT::fixed_vector(6, I16), PackedRegs).getReg(0);
6878 return B.buildBitcast(LLT::fixed_vector(3, I32), Reg).getReg(0);
6879 }
6880
6881 if (StoreVT.getNumElements() == 4) {
6882 SmallVector<Register, 4> PackedRegs;
6883 Reg = B.buildBitcast(LLT::fixed_vector(2, I32), RegI16).getReg(0);
6884 auto Unmerge = B.buildUnmerge(I32, Reg);
6885 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
6886 PackedRegs.push_back(Unmerge.getReg(I));
6887 PackedRegs.resize(4, B.buildUndef(I32).getReg(0));
6888 return B.buildBuildVector(LLT::fixed_vector(4, I32), PackedRegs)
6889 .getReg(0);
6890 }
6891
6892 llvm_unreachable("invalid data type");
6893 }
6894
6895 if (StoreVT.isVector() && StoreVT.getNumElements() == 3 &&
6896 StoreVT.getElementType().getSizeInBits() == 16) {
6897 Reg = B.buildPadVectorWithUndefElements(
6898 LLT::fixed_vector(4, StoreVT.getElementType()), Reg)
6899 .getReg(0);
6900 }
6901 return Reg;
6902}
6903
6905 Register VData, LLT MemTy,
6906 bool IsFormat) const {
6907 MachineRegisterInfo *MRI = B.getMRI();
6908 LLT Ty = MRI->getType(VData);
6909
6910 // Fixup buffer resources themselves needing to be v4i128.
6912 return castBufferRsrcToV4I32(VData, B);
6913
6914 if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
6915 Ty = getBitcastRegisterType(Ty);
6916 VData = B.buildBitcast(Ty, VData).getReg(0);
6917 }
6918 if (Ty.isFloat(16)) {
6919 Ty = LLT::integer(16);
6920 VData = B.buildBitcast(Ty, VData).getReg(0);
6921 }
6922
6923 // Fixup illegal register types for i8 stores.
6924 if (Ty == LLT::integer(8) || Ty == LLT::integer(16))
6925 return B.buildAnyExt(LLT::integer(32), VData).getReg(0);
6926
6927 if (Ty.isVector()) {
6928 if (Ty.getElementType().getSizeInBits() == 16 && Ty.getNumElements() <= 4) {
6929 if (IsFormat)
6930 return handleD16VData(B, *MRI, VData);
6931 }
6932 }
6933
6934 return VData;
6935}
6936
6939 const AMDGPU::BufferIntrinsicInfo &Info) const {
6940 MachineIRBuilder &B = Helper.MIRBuilder;
6941 MachineRegisterInfo &MRI = *B.getMRI();
6942 auto OpIdx = [&](unsigned Arg) { return AMDGPU::getMIROperandIdx(MI, Arg); };
6943 const bool IsTyped = Info.isTyped();
6944 const bool IsFormat = Info.isFormat();
6945
6946 Register VData = MI.getOperand(OpIdx(*Info.getVDataArg())).getReg();
6947 LLT Ty = MRI.getType(VData);
6948 LLT EltTy = Ty.getScalarType();
6949 const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
6950 const LLT I32 = LLT::integer(32);
6951
6952 MachineMemOperand *MMO = *MI.memoperands_begin();
6953 const int MemSize = MMO->getSize().getValue();
6954 LLT MemTy = MMO->getMemoryType();
6955
6956 if (IsFormat && !IsTyped && !IsD16 && MemTy.getSizeInBits() < 32) {
6957 const Function &Fn = B.getMF().getFunction();
6959 Fn, "unsupported sub-dword format buffer store", MI.getDebugLoc()));
6960 MI.eraseFromParent();
6961 return true;
6962 }
6963
6964 VData = fixStoreSourceType(B, VData, MemTy, IsFormat);
6965
6966 castBufferRsrcArgToV4I32(MI, B, OpIdx(Info.RsrcArg));
6967 Register RSrc = MI.getOperand(OpIdx(Info.RsrcArg)).getReg();
6968
6969 unsigned ImmOffset;
6970
6971 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
6972 const bool HasVIndex = VIndexArg.has_value();
6973 Register VIndex = HasVIndex ? MI.getOperand(OpIdx(*VIndexArg)).getReg()
6974 : B.buildConstant(I32, 0).getReg(0);
6975
6976 Register VOffset = MI.getOperand(OpIdx(*Info.getVOffsetArg())).getReg();
6977 Register SOffset = MI.getOperand(OpIdx(*Info.getSOffsetArg())).getReg();
6978
6979 unsigned Format = 0;
6980 if (std::optional<unsigned> FormatArg = Info.getFormatArg())
6981 Format = MI.getOperand(OpIdx(*FormatArg)).getImm();
6982
6983 unsigned AuxiliaryData = MI.getOperand(OpIdx(*Info.getAuxArg())).getImm();
6984
6985 std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
6986
6987 unsigned Opc;
6988 if (IsTyped) {
6989 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16 :
6990 AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT;
6991 } else if (IsFormat) {
6992 Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16 :
6993 AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT;
6994 } else {
6995 switch (MemSize) {
6996 case 1:
6997 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE;
6998 break;
6999 case 2:
7000 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT;
7001 break;
7002 default:
7003 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE;
7004 break;
7005 }
7006 }
7007
7008 auto MIB = B.buildInstr(Opc)
7009 .addUse(VData) // vdata
7010 .addUse(RSrc) // rsrc
7011 .addUse(VIndex) // vindex
7012 .addUse(VOffset) // voffset
7013 .addUse(SOffset) // soffset
7014 .addImm(ImmOffset); // offset(imm)
7015
7016 if (IsTyped)
7017 MIB.addImm(Format);
7018
7019 MIB.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
7020 .addImm(HasVIndex ? -1 : 0) // idxen(imm)
7021 .addMemOperand(MMO);
7022
7023 MI.eraseFromParent();
7024 return true;
7025}
7026
7027static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc,
7028 Register VIndex, Register VOffset, Register SOffset,
7029 unsigned ImmOffset, unsigned Format,
7030 unsigned AuxiliaryData, MachineMemOperand *MMO,
7031 bool IsTyped, bool HasVIndex, MachineIRBuilder &B) {
7032 auto MIB = B.buildInstr(Opc)
7033 .addDef(LoadDstReg) // vdata
7034 .addUse(RSrc) // rsrc
7035 .addUse(VIndex) // vindex
7036 .addUse(VOffset) // voffset
7037 .addUse(SOffset) // soffset
7038 .addImm(ImmOffset); // offset(imm)
7039
7040 if (IsTyped)
7041 MIB.addImm(Format);
7042
7043 MIB.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
7044 .addImm(HasVIndex ? -1 : 0) // idxen(imm)
7045 .addMemOperand(MMO);
7046}
7047
7048static void buildTFEBufferLoad(unsigned Opc, ArrayRef<Register> ValueDsts,
7049 Register StatusDst, Register RSrc,
7050 Register VIndex, Register VOffset,
7051 Register SOffset, unsigned ImmOffset,
7052 unsigned Format, unsigned AuxiliaryData,
7053 MachineMemOperand *MMO, bool IsTyped,
7054 bool HasVIndex, MachineIRBuilder &B) {
7055 LLT LoadTy = LLT::fixed_vector(ValueDsts.size() + 1, LLT::integer(32));
7056 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(LoadTy);
7057 buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
7058 Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
7059 SmallVector<Register, 5> Unmerge(ValueDsts);
7060 Unmerge.push_back(StatusDst);
7061 B.buildUnmerge(Unmerge, LoadDstReg);
7062}
7063
7066 const AMDGPU::BufferIntrinsicInfo &Info) const {
7067 MachineIRBuilder &B = Helper.MIRBuilder;
7068 MachineRegisterInfo &MRI = *B.getMRI();
7069 GISelChangeObserver &Observer = Helper.Observer;
7070 auto OpIdx = [&](unsigned Arg) { return AMDGPU::getMIROperandIdx(MI, Arg); };
7071 const bool IsTyped = Info.isTyped();
7072 const bool IsFormat = Info.isFormat();
7073
7074 // FIXME: Verifier should enforce 1 MMO for these intrinsics.
7075 MachineMemOperand *MMO = *MI.memoperands_begin();
7076 const LLT MemTy = MMO->getMemoryType();
7077 const LLT I32 = LLT::integer(32);
7078
7079 Register Dst = MI.getOperand(0).getReg();
7080
7081 Register StatusDst;
7082 assert(MI.getNumExplicitDefs() == 1 || MI.getNumExplicitDefs() == 2);
7083 bool IsTFE = MI.getNumExplicitDefs() == 2;
7084 if (IsTFE)
7085 StatusDst = MI.getOperand(1).getReg();
7086
7087 castBufferRsrcArgToV4I32(MI, B, OpIdx(Info.RsrcArg));
7088 Register RSrc = MI.getOperand(OpIdx(Info.RsrcArg)).getReg();
7089
7090 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
7091 const bool HasVIndex = VIndexArg.has_value();
7092 Register VIndex = HasVIndex ? MI.getOperand(OpIdx(*VIndexArg)).getReg()
7093 : B.buildConstant(I32, 0).getReg(0);
7094
7095 Register VOffset = MI.getOperand(OpIdx(*Info.getVOffsetArg())).getReg();
7096 Register SOffset = MI.getOperand(OpIdx(*Info.getSOffsetArg())).getReg();
7097
7098 unsigned Format = 0;
7099 if (std::optional<unsigned> FormatArg = Info.getFormatArg())
7100 Format = MI.getOperand(OpIdx(*FormatArg)).getImm();
7101
7102 unsigned AuxiliaryData = MI.getOperand(OpIdx(*Info.getAuxArg())).getImm();
7103 unsigned ImmOffset;
7104
7105 LLT Ty = MRI.getType(Dst);
7106 // Make addrspace 8 pointers loads into 4xi32 loads here, so the rest of the
7107 // logic doesn't have to handle that case.
7108 if (hasBufferRsrcWorkaround(Ty)) {
7109 Observer.changingInstr(MI);
7110 Ty = castBufferRsrcFromV4I32(MI, B, MRI, 0);
7111 Observer.changedInstr(MI);
7112 Dst = MI.getOperand(0).getReg();
7113 B.setInsertPt(B.getMBB(), MI);
7114 }
7115 if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
7116 Ty = getBitcastRegisterType(Ty);
7117 Observer.changingInstr(MI);
7118 Helper.bitcastDst(MI, Ty, 0);
7119 Observer.changedInstr(MI);
7120 Dst = MI.getOperand(0).getReg();
7121 B.setInsertPt(B.getMBB(), MI);
7122 }
7123
7124 LLT EltTy = Ty.getScalarType();
7125 const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
7126 const bool Unpacked = ST.hasUnpackedD16VMem();
7127
7128 if (IsFormat && !IsTyped && !IsD16 && MemTy.getSizeInBits() < 32) {
7129 const Function &Fn = B.getMF().getFunction();
7131 Fn, "unsupported sub-dword format buffer load", MI.getDebugLoc()));
7132 B.buildUndef(Dst);
7133 if (IsTFE)
7134 B.buildUndef(StatusDst);
7135 MI.eraseFromParent();
7136 return true;
7137 }
7138
7139 if (!IsTyped && IsD16 && IsTFE && !ST.hasBufferTFEFormatD16()) {
7140 const Function &Fn = B.getMF().getFunction();
7142 Fn, "TFE D16 format buffer load is not supported on this GPU",
7143 MI.getDebugLoc()));
7144 B.buildUndef(Dst);
7145 B.buildUndef(StatusDst);
7146 MI.eraseFromParent();
7147 return true;
7148 }
7149
7150 std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
7151
7152 unsigned Opc;
7153
7154 // TODO: Support TFE for typed and narrow loads.
7155 if (IsTyped) {
7156 if (IsTFE)
7157 return false;
7158 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 :
7159 AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
7160 } else if (IsFormat) {
7161 if (IsD16) {
7162 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE
7163 : AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
7164 } else {
7165 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE
7166 : AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
7167 }
7168 } else {
7169 switch (MemTy.getSizeInBits()) {
7170 case 8:
7171 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE
7172 : AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
7173 break;
7174 case 16:
7175 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE
7176 : AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
7177 break;
7178 default:
7179 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE
7180 : AMDGPU::G_AMDGPU_BUFFER_LOAD;
7181 break;
7182 }
7183 }
7184
7185 if (IsTFE && IsD16 && Ty.isVector()) {
7186 // Value dwords need not cover Ty exactly: v3i16 needs 2 dwords for 48 bits.
7187 const unsigned NumElts = Ty.getNumElements();
7188 const unsigned NumValueDWords = Unpacked ? NumElts : divideCeil(NumElts, 2);
7189
7190 SmallVector<Register, 4> ValueDWords;
7191 for (unsigned I = 0; I != NumValueDWords; ++I)
7192 ValueDWords.push_back(MRI.createGenericVirtualRegister(I32));
7193 buildTFEBufferLoad(Opc, ValueDWords, StatusDst, RSrc, VIndex, VOffset,
7194 SOffset, ImmOffset, Format, AuxiliaryData, MMO, IsTyped,
7195 HasVIndex, B);
7196
7197 if (Unpacked) {
7198 for (Register &R : ValueDWords)
7199 R = B.buildTrunc(EltTy, R).getReg(0);
7200 B.buildMergeLikeInstr(Dst, ValueDWords);
7201 } else {
7202 Register Merged =
7203 NumValueDWords == 1
7204 ? ValueDWords[0]
7205 : B.buildMergeLikeInstr(LLT::fixed_vector(NumValueDWords, I32),
7206 ValueDWords)
7207 .getReg(0);
7208 LLT PackedTy = LLT::fixed_vector(NumValueDWords * 2, EltTy);
7209 if (PackedTy == Ty) {
7210 B.buildBitcast(Dst, Merged);
7211 } else {
7212 Register Packed = B.buildBitcast(PackedTy, Merged).getReg(0);
7213 B.buildDeleteTrailingVectorElements(Dst, Packed);
7214 }
7215 }
7216 } else if (IsTFE) {
7217 const unsigned NumValueDWords = divideCeil(Ty.getSizeInBits(), 32);
7218 Register DstInt =
7219 EltTy.isFloat() ? MRI.createGenericVirtualRegister(Ty.changeElementType(
7220 LLT::integer(EltTy.getSizeInBits())))
7221 : Dst;
7222 if (MemTy.getSizeInBits() < 32) {
7223 Register ExtDst = MRI.createGenericVirtualRegister(I32);
7224 buildTFEBufferLoad(Opc, ExtDst, StatusDst, RSrc, VIndex, VOffset, SOffset,
7225 ImmOffset, Format, AuxiliaryData, MMO, IsTyped,
7226 HasVIndex, B);
7227 B.buildTrunc(DstInt, ExtDst);
7228 } else if (NumValueDWords == 1) {
7229 buildTFEBufferLoad(Opc, DstInt, StatusDst, RSrc, VIndex, VOffset, SOffset,
7230 ImmOffset, Format, AuxiliaryData, MMO, IsTyped,
7231 HasVIndex, B);
7232 } else {
7233 SmallVector<Register, 4> ValueDWords;
7234 for (unsigned I = 0; I != NumValueDWords; ++I)
7235 ValueDWords.push_back(MRI.createGenericVirtualRegister(I32));
7236 buildTFEBufferLoad(Opc, ValueDWords, StatusDst, RSrc, VIndex, VOffset,
7237 SOffset, ImmOffset, Format, AuxiliaryData, MMO,
7238 IsTyped, HasVIndex, B);
7239 B.buildMergeLikeInstr(DstInt, ValueDWords);
7240 }
7241 if (DstInt != Dst)
7242 B.buildBitcast(Dst, DstInt);
7243 } else if ((!IsD16 && MemTy.getSizeInBits() < 32) ||
7244 (IsD16 && !Ty.isVector())) {
7245 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(I32);
7246 buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
7247 Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
7248 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
7249 B.buildTrunc(Dst, LoadDstReg);
7250 } else if (Unpacked && IsD16 && Ty.isVector()) {
7251 LLT UnpackedTy = LLT::fixed_vector(Ty.getNumElements(), LLT::integer(32));
7252 Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(UnpackedTy);
7253 buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
7254 Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
7255 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
7256 // FIXME: G_TRUNC should work, but legalization currently fails
7257 auto Unmerge = B.buildUnmerge(I32, LoadDstReg);
7259 for (unsigned I = 0, N = Unmerge->getNumOperands() - 1; I != N; ++I)
7260 Repack.push_back(B.buildTrunc(EltTy, Unmerge.getReg(I)).getReg(0));
7261 B.buildMergeLikeInstr(Dst, Repack);
7262 } else {
7263 buildBufferLoad(Opc, Dst, RSrc, VIndex, VOffset, SOffset, ImmOffset, Format,
7264 AuxiliaryData, MMO, IsTyped, HasVIndex, B);
7265 }
7266
7267 MI.eraseFromParent();
7268 return true;
7269}
7270
7271static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
7272 switch (IntrID) {
7273 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
7274 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
7275 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
7276 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
7277 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP;
7278 case Intrinsic::amdgcn_raw_buffer_atomic_add:
7279 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
7280 case Intrinsic::amdgcn_struct_buffer_atomic_add:
7281 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
7282 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD;
7283 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
7284 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
7285 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
7286 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
7287 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB;
7288 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
7289 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
7290 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
7291 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
7292 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN;
7293 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
7294 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
7295 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
7296 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
7297 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN;
7298 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
7299 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
7300 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
7301 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
7302 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX;
7303 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
7304 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
7305 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
7306 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
7307 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX;
7308 case Intrinsic::amdgcn_raw_buffer_atomic_and:
7309 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
7310 case Intrinsic::amdgcn_struct_buffer_atomic_and:
7311 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
7312 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND;
7313 case Intrinsic::amdgcn_raw_buffer_atomic_or:
7314 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
7315 case Intrinsic::amdgcn_struct_buffer_atomic_or:
7316 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
7317 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR;
7318 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
7319 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
7320 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
7321 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
7322 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR;
7323 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
7324 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
7325 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
7326 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
7327 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC;
7328 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
7329 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
7330 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
7331 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
7332 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC;
7333 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
7334 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
7335 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
7336 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
7337 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP;
7338 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
7339 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
7340 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
7341 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
7342 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD;
7343 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
7344 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
7345 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
7346 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
7347 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN;
7348 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
7349 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
7350 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
7351 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
7352 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX;
7353 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
7354 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
7355 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
7356 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
7357 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32;
7358 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
7359 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
7360 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
7361 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
7362 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32;
7363 default:
7364 llvm_unreachable("unhandled atomic opcode");
7365 }
7366}
7367
7370 const AMDGPU::BufferIntrinsicInfo &Info) const {
7371 auto OpIdx = [&](unsigned Arg) { return AMDGPU::getMIROperandIdx(MI, Arg); };
7372 std::optional<unsigned> CmpArg = Info.getCmpArg();
7373 const bool IsCmpSwap = CmpArg.has_value();
7374
7375 Register Dst = MI.getOperand(0).getReg();
7376 // Since we don't have 128-bit atomics, we don't need to handle the case of
7377 // p8 argmunents to the atomic itself
7378 Register VData = MI.getOperand(OpIdx(*Info.getVDataArg())).getReg();
7379
7380 Register CmpVal;
7381 if (IsCmpSwap)
7382 CmpVal = MI.getOperand(OpIdx(*CmpArg)).getReg();
7383
7384 castBufferRsrcArgToV4I32(MI, B, OpIdx(Info.RsrcArg));
7385 Register RSrc = MI.getOperand(OpIdx(Info.RsrcArg)).getReg();
7386
7387 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
7388 const bool HasVIndex = VIndexArg.has_value();
7389 Register VIndex = HasVIndex ? MI.getOperand(OpIdx(*VIndexArg)).getReg()
7390 : B.buildConstant(LLT::integer(32), 0).getReg(0);
7391
7392 Register VOffset = MI.getOperand(OpIdx(*Info.getVOffsetArg())).getReg();
7393 Register SOffset = MI.getOperand(OpIdx(*Info.getSOffsetArg())).getReg();
7394 unsigned AuxiliaryData = MI.getOperand(OpIdx(*Info.getAuxArg())).getImm();
7395
7396 MachineMemOperand *MMO = *MI.memoperands_begin();
7397
7398 unsigned ImmOffset;
7399 std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
7400
7401 auto MIB = B.buildInstr(getBufferAtomicPseudo(Info.Intr))
7402 .addDef(Dst)
7403 .addUse(VData); // vdata
7404
7405 if (IsCmpSwap)
7406 MIB.addReg(CmpVal);
7407
7408 MIB.addUse(RSrc) // rsrc
7409 .addUse(VIndex) // vindex
7410 .addUse(VOffset) // voffset
7411 .addUse(SOffset) // soffset
7412 .addImm(ImmOffset) // offset(imm)
7413 .addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
7414 .addImm(HasVIndex ? -1 : 0) // idxen(imm)
7415 .addMemOperand(MMO);
7416
7417 MI.eraseFromParent();
7418 return true;
7419}
7420
7421/// Turn a set of f16 typed registers in \p AddrRegs into a dword sized
7422/// vector with f16 typed elements.
7424 SmallVectorImpl<Register> &PackedAddrs,
7425 unsigned ArgOffset,
7427 bool IsA16, bool IsG16) {
7428 auto EndIdx = Intr->VAddrEnd;
7429
7430 for (unsigned I = Intr->VAddrStart; I < EndIdx; I++) {
7431 MachineOperand &SrcOp = MI.getOperand(ArgOffset + I);
7432 if (!SrcOp.isReg())
7433 continue; // _L to _LZ may have eliminated this.
7434
7435 Register AddrReg = SrcOp.getReg();
7436
7437 if ((I < Intr->GradientStart) ||
7438 (I >= Intr->GradientStart && I < Intr->CoordStart && !IsG16) ||
7439 (I >= Intr->CoordStart && !IsA16)) {
7440 if ((I < Intr->GradientStart) && IsA16 &&
7441 (B.getMRI()->getType(AddrReg) == F16)) {
7442 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
7443 // Special handling of bias when A16 is on. Bias is of type half but
7444 // occupies full 32-bit.
7445 PackedAddrs.push_back(
7446 B.buildBuildVector(V2F16, {AddrReg, B.buildUndef(F16).getReg(0)})
7447 .getReg(0));
7448 } else {
7449 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
7450 "Bias needs to be converted to 16 bit in A16 mode");
7451 // Handle any gradient or coordinate operands that should not be packed
7452 AddrReg = B.buildBitcast(V2F16, AddrReg).getReg(0);
7453 PackedAddrs.push_back(AddrReg);
7454 }
7455 } else {
7456 const LLT EltTy = B.getMRI()->getType(AddrReg);
7457 const LLT V2EltTy = LLT::fixed_vector(2, EltTy);
7458 // Dz/dh, dz/dv and the last odd coord are packed with undef. Also, in 1D,
7459 // derivatives dx/dh and dx/dv are packed with undef.
7460 if (((I + 1) >= EndIdx) ||
7461 ((Intr->NumGradients / 2) % 2 == 1 &&
7462 (I == static_cast<unsigned>(Intr->GradientStart +
7463 (Intr->NumGradients / 2) - 1) ||
7464 I == static_cast<unsigned>(Intr->GradientStart +
7465 Intr->NumGradients - 1))) ||
7466 // Check for _L to _LZ optimization
7467 !MI.getOperand(ArgOffset + I + 1).isReg()) {
7468 PackedAddrs.push_back(
7469 B.buildBuildVector(V2EltTy,
7470 {AddrReg, B.buildUndef(EltTy).getReg(0)})
7471 .getReg(0));
7472 } else {
7473 PackedAddrs.push_back(
7474 B.buildBuildVector(
7475 V2EltTy, {AddrReg, MI.getOperand(ArgOffset + I + 1).getReg()})
7476 .getReg(0));
7477 ++I;
7478 }
7479 }
7480 }
7481}
7482
7483/// Convert from separate vaddr components to a single vector address register,
7484/// and replace the remaining operands with $noreg.
7486 int DimIdx, int NumVAddrs) {
7487 SmallVector<Register, 8> AddrRegs;
7488 for (int I = 0; I != NumVAddrs; ++I) {
7489 MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
7490 if (SrcOp.isReg()) {
7492 LLT I32 = LLT::integer(32);
7493 assert(B.getMRI()->getType(Reg).getSizeInBits() == 32);
7494 if (B.getMRI()->getType(Reg) != I32)
7495 Reg = B.buildBitcast(I32, Reg).getReg(0);
7496 AddrRegs.push_back(Reg);
7497 }
7498 }
7499
7500 int NumAddrRegs = AddrRegs.size();
7501 if (NumAddrRegs != 1) {
7502 LLT EltTy = B.getMRI()->getType(AddrRegs[0]);
7503 auto VAddr =
7504 B.buildBuildVector(LLT::fixed_vector(NumAddrRegs, EltTy), AddrRegs);
7505 MI.getOperand(DimIdx).setReg(VAddr.getReg(0));
7506 }
7507
7508 for (int I = 1; I != NumVAddrs; ++I) {
7509 MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
7510 if (SrcOp.isReg())
7511 MI.getOperand(DimIdx + I).setReg(AMDGPU::NoRegister);
7512 }
7513}
7514
7515/// Rewrite image intrinsics to use register layouts expected by the subtarget.
7516///
7517/// Depending on the subtarget, load/store with 16-bit element data need to be
7518/// rewritten to use the low half of 32-bit registers, or directly use a packed
7519/// layout. 16-bit addresses should also sometimes be packed into 32-bit
7520/// registers.
7521///
7522/// We don't want to directly select image instructions just yet, but also want
7523/// to exposes all register repacking to the legalizer/combiners. We also don't
7524/// want a selected instruction entering RegBankSelect. In order to avoid
7525/// defining a multitude of intermediate image instructions, directly hack on
7526/// the intrinsic's arguments. In cases like a16 addresses, this requires
7527/// padding now unnecessary arguments with $noreg.
7530 const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
7531
7532 const MachineFunction &MF = *MI.getMF();
7533 const unsigned NumDefs = MI.getNumExplicitDefs();
7534 const unsigned ArgOffset = NumDefs + 1;
7535 bool IsTFE = NumDefs == 2;
7536 // We are only processing the operands of d16 image operations on subtargets
7537 // that use the unpacked register layout, or need to repack the TFE result.
7538
7539 // TODO: Do we need to guard against already legalized intrinsics?
7540 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
7542
7543 MachineRegisterInfo *MRI = B.getMRI();
7544 const LLT I32 = LLT::integer(32);
7545 const LLT I16 = LLT::integer(16);
7546 const LLT V2I16 = LLT::fixed_vector(2, I16);
7547
7548 unsigned DMask = 0;
7549 Register VData;
7550 LLT Ty;
7551
7552 if (!BaseOpcode->NoReturn || BaseOpcode->Store) {
7553 VData = MI.getOperand(NumDefs == 0 ? 1 : 0).getReg();
7554 Ty = MRI->getType(VData);
7555 }
7556
7557 const bool IsAtomicPacked16Bit =
7558 (BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
7559 BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16);
7560
7561 // Check for 16 bit addresses and pack if true.
7562 LLT GradTy =
7563 MRI->getType(MI.getOperand(ArgOffset + Intr->GradientStart).getReg());
7564 LLT AddrTy =
7565 MRI->getType(MI.getOperand(ArgOffset + Intr->CoordStart).getReg());
7566 const bool GradTyIs16 = GradTy == I16 || GradTy == F16;
7567 const bool AddrTyIs16 = AddrTy == I16 || AddrTy == F16;
7568 const bool DataTyIs16 =
7569 Ty.getScalarType() == I16 || Ty.getScalarType() == F16;
7570 const bool IsG16 =
7571 ST.hasG16() ? (BaseOpcode->Gradients && GradTyIs16) : GradTyIs16;
7572 const bool IsA16 = AddrTyIs16;
7573 const bool IsD16 = !IsAtomicPacked16Bit && DataTyIs16;
7574
7575 int DMaskLanes = 0;
7576 if (!BaseOpcode->Atomic) {
7577 DMask = MI.getOperand(ArgOffset + Intr->DMaskIndex).getImm();
7578 if (BaseOpcode->Gather4) {
7579 DMaskLanes = 4;
7580 } else if (DMask != 0) {
7581 DMaskLanes = llvm::popcount(DMask);
7582 } else if (!IsTFE && !BaseOpcode->Store) {
7583 // If dmask is 0, this is a no-op load. This can be eliminated.
7584 B.buildUndef(MI.getOperand(0));
7585 MI.eraseFromParent();
7586 return true;
7587 }
7588 }
7589
7590 Observer.changingInstr(MI);
7591 scope_exit ChangedInstr([&] { Observer.changedInstr(MI); });
7592
7593 const unsigned StoreOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16
7594 : AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE;
7595 const unsigned LoadOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16
7596 : AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD;
7597 unsigned NewOpcode = LoadOpcode;
7598 if (BaseOpcode->Store)
7599 NewOpcode = StoreOpcode;
7600 else if (BaseOpcode->NoReturn)
7601 NewOpcode = AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET;
7602
7603 // Track that we legalized this
7604 MI.setDesc(B.getTII().get(NewOpcode));
7605
7606 // Expecting to get an error flag since TFC is on - and dmask is 0 Force
7607 // dmask to be at least 1 otherwise the instruction will fail
7608 if (IsTFE && DMask == 0) {
7609 DMask = 0x1;
7610 DMaskLanes = 1;
7611 MI.getOperand(ArgOffset + Intr->DMaskIndex).setImm(DMask);
7612 }
7613
7614 if (BaseOpcode->Atomic) {
7615 Register VData0 = MI.getOperand(2).getReg();
7616 LLT Ty = MRI->getType(VData0);
7617
7618 // TODO: Allow atomic swap and bit ops for v2f16/v4f16
7619 if (Ty.isVector() && !IsAtomicPacked16Bit)
7620 return false;
7621
7622 if (BaseOpcode->AtomicX2) {
7623 Register VData1 = MI.getOperand(3).getReg();
7624 // The two values are packed in one register.
7625 LLT PackedTy = LLT::fixed_vector(2, Ty);
7626 auto Concat = B.buildBuildVector(PackedTy, {VData0, VData1});
7627 MI.getOperand(2).setReg(Concat.getReg(0));
7628 MI.getOperand(3).setReg(AMDGPU::NoRegister);
7629 }
7630 }
7631
7632 unsigned CorrectedNumVAddrs = Intr->NumVAddrs;
7633
7634 // Rewrite the addressing register layout before doing anything else.
7635 if (BaseOpcode->Gradients && !ST.hasG16() && (IsA16 != IsG16)) {
7636 // 16 bit gradients are supported, but are tied to the A16 control
7637 // so both gradients and addresses must be 16 bit
7638 return false;
7639 }
7640
7641 if (IsA16 && !ST.hasA16()) {
7642 // A16 not supported
7643 return false;
7644 }
7645
7646 const unsigned NSAMaxSize = ST.getNSAMaxSize(BaseOpcode->Sampler);
7647 const unsigned HasPartialNSA = ST.hasPartialNSAEncoding();
7648
7649 if (IsA16 || IsG16) {
7650 // Even if NumVAddrs == 1 we should pack it into a 32-bit value, because the
7651 // instructions expect VGPR_32
7652 SmallVector<Register, 4> PackedRegs;
7653
7654 packImage16bitOpsToDwords(B, MI, PackedRegs, ArgOffset, Intr, IsA16, IsG16);
7655
7656 // See also below in the non-a16 branch
7657 const bool UseNSA = ST.hasNSAEncoding() &&
7658 PackedRegs.size() >= ST.getNSAThreshold(MF) &&
7659 (PackedRegs.size() <= NSAMaxSize || HasPartialNSA);
7660 const bool UsePartialNSA =
7661 UseNSA && HasPartialNSA && PackedRegs.size() > NSAMaxSize;
7662
7663 if (UsePartialNSA) {
7664 // Pack registers that would go over NSAMaxSize into last VAddr register
7665 LLT PackedAddrTy =
7666 LLT::fixed_vector(2 * (PackedRegs.size() - NSAMaxSize + 1), F16);
7667 auto Concat = B.buildConcatVectors(
7668 PackedAddrTy, ArrayRef(PackedRegs).slice(NSAMaxSize - 1));
7669 PackedRegs[NSAMaxSize - 1] = Concat.getReg(0);
7670 PackedRegs.resize(NSAMaxSize);
7671 } else if (!UseNSA && PackedRegs.size() > 1) {
7672 LLT PackedAddrTy = LLT::fixed_vector(2 * PackedRegs.size(), F16);
7673 auto Concat = B.buildConcatVectors(PackedAddrTy, PackedRegs);
7674 PackedRegs[0] = Concat.getReg(0);
7675 PackedRegs.resize(1);
7676 }
7677
7678 const unsigned NumPacked = PackedRegs.size();
7679 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
7680 MachineOperand &SrcOp = MI.getOperand(ArgOffset + I);
7681 if (!SrcOp.isReg()) {
7682 assert(SrcOp.isImm() && SrcOp.getImm() == 0);
7683 continue;
7684 }
7685
7686 assert(SrcOp.getReg() != AMDGPU::NoRegister);
7687
7688 if (I - Intr->VAddrStart < NumPacked)
7689 SrcOp.setReg(PackedRegs[I - Intr->VAddrStart]);
7690 else
7691 SrcOp.setReg(AMDGPU::NoRegister);
7692 }
7693 } else {
7694 // If the register allocator cannot place the address registers contiguously
7695 // without introducing moves, then using the non-sequential address encoding
7696 // is always preferable, since it saves VALU instructions and is usually a
7697 // wash in terms of code size or even better.
7698 //
7699 // However, we currently have no way of hinting to the register allocator
7700 // that MIMG addresses should be placed contiguously when it is possible to
7701 // do so, so force non-NSA for the common 2-address case as a heuristic.
7702 //
7703 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
7704 // allocation when possible.
7705 //
7706 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
7707 // set of the remaining addresses.
7708 const bool UseNSA = ST.hasNSAEncoding() &&
7709 CorrectedNumVAddrs >= ST.getNSAThreshold(MF) &&
7710 (CorrectedNumVAddrs <= NSAMaxSize || HasPartialNSA);
7711 const bool UsePartialNSA =
7712 UseNSA && HasPartialNSA && CorrectedNumVAddrs > NSAMaxSize;
7713
7714 if (UsePartialNSA) {
7716 ArgOffset + Intr->VAddrStart + NSAMaxSize - 1,
7717 Intr->NumVAddrs - NSAMaxSize + 1);
7718 } else if (!UseNSA && Intr->NumVAddrs > 1) {
7719 convertImageAddrToPacked(B, MI, ArgOffset + Intr->VAddrStart,
7720 Intr->NumVAddrs);
7721 }
7722 }
7723
7724 int Flags = 0;
7725 if (IsA16)
7726 Flags |= 1;
7727 if (IsG16)
7728 Flags |= 2;
7729 MI.addOperand(MachineOperand::CreateImm(Flags));
7730
7731 if (BaseOpcode->NoReturn) { // No TFE for stores?
7732 // TODO: Handle dmask trim
7733 if (!Ty.isVector() || !IsD16)
7734 return true;
7735
7736 Register RepackedReg = handleD16VData(B, *MRI, VData, true);
7737 if (RepackedReg != VData) {
7738 MI.getOperand(1).setReg(RepackedReg);
7739 }
7740
7741 return true;
7742 }
7743
7744 Register DstReg = MI.getOperand(0).getReg();
7745 const LLT EltTy = Ty.getScalarType();
7746 const int NumElts = Ty.isVector() ? Ty.getNumElements() : 1;
7747
7748 // Confirm that the return type is large enough for the dmask specified
7749 if (NumElts < DMaskLanes)
7750 return false;
7751
7752 if (NumElts > 4 || DMaskLanes > 4)
7753 return false;
7754
7755 // Image atomic instructions are using DMask to specify how many bits
7756 // input/output data will have. 32-bits (i32, f32, v2f16) or 64-bits (i64,
7757 // f64, v4f16).
7758 // DMaskLanes for image atomic has default value '0'.
7759 // We must be sure that atomic variants (especially packed) will not be
7760 // truncated from v2f16 or v4f16 to f16 type.
7761 //
7762 // ChangeElementCount will be needed for image load where Ty is always scalar.
7763 const unsigned AdjustedNumElts = DMaskLanes == 0 ? 1 : DMaskLanes;
7764 const LLT AdjustedTy =
7765 DMaskLanes == 0
7766 ? Ty
7767 : Ty.changeElementCount(ElementCount::getFixed(AdjustedNumElts));
7768
7769 // The raw dword aligned data component of the load. The only legal cases
7770 // where this matters should be when using the packed D16 format, for
7771 // f16 -> <2 x f16>, and <3 x f16> -> <4 x f16>,
7772 LLT RoundedTy;
7773
7774 // I32 vector to cover all data, plus TFE result element.
7775 LLT TFETy;
7776
7777 // Register type to use for each loaded component. Will be I32 or V2I16.
7778 LLT RegTy;
7779
7780 if (IsD16 && ST.hasUnpackedD16VMem()) {
7781 RoundedTy =
7782 LLT::scalarOrVector(ElementCount::getFixed(AdjustedNumElts), I32);
7783 TFETy = LLT::fixed_vector(AdjustedNumElts + 1, I32);
7784 RegTy = I32;
7785 } else {
7786 unsigned EltSize = EltTy.getSizeInBits();
7787 unsigned RoundedElts = (AdjustedTy.getSizeInBits() + 31) / 32;
7788 unsigned RoundedSize = 32 * RoundedElts;
7789 RoundedTy = LLT::scalarOrVector(
7790 ElementCount::getFixed(RoundedSize / EltSize), EltTy);
7791 TFETy = LLT::fixed_vector(RoundedSize / 32 + 1, I32);
7792 RegTy = !IsTFE && EltSize == 16 ? V2I16 : I32;
7793 }
7794
7795 // The return type does not need adjustment.
7796 // TODO: Should we change f16 case to i32 or <2 x f16>?
7797 if (!IsTFE && (RoundedTy == Ty || !Ty.isVector()))
7798 return true;
7799
7800 Register Dst1Reg;
7801
7802 // Insert after the instruction.
7803 B.setInsertPt(*MI.getParent(), ++MI.getIterator());
7804
7805 // TODO: For TFE with d16, if we used a TFE type that was a multiple of <2 x
7806 // f16> instead of i32, we would only need 1 bitcast instead of multiple.
7807 const LLT LoadResultTy = IsTFE ? TFETy : RoundedTy;
7808 const int ResultNumRegs = LoadResultTy.getSizeInBits() / 32;
7809
7810 Register NewResultReg = MRI->createGenericVirtualRegister(LoadResultTy);
7811
7812 MI.getOperand(0).setReg(NewResultReg);
7813
7814 // In the IR, TFE is supposed to be used with a 2 element struct return
7815 // type. The instruction really returns these two values in one contiguous
7816 // register, with one additional dword beyond the loaded data. Rewrite the
7817 // return type to use a single register result.
7818
7819 if (IsTFE) {
7820 Dst1Reg = MI.getOperand(1).getReg();
7821 if (MRI->getType(Dst1Reg) != I32)
7822 return false;
7823
7824 // TODO: Make sure the TFE operand bit is set.
7825 MI.removeOperand(1);
7826
7827 // Handle the easy case that requires no repack instructions.
7828 if (!Ty.isVector() && Ty.getSizeInBits() == 32) {
7829 auto Unmerge = B.buildUnmerge({I32, I32}, NewResultReg);
7830 B.buildBitcast(DstReg, Unmerge.getReg(0));
7831 B.buildCopy(Dst1Reg, Unmerge.getReg(1));
7832 return true;
7833 }
7834 }
7835
7836 // Now figure out how to copy the new result register back into the old
7837 // result.
7838 SmallVector<Register, 5> ResultRegs(ResultNumRegs, Dst1Reg);
7839
7840 const int NumDataRegs = IsTFE ? ResultNumRegs - 1 : ResultNumRegs;
7841
7842 if (ResultNumRegs == 1) {
7843 assert(!IsTFE);
7844 ResultRegs[0] = NewResultReg;
7845 } else {
7846 // We have to repack into a new vector of some kind.
7847 for (int I = 0; I != NumDataRegs; ++I)
7848 ResultRegs[I] = MRI->createGenericVirtualRegister(RegTy);
7849 B.buildUnmerge(ResultRegs, NewResultReg);
7850
7851 // Drop the final TFE element to get the data part. The TFE result is
7852 // directly written to the right place already.
7853 if (IsTFE)
7854 ResultRegs.resize(NumDataRegs);
7855 }
7856
7857 // For an f16 scalar result, we form an i32 result with a truncate regardless
7858 // of packed vs. unpacked.
7859 if (IsD16 && !Ty.isVector()) {
7860 B.buildTrunc(DstReg, ResultRegs[0]);
7861 return true;
7862 }
7863
7864 // Avoid a build/concat_vector of 1 entry.
7865 if ((Ty == V2I16 || Ty == V2F16) && NumDataRegs == 1 &&
7866 !ST.hasUnpackedD16VMem()) {
7867 B.buildBitcast(DstReg, ResultRegs[0]);
7868 return true;
7869 }
7870
7871 assert(Ty.isVector());
7872
7873 if (IsD16) {
7874 // For packed D16 results with TFE enabled, all the data components are
7875 // I32. Cast back to the expected type.
7876 //
7877 // TODO: We don't really need to use load i32 elements. We would only need
7878 // one cast for the TFE result if a multiple of v2f16 was used.
7879 if (RegTy != V2I16 && !ST.hasUnpackedD16VMem()) {
7880 for (Register &Reg : ResultRegs)
7881 Reg = B.buildBitcast(V2I16, Reg).getReg(0);
7882 } else if (ST.hasUnpackedD16VMem()) {
7883 for (Register &Reg : ResultRegs)
7884 Reg = B.buildTrunc(I16, Reg).getReg(0);
7885 }
7886 }
7887
7888 auto padWithUndef = [&](LLT Ty, int NumElts) {
7889 if (NumElts == 0)
7890 return;
7891 Register Undef = B.buildUndef(Ty).getReg(0);
7892 for (int I = 0; I != NumElts; ++I)
7893 ResultRegs.push_back(Undef);
7894 };
7895
7896 // Pad out any elements eliminated due to the dmask.
7897 LLT ResTy = MRI->getType(ResultRegs[0]);
7898 if (!ResTy.isVector()) {
7899 padWithUndef(ResTy, NumElts - ResultRegs.size());
7900 B.buildBuildVector(DstReg, ResultRegs);
7901 return true;
7902 }
7903
7904 assert(!ST.hasUnpackedD16VMem() && (ResTy == V2I16 || ResTy == V2F16));
7905 const int RegsToCover = (Ty.getSizeInBits() + 31) / 32;
7906
7907 // Deal with the one annoying legal case.
7908 const LLT V3I16 = LLT::fixed_vector(3, I16);
7909 const LLT V3F16 = LLT::fixed_vector(3, F16);
7910 if (Ty == V3I16 || Ty == V3F16) {
7911 if (IsTFE) {
7912 if (ResultRegs.size() == 1) {
7913 NewResultReg = ResultRegs[0];
7914 } else if (ResultRegs.size() == 2) {
7915 LLT V4I16 = LLT::fixed_vector(4, I16);
7916 NewResultReg = B.buildConcatVectors(V4I16, ResultRegs).getReg(0);
7917 } else {
7918 return false;
7919 }
7920 }
7921
7922 LLT DstTy = MRI->getType(DstReg);
7923 LLT NewResTy = MRI->getType(NewResultReg);
7924 LLT ResEltTy = NewResTy.getElementType();
7925 Register ResizeDst = DstTy.getElementType() == ResEltTy
7926 ? DstReg
7927 : MRI->createGenericVirtualRegister(
7928 DstTy.changeElementType(ResEltTy));
7929
7930 if (DstTy.getNumElements() < NewResTy.getNumElements()) {
7931 B.buildDeleteTrailingVectorElements(ResizeDst, NewResultReg);
7932 } else {
7933 B.buildPadVectorWithUndefElements(ResizeDst, NewResultReg);
7934 }
7935 if (ResizeDst != DstReg)
7936 B.buildBitcast(DstReg, ResizeDst);
7937 return true;
7938 }
7939
7940 padWithUndef(ResTy, RegsToCover - ResultRegs.size());
7941 B.buildConcatVectors(DstReg, ResultRegs);
7942 return true;
7943}
7944
7946 MachineInstr &MI) const {
7947 MachineIRBuilder &B = Helper.MIRBuilder;
7948 GISelChangeObserver &Observer = Helper.Observer;
7949
7950 Register OrigDst = MI.getOperand(0).getReg();
7951 Register Dst;
7952 LLT Ty = B.getMRI()->getType(OrigDst);
7953 unsigned Size = Ty.getSizeInBits();
7954 MachineFunction &MF = B.getMF();
7955 bool HasMMO = !MI.memoperands_empty();
7956
7957 // S_BUFFER_LOAD only produces values that fill whole SGPRs, apart from the
7958 // subword loads below. Those truncate from an s32 result, so they need a
7959 // scalar destination.
7960 bool IsSubwordLoad = Ty.isScalar() && Size < 32 && ST.hasScalarSubwordLoads();
7961 if (Size % 32 != 0 && !IsSubwordLoad) {
7962 const Function &Fn = MF.getFunction();
7964 Fn, "unsupported s_buffer_load result type", MI.getDebugLoc()));
7965 B.buildUndef(OrigDst);
7966 MI.eraseFromParent();
7967 return true;
7968 }
7969
7970 unsigned Opc = 0;
7971 if (IsSubwordLoad) {
7972 assert(Size == 8 || Size == 16);
7973 Opc = Size == 8 ? AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE
7974 : AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT;
7975 // The 8-bit and 16-bit scalar buffer load instructions have 32-bit
7976 // destination register.
7977 Dst = B.getMRI()->createGenericVirtualRegister(LLT::integer(32));
7978 } else {
7979 Opc = AMDGPU::G_AMDGPU_S_BUFFER_LOAD;
7980 Dst = OrigDst;
7981 }
7982
7983 Observer.changingInstr(MI);
7984
7985 // Handle needing to s.buffer.load() a p8 value.
7986 if (hasBufferRsrcWorkaround(Ty)) {
7987 Ty = castBufferRsrcFromV4I32(MI, B, *B.getMRI(), 0);
7988 B.setInsertPt(B.getMBB(), MI);
7989 }
7991 Ty = getBitcastRegisterType(Ty);
7992 Helper.bitcastDst(MI, Ty, 0);
7993 B.setInsertPt(B.getMBB(), MI);
7994 }
7995
7996 MI.setDesc(B.getTII().get(Opc));
7997 MI.removeOperand(1);
7999
8000 if (!HasMMO) {
8001 // Legacy intrinsic that doesn't take a pointer and so can't already have an
8002 // MMO.
8003 const unsigned MemSize = (Size + 7) / 8;
8004 const Align MemAlign = B.getDataLayout().getABITypeAlign(
8010 MemSize, MemAlign);
8011 MI.addMemOperand(MF, MMO);
8012 }
8013 if (Dst != OrigDst) {
8014 MI.getOperand(0).setReg(Dst);
8015 B.setInsertPt(B.getMBB(), ++B.getInsertPt());
8016 B.buildTrunc(OrigDst, Dst);
8017 }
8018
8019 // If we don't have 96-bit result scalar loads, widening to 128-bit should
8020 // always be legal. We may need to restore this to a 96-bit result if it turns
8021 // out this needs to be converted to a vector load during RegBankSelect.
8022 if (!isPowerOf2_32(Size) && (Size != 96 || !ST.hasScalarDwordx3Loads())) {
8023 if (Ty.isVector())
8025 else
8026 Helper.widenScalarDst(MI, getPow2ScalarType(Ty), 0);
8027 }
8028
8029 Observer.changedInstr(MI);
8030 return true;
8031}
8032
8034 MachineInstr &MI) const {
8035 MachineIRBuilder &B = Helper.MIRBuilder;
8036 GISelChangeObserver &Observer = Helper.Observer;
8037 Observer.changingInstr(MI);
8038 MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH));
8039 MI.removeOperand(0); // Remove intrinsic ID
8041 Observer.changedInstr(MI);
8042 return true;
8043}
8044
8045// TODO: Move to selection
8047 MachineInstr &MI) const {
8048 MachineIRBuilder &B = Helper.MIRBuilder;
8049 MachineRegisterInfo &MRI = *B.getMRI();
8050 if (!ST.hasTrapHandler() ||
8051 ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
8052 return legalizeTrapEndpgm(Helper, MI);
8053
8054 return ST.supportsGetDoorbellID() ? legalizeTrapHsa(Helper, MI)
8056}
8057
8059 MachineInstr &MI) const {
8060 MachineIRBuilder &B = Helper.MIRBuilder;
8061 GISelChangeObserver &Observer = Helper.Observer;
8062 const DebugLoc &DL = MI.getDebugLoc();
8063 MachineBasicBlock &BB = B.getMBB();
8064 MachineFunction *MF = BB.getParent();
8065
8066 if (BB.succ_empty() && std::next(MI.getIterator()) == BB.end()) {
8067 BuildMI(BB, BB.end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
8068 .addImm(0);
8069 MI.eraseFromParent();
8070 return true;
8071 }
8072
8073 // We need a block split to make the real endpgm a terminator. We also don't
8074 // want to break phis in successor blocks, so we can't just delete to the
8075 // end of the block.
8076 // An instruction's parent block is part of its CSE profile, so notify
8077 // observers about the instructions moved by the split.
8079 MachineBasicBlock::iterator SplitPoint(&MI);
8080 ++SplitPoint;
8081 for (auto I = SplitPoint, E = BB.end(); I != E; ++I) {
8082 Observer.changingInstr(*I);
8083 MovedInstrs.push_back(&*I);
8084 }
8085 BB.splitAt(MI, false /*UpdateLiveIns*/);
8086 for (MachineInstr *MovedMI : MovedInstrs)
8087 Observer.changedInstr(*MovedMI);
8089 MF->push_back(TrapBB);
8090 BuildMI(*TrapBB, TrapBB->end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
8091 .addImm(0);
8092 BuildMI(BB, &MI, DL, B.getTII().get(AMDGPU::S_CBRANCH_EXECNZ))
8093 .addMBB(TrapBB);
8094
8095 BB.addSuccessor(TrapBB);
8096 MI.eraseFromParent();
8097 return true;
8098}
8099
8102 MachineFunction &MF = B.getMF();
8103 const LLT I64 = LLT::integer(64);
8104
8105 Register SGPR01(AMDGPU::SGPR0_SGPR1);
8106 // For code object version 5, queue_ptr is passed through implicit kernarg.
8111 uint64_t Offset =
8112 ST.getTargetLowering()->getImplicitParameterOffset(B.getMF(), Param);
8113
8114 Register KernargPtrReg = MRI.createGenericVirtualRegister(
8116
8117 if (!loadInputValue(KernargPtrReg, B,
8119 return false;
8120
8121 // TODO: can we be smarter about machine pointer info?
8124 PtrInfo.getWithOffset(Offset),
8128
8129 // Pointer address
8130 Register LoadAddr = MRI.createGenericVirtualRegister(
8132 B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
8133 B.buildConstant(LLT::integer(64), Offset).getReg(0));
8134 // Load address
8135 Register Temp = B.buildLoad(I64, LoadAddr, *MMO).getReg(0);
8136 B.buildCopy(SGPR01, Temp);
8137 B.buildInstr(AMDGPU::S_TRAP)
8138 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
8139 .addReg(SGPR01, RegState::Implicit);
8140 MI.eraseFromParent();
8141 return true;
8142 }
8143
8144 // Pass queue pointer to trap handler as input, and insert trap instruction
8145 // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
8146 Register LiveIn =
8147 MRI.createGenericVirtualRegister(LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
8149 return false;
8150
8151 B.buildCopy(SGPR01, LiveIn);
8152 B.buildInstr(AMDGPU::S_TRAP)
8153 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
8154 .addReg(SGPR01, RegState::Implicit);
8155
8156 MI.eraseFromParent();
8157 return true;
8158}
8159
8161 MachineInstr &MI) const {
8162 MachineIRBuilder &B = Helper.MIRBuilder;
8163 MachineRegisterInfo &MRI = *B.getMRI();
8164
8165 // We need to simulate the 's_trap 2' instruction on targets that run in
8166 // PRIV=1 (where it is treated as a nop).
8167 if (ST.hasPrivEnabledTrap2NopBug()) {
8168 // insertSimulatedTrap may split the block after MI. An instruction's
8169 // parent block is part of its CSE profile, so notify observers about the
8170 // instructions that may be moved by the split.
8171 GISelChangeObserver &Observer = Helper.Observer;
8172 MachineBasicBlock &MBB = B.getMBB();
8174 for (MachineInstr &I :
8175 make_range(std::next(MachineBasicBlock::iterator(MI)), MBB.end())) {
8176 Observer.changingInstr(I);
8177 MovedInstrs.push_back(&I);
8178 }
8179 ST.getInstrInfo()->insertSimulatedTrap(MRI, MBB, MI, MI.getDebugLoc());
8180 for (MachineInstr *MovedMI : MovedInstrs)
8181 Observer.changedInstr(*MovedMI);
8182 MI.eraseFromParent();
8183 return true;
8184 }
8185
8186 B.buildInstr(AMDGPU::S_TRAP)
8187 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
8188 MI.eraseFromParent();
8189 return true;
8190}
8191
8194 MachineIRBuilder &B) const {
8195 // Is non-HSA path or trap-handler disabled? Then, report a warning
8196 // accordingly
8197 if (!ST.hasTrapHandler() ||
8198 ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
8199 Function &Fn = B.getMF().getFunction();
8201 Fn, "debugtrap handler not supported", MI.getDebugLoc(), DS_Warning));
8202 } else {
8203 // Insert debug-trap instruction
8204 B.buildInstr(AMDGPU::S_TRAP)
8205 .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap));
8206 }
8207
8208 MI.eraseFromParent();
8209 return true;
8210}
8211
8213 MachineInstr &MI, MachineIRBuilder &B) const {
8214 MachineRegisterInfo &MRI = *B.getMRI();
8215 const LLT I16 = LLT::integer(16);
8216 const LLT I32 = LLT::integer(32);
8217 const LLT V2I16 = LLT::fixed_vector(2, I16);
8218 const LLT V3I32 = LLT::fixed_vector(3, I32);
8219 const LLT V3I16 = LLT::fixed_vector(3, I16);
8220
8221 Register DstReg = MI.getOperand(0).getReg();
8222 Register NodePtr = MI.getOperand(2).getReg();
8223 Register RayExtent = MI.getOperand(3).getReg();
8224 Register RayOrigin = MI.getOperand(4).getReg();
8225 Register RayDir = MI.getOperand(5).getReg();
8226 Register RayInvDir = MI.getOperand(6).getReg();
8227 Register TDescr = MI.getOperand(7).getReg();
8228
8229 RayExtent = B.buildBitcast(I32, RayExtent).getReg(0);
8230
8231 const bool IsGFX11 = AMDGPU::isGFX11(ST);
8232 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(ST);
8233 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(ST);
8234 const bool IsA16 = MRI.getType(RayDir).getElementType().getSizeInBits() == 16;
8235 const bool Is64 = MRI.getType(NodePtr).getSizeInBits() == 64;
8236 const unsigned NumVDataDwords = 4;
8237 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
8238 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
8239 const bool UseNSA =
8240 IsGFX12Plus || (ST.hasNSAEncoding() && NumVAddrs <= ST.getNSAMaxSize());
8241
8242 const unsigned BaseOpcodes[2][2] = {
8243 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
8244 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
8245 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
8246 int Opcode;
8247 if (UseNSA) {
8248 Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
8249 IsGFX12Plus ? AMDGPU::MIMGEncGfx12
8250 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
8251 : AMDGPU::MIMGEncGfx10NSA,
8252 NumVDataDwords, NumVAddrDwords);
8253 } else {
8254 assert(!IsGFX12Plus);
8255 Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
8256 IsGFX11 ? AMDGPU::MIMGEncGfx11Default
8257 : AMDGPU::MIMGEncGfx10Default,
8258 NumVDataDwords, NumVAddrDwords);
8259 }
8260 assert(Opcode != -1);
8261
8263 if (UseNSA && IsGFX11Plus) {
8264 auto packLanes = [&Ops, &I32, &V3I32, &B](Register Src) {
8265 auto SrcInt = B.buildBitcast(V3I32, Src);
8266 auto Unmerge = B.buildUnmerge({I32, I32, I32}, SrcInt);
8267 auto Merged = B.buildMergeLikeInstr(
8268 V3I32, {Unmerge.getReg(0), Unmerge.getReg(1), Unmerge.getReg(2)});
8269 Ops.push_back(Merged.getReg(0));
8270 };
8271
8272 Ops.push_back(NodePtr);
8273 Ops.push_back(RayExtent);
8274 packLanes(RayOrigin);
8275
8276 if (IsA16) {
8277 auto UnmergeRayDir =
8278 B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayDir));
8279 auto UnmergeRayInvDir =
8280 B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayInvDir));
8281 auto MergedDir = B.buildMergeLikeInstr(
8282 V3I32,
8283 {B.buildBitcast(
8284 I32, B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(0),
8285 UnmergeRayDir.getReg(0)}))
8286 .getReg(0),
8287 B.buildBitcast(
8288 I32, B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(1),
8289 UnmergeRayDir.getReg(1)}))
8290 .getReg(0),
8291 B.buildBitcast(
8292 I32, B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(2),
8293 UnmergeRayDir.getReg(2)}))
8294 .getReg(0)});
8295 Ops.push_back(MergedDir.getReg(0));
8296 } else {
8297 packLanes(RayDir);
8298 packLanes(RayInvDir);
8299 }
8300 } else {
8301 if (Is64) {
8302 auto Unmerge = B.buildUnmerge({I32, I32}, NodePtr);
8303 Ops.push_back(Unmerge.getReg(0));
8304 Ops.push_back(Unmerge.getReg(1));
8305 } else {
8306 Ops.push_back(NodePtr);
8307 }
8308 Ops.push_back(RayExtent);
8309
8310 auto packLanes = [&Ops, &I32, &V3I32, &B](Register Src) {
8311 auto SrcInt = B.buildBitcast(V3I32, Src);
8312 auto Unmerge = B.buildUnmerge({I32, I32, I32}, SrcInt);
8313 Ops.push_back(Unmerge.getReg(0));
8314 Ops.push_back(Unmerge.getReg(1));
8315 Ops.push_back(Unmerge.getReg(2));
8316 };
8317
8318 packLanes(RayOrigin);
8319 if (IsA16) {
8320 auto UnmergeRayDir =
8321 B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayDir));
8322 auto UnmergeRayInvDir =
8323 B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayInvDir));
8324 Register R1 = MRI.createGenericVirtualRegister(I32);
8325 Register R2 = MRI.createGenericVirtualRegister(I32);
8326 Register R3 = MRI.createGenericVirtualRegister(I32);
8327 B.buildMergeLikeInstr(R1,
8328 {UnmergeRayDir.getReg(0), UnmergeRayDir.getReg(1)});
8329 B.buildMergeLikeInstr(
8330 R2, {UnmergeRayDir.getReg(2), UnmergeRayInvDir.getReg(0)});
8331 B.buildMergeLikeInstr(
8332 R3, {UnmergeRayInvDir.getReg(1), UnmergeRayInvDir.getReg(2)});
8333 Ops.push_back(R1);
8334 Ops.push_back(R2);
8335 Ops.push_back(R3);
8336 } else {
8337 packLanes(RayDir);
8338 packLanes(RayInvDir);
8339 }
8340 }
8341
8342 if (!UseNSA) {
8343 // Build a single vector containing all the operands so far prepared.
8344 LLT OpTy = LLT::fixed_vector(Ops.size(), I32);
8345 Register MergedOps = B.buildMergeLikeInstr(OpTy, Ops).getReg(0);
8346 Ops.clear();
8347 Ops.push_back(MergedOps);
8348 }
8349
8350 auto MIB = B.buildInstr(AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY)
8351 .addDef(DstReg)
8352 .addImm(Opcode);
8353
8354 for (Register R : Ops) {
8355 MIB.addUse(R);
8356 }
8357
8358 MIB.addUse(TDescr)
8359 .addImm(IsA16 ? 1 : 0)
8360 .cloneMemRefs(MI);
8361
8362 MI.eraseFromParent();
8363 return true;
8364}
8365
8367 MachineInstr &MI, MachineIRBuilder &B) const {
8368 const LLT I32 = LLT::integer(32);
8369 const LLT V2I32 = LLT::fixed_vector(2, I32);
8370
8371 Register DstReg = MI.getOperand(0).getReg();
8372 Register DstOrigin = MI.getOperand(1).getReg();
8373 Register DstDir = MI.getOperand(2).getReg();
8374 Register NodePtr = MI.getOperand(4).getReg();
8375 Register RayExtent = MI.getOperand(5).getReg();
8376 Register InstanceMask = MI.getOperand(6).getReg();
8377 Register RayOrigin = MI.getOperand(7).getReg();
8378 Register RayDir = MI.getOperand(8).getReg();
8379 Register Offsets = MI.getOperand(9).getReg();
8380 Register TDescr = MI.getOperand(10).getReg();
8381
8382 bool IsBVH8 = cast<GIntrinsic>(MI).getIntrinsicID() ==
8383 Intrinsic::amdgcn_image_bvh8_intersect_ray;
8384 const unsigned NumVDataDwords = 10;
8385 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
8386 int Opcode = AMDGPU::getMIMGOpcode(
8387 IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
8388 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
8389 AMDGPU::MIMGEncGfx12, NumVDataDwords, NumVAddrDwords);
8390 assert(Opcode != -1);
8391
8392 auto RayExtentInstanceMaskVec =
8393 B.buildMergeLikeInstr(V2I32, {B.buildBitcast(I32, RayExtent),
8394 B.buildAnyExt(I32, InstanceMask)});
8395
8396 B.buildInstr(IsBVH8 ? AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY
8397 : AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY)
8398 .addDef(DstReg)
8399 .addDef(DstOrigin)
8400 .addDef(DstDir)
8401 .addImm(Opcode)
8402 .addUse(NodePtr)
8403 .addUse(RayExtentInstanceMaskVec.getReg(0))
8404 .addUse(RayOrigin)
8405 .addUse(RayDir)
8406 .addUse(Offsets)
8407 .addUse(TDescr)
8408 .cloneMemRefs(MI);
8409
8410 MI.eraseFromParent();
8411 return true;
8412}
8413
8415 MachineIRBuilder &B) const {
8416 // Entry functions still hold the SP_REG placeholder until finalizeLowering.
8417 const SIMachineFunctionInfo *Info =
8418 B.getMF().getInfo<SIMachineFunctionInfo>();
8419 Register StackPtr = Info->getStackPtrOffsetReg();
8420 Register DstReg = MI.getOperand(0).getReg();
8421 B.buildInstr(AMDGPU::G_AMDGPU_WAVE_ADDRESS, {DstReg}, {StackPtr});
8422 MI.eraseFromParent();
8423 return true;
8424}
8425
8427 MachineIRBuilder &B) const {
8428 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
8429 if (!ST.hasArchitectedSGPRs())
8430 return false;
8431 LLT I32 = LLT::integer(32);
8432 Register DstReg = MI.getOperand(0).getReg();
8433 auto TTMP8 = B.buildCopy(I32, Register(AMDGPU::TTMP8));
8434 auto LSB = B.buildConstant(I32, 25);
8435 auto Width = B.buildConstant(I32, 5);
8436 B.buildUbfx(DstReg, TTMP8, LSB, Width);
8437 MI.eraseFromParent();
8438 return true;
8439}
8440
8443 AMDGPU::Hwreg::Id HwReg,
8444 unsigned LowBit,
8445 unsigned Width) const {
8446 MachineRegisterInfo &MRI = *B.getMRI();
8447 Register DstReg = MI.getOperand(0).getReg();
8448 Register Result = MRI.createVirtualRegister(
8449 {&AMDGPU::SReg_32RegClass, MRI.getType(DstReg)});
8450 B.buildInstr(AMDGPU::S_GETREG_B32_const)
8451 .addDef(Result)
8452 .addImm(AMDGPU::Hwreg::HwregEncoding::encode(HwReg, LowBit, Width));
8453 B.buildCopy(DstReg, Result);
8454 MI.eraseFromParent();
8455 return true;
8456}
8457
8458static constexpr unsigned FPEnvModeBitField =
8460
8461static constexpr unsigned FPEnvTrapBitField =
8463
8466 MachineIRBuilder &B) const {
8467 const LLT I32 = LLT::integer(32);
8468 const LLT I64 = LLT::integer(64);
8469 Register Src = MI.getOperand(0).getReg();
8470 if (MRI.getType(Src) != I64)
8471 return false;
8472
8473 auto ModeReg =
8474 B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {I32},
8475 /*HasSideEffects=*/true, /*isConvergent=*/false)
8476 .addImm(FPEnvModeBitField);
8477 auto TrapReg =
8478 B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {I32},
8479 /*HasSideEffects=*/true, /*isConvergent=*/false)
8480 .addImm(FPEnvTrapBitField);
8481 B.buildMergeLikeInstr(Src, {ModeReg, TrapReg});
8482 MI.eraseFromParent();
8483 return true;
8484}
8485
8488 MachineIRBuilder &B) const {
8489 const LLT I32 = LLT::integer(32);
8490 const LLT I64 = LLT::integer(64);
8491 Register Src = MI.getOperand(0).getReg();
8492 if (MRI.getType(Src) != I64)
8493 return false;
8494
8495 auto Unmerge = B.buildUnmerge({I32, I32}, MI.getOperand(0));
8496 B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef<DstOp>(),
8497 /*HasSideEffects=*/true, /*isConvergent=*/false)
8498 .addImm(static_cast<int16_t>(FPEnvModeBitField))
8499 .addReg(Unmerge.getReg(0));
8500 B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef<DstOp>(),
8501 /*HasSideEffects=*/true, /*isConvergent=*/false)
8502 .addImm(static_cast<int16_t>(FPEnvTrapBitField))
8503 .addReg(Unmerge.getReg(1));
8504 MI.eraseFromParent();
8505 return true;
8506}
8507
8509 MachineInstr &MI) const {
8510 MachineIRBuilder &B = Helper.MIRBuilder;
8511 MachineRegisterInfo &MRI = *B.getMRI();
8512
8513 // Replace the use G_BRCOND with the exec manipulate and branch pseudos.
8514 auto IntrID = cast<GIntrinsic>(MI).getIntrinsicID();
8515
8516 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
8518 switch (BufInfo->Kind) {
8519 case AMDGPU::BufferKindSLoad:
8520 return legalizeSBufferLoad(Helper, MI);
8521 case AMDGPU::BufferKindPrefetch:
8522 return legalizeSBufferPrefetch(Helper, MI);
8523 case AMDGPU::BufferKindStore:
8524 return legalizeBufferStore(MI, Helper, *BufInfo);
8525 case AMDGPU::BufferKindLoad:
8526 case AMDGPU::BufferKindAtomicLoad:
8527 return legalizeBufferLoad(MI, Helper, *BufInfo);
8528 case AMDGPU::BufferKindAtomicRMW:
8529 case AMDGPU::BufferKindAtomicCmpSwap:
8530 return legalizeBufferAtomic(MI, B, *BufInfo);
8531 case AMDGPU::BufferKindLoadToLDS:
8532 // Selected directly.
8533 return true;
8534 }
8535 llvm_unreachable("unhandled buffer intrinsic kind");
8536 }
8537
8538 switch (IntrID) {
8539 case Intrinsic::sponentry:
8540 if (B.getMF().getInfo<SIMachineFunctionInfo>()->isBottomOfStack()) {
8541 // FIXME: The imported pattern checks for i32 instead of p5; if we fix
8542 // that we can remove this cast.
8543 const LLT I32 = LLT::integer(32);
8544 Register TmpReg = MRI.createGenericVirtualRegister(I32);
8545 B.buildInstr(AMDGPU::G_AMDGPU_SPONENTRY).addDef(TmpReg);
8546
8547 Register DstReg = MI.getOperand(0).getReg();
8548 B.buildIntToPtr(DstReg, TmpReg);
8549 MI.eraseFromParent();
8550 } else {
8551 int FI = B.getMF().getFrameInfo().CreateFixedObject(
8552 1, 0, /*IsImmutable=*/false);
8553 B.buildFrameIndex(MI.getOperand(0), FI);
8554 MI.eraseFromParent();
8555 }
8556 return true;
8557 case Intrinsic::amdgcn_if:
8558 case Intrinsic::amdgcn_else: {
8559 MachineInstr *Br = nullptr;
8560 MachineBasicBlock *UncondBrTarget = nullptr;
8561 bool Negated = false;
8562 if (MachineInstr *BrCond =
8563 verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
8564 const SIRegisterInfo *TRI
8565 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
8566
8567 Register Def = MI.getOperand(1).getReg();
8568 Register Use = MI.getOperand(3).getReg();
8569 const TargetRegisterClass *WaveMaskRC = TRI->getWaveMaskRegClass();
8570 Register NewDef =
8571 MRI.createVirtualRegister({WaveMaskRC, MRI.getType(Def)});
8572 Register NewUse =
8573 MRI.createVirtualRegister({WaveMaskRC, MRI.getType(Use)});
8574
8575 MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
8576
8577 if (Negated)
8578 std::swap(CondBrTarget, UncondBrTarget);
8579
8580 B.setInsertPt(B.getMBB(), BrCond->getIterator());
8581 B.buildCopy(NewUse, Use);
8582 if (IntrID == Intrinsic::amdgcn_if) {
8583 B.buildInstr(AMDGPU::SI_IF)
8584 .addDef(NewDef)
8585 .addUse(NewUse)
8586 .addMBB(UncondBrTarget)
8587 .setOperandDead(4); // implicit-def $scc
8588 } else {
8589 B.buildInstr(AMDGPU::SI_ELSE)
8590 .addDef(NewDef)
8591 .addUse(NewUse)
8592 .addMBB(UncondBrTarget)
8593 .setOperandDead(4); // implicit-def $scc
8594 }
8595
8596 if (Br) {
8597 Br->getOperand(0).setMBB(CondBrTarget);
8598 } else {
8599 // The IRTranslator skips inserting the G_BR for fallthrough cases, but
8600 // since we're swapping branch targets it needs to be reinserted.
8601 // FIXME: IRTranslator should probably not do this
8602 B.buildBr(*CondBrTarget);
8603 }
8604
8605 MI.eraseFromParent();
8606 BrCond->eraseFromParent();
8607 // SI_IF and SI_ELSE are terminators, so replace uses of Def rather than
8608 // defining Def with a following copy.
8609 Helper.Observer.changingAllUsesOfReg(MRI, Def);
8610 MRI.replaceRegWith(Def, NewDef);
8612 return true;
8613 }
8614
8615 return false;
8616 }
8617 case Intrinsic::amdgcn_loop: {
8618 MachineInstr *Br = nullptr;
8619 MachineBasicBlock *UncondBrTarget = nullptr;
8620 bool Negated = false;
8621 if (MachineInstr *BrCond =
8622 verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
8623 const SIRegisterInfo *TRI
8624 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
8625
8626 MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
8627 Register Reg = MI.getOperand(2).getReg();
8628 Register NewReg = MRI.createVirtualRegister(
8629 {TRI->getWaveMaskRegClass(), MRI.getType(Reg)});
8630
8631 if (Negated)
8632 std::swap(CondBrTarget, UncondBrTarget);
8633
8634 B.setInsertPt(B.getMBB(), BrCond->getIterator());
8635 B.buildCopy(NewReg, Reg);
8636 B.buildInstr(AMDGPU::SI_LOOP)
8637 .addUse(NewReg)
8638 .addMBB(UncondBrTarget)
8639 .setOperandDead(3); // implicit-def $scc
8640
8641 if (Br)
8642 Br->getOperand(0).setMBB(CondBrTarget);
8643 else
8644 B.buildBr(*CondBrTarget);
8645
8646 MI.eraseFromParent();
8647 BrCond->eraseFromParent();
8648 return true;
8649 }
8650
8651 return false;
8652 }
8653 case Intrinsic::amdgcn_wave_reduce_min:
8654 case Intrinsic::amdgcn_wave_reduce_umin:
8655 case Intrinsic::amdgcn_wave_reduce_fmin:
8656 case Intrinsic::amdgcn_wave_reduce_max:
8657 case Intrinsic::amdgcn_wave_reduce_umax:
8658 case Intrinsic::amdgcn_wave_reduce_fmax:
8659 case Intrinsic::amdgcn_wave_reduce_add:
8660 case Intrinsic::amdgcn_wave_reduce_fadd:
8661 case Intrinsic::amdgcn_wave_reduce_sub:
8662 case Intrinsic::amdgcn_wave_reduce_fsub:
8663 case Intrinsic::amdgcn_wave_reduce_and:
8664 case Intrinsic::amdgcn_wave_reduce_or:
8665 case Intrinsic::amdgcn_wave_reduce_xor: {
8666 Register SrcReg = MI.getOperand(2).getReg();
8667 if (MRI.getType(SrcReg).getSizeInBits() != 16)
8668 return true;
8669 Register DstReg = MI.getOperand(0).getReg();
8670 bool IsFPOp = IntrID == Intrinsic::amdgcn_wave_reduce_fmin ||
8671 IntrID == Intrinsic::amdgcn_wave_reduce_fmax ||
8672 IntrID == Intrinsic::amdgcn_wave_reduce_fadd ||
8673 IntrID == Intrinsic::amdgcn_wave_reduce_fsub;
8674 bool NeedsSignExt = IntrID == Intrinsic::amdgcn_wave_reduce_min ||
8675 IntrID == Intrinsic::amdgcn_wave_reduce_max ||
8676 IntrID == Intrinsic::amdgcn_wave_reduce_add ||
8677 IntrID == Intrinsic::amdgcn_wave_reduce_sub;
8678 auto Ext = IsFPOp ? B.buildFPExt(F32, SrcReg)
8679 : NeedsSignExt ? B.buildSExt(LLT::integer(32), SrcReg)
8680 : B.buildZExt(LLT::integer(32), SrcReg);
8681 auto NewDst =
8682 MRI.createGenericVirtualRegister(IsFPOp ? F32 : LLT::integer(32));
8683 B.buildIntrinsic(IntrID, ArrayRef<Register>{NewDst},
8684 /*hasSideEffects=*/false, /*isConvergent=*/true)
8685 .addUse(Ext.getReg(0))
8686 .addImm(MI.getOperand(3).getImm()); // strategy
8687 if (IsFPOp)
8688 B.buildFPTrunc(DstReg, NewDst);
8689 else
8690 B.buildTrunc(DstReg, NewDst);
8691 MI.eraseFromParent();
8692 return true;
8693 }
8694 case Intrinsic::amdgcn_make_buffer_rsrc:
8696 case Intrinsic::amdgcn_kernarg_segment_ptr:
8697 if (!AMDGPU::isKernel(B.getMF().getFunction())) {
8698 // This only makes sense to call in a kernel, so just lower to null.
8699 B.buildConstant(MI.getOperand(0).getReg(), 0);
8700 MI.eraseFromParent();
8701 return true;
8702 }
8703
8706 case Intrinsic::amdgcn_implicitarg_ptr:
8707 return legalizeImplicitArgPtr(MI, MRI, B);
8708 case Intrinsic::amdgcn_workitem_id_x:
8709 return legalizeWorkitemIDIntrinsic(MI, MRI, B, 0,
8711 case Intrinsic::amdgcn_workitem_id_y:
8712 return legalizeWorkitemIDIntrinsic(MI, MRI, B, 1,
8714 case Intrinsic::amdgcn_workitem_id_z:
8715 return legalizeWorkitemIDIntrinsic(MI, MRI, B, 2,
8717 case Intrinsic::amdgcn_workgroup_id_x:
8718 return legalizeWorkGroupId(
8722 case Intrinsic::amdgcn_workgroup_id_y:
8723 return legalizeWorkGroupId(
8727 case Intrinsic::amdgcn_workgroup_id_z:
8728 return legalizeWorkGroupId(
8732 case Intrinsic::amdgcn_cluster_id_x:
8733 return ST.hasClusters() &&
8736 case Intrinsic::amdgcn_cluster_id_y:
8737 return ST.hasClusters() &&
8740 case Intrinsic::amdgcn_cluster_id_z:
8741 return ST.hasClusters() &&
8744 case Intrinsic::amdgcn_cluster_workgroup_id_x:
8745 return ST.hasClusters() &&
8748 case Intrinsic::amdgcn_cluster_workgroup_id_y:
8749 return ST.hasClusters() &&
8752 case Intrinsic::amdgcn_cluster_workgroup_id_z:
8753 return ST.hasClusters() &&
8756 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
8757 return ST.hasClusters() &&
8759 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
8760 return ST.hasClusters() &&
8763 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
8764 return ST.hasClusters() &&
8767 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
8768 return ST.hasClusters() &&
8771 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
8772 return ST.hasClusters() &&
8774 MI, MRI, B,
8776 case Intrinsic::amdgcn_wave_id:
8777 return legalizeWaveID(MI, B);
8778 case Intrinsic::amdgcn_lds_kernel_id:
8781 case Intrinsic::amdgcn_dispatch_ptr:
8784 case Intrinsic::amdgcn_queue_ptr:
8787 case Intrinsic::amdgcn_implicit_buffer_ptr: {
8788 const Function &Fn = B.getMF().getFunction();
8789 const SIMachineFunctionInfo *MFI =
8790 B.getMF().getInfo<SIMachineFunctionInfo>();
8791 if (!MFI->getUserSGPRInfo().hasImplicitBufferPtr()) {
8792 const char *Error =
8793 ST.hasArchitectedFlatScratch()
8794 ? "intrinsic not supported on subtarget"
8795 : !ST.isMesa3DOS() ? "intrinsic not supported for OS"
8796 : "intrinsic not supported for calling convention";
8797 Fn.getContext().diagnose(
8798 DiagnosticInfoUnsupported(Fn, Error, MI.getDebugLoc()));
8799 B.buildUndef(MI.getOperand(0).getReg());
8800 MI.eraseFromParent();
8801 return true;
8802 }
8805 }
8806 case Intrinsic::amdgcn_dispatch_id:
8809 case Intrinsic::r600_read_ngroups_x:
8810 // TODO: Emit error for hsa
8813 case Intrinsic::r600_read_ngroups_y:
8816 case Intrinsic::r600_read_ngroups_z:
8819 case Intrinsic::r600_read_local_size_x:
8820 // TODO: Could insert G_ASSERT_ZEXT from i16
8822 case Intrinsic::r600_read_local_size_y:
8823 // TODO: Could insert G_ASSERT_ZEXT from i16
8825 // TODO: Could insert G_ASSERT_ZEXT from i16
8826 case Intrinsic::r600_read_local_size_z:
8829 case Intrinsic::amdgcn_fdiv_fast:
8830 return legalizeFDIVFastIntrin(MI, MRI, B);
8831 case Intrinsic::amdgcn_is_shared:
8833 case Intrinsic::amdgcn_is_private:
8835 case Intrinsic::amdgcn_wavefrontsize: {
8836 B.buildConstant(MI.getOperand(0), ST.getWavefrontSize());
8837 MI.eraseFromParent();
8838 return true;
8839 }
8840 case Intrinsic::amdgcn_rsq_clamp:
8842 case Intrinsic::amdgcn_image_bvh_intersect_ray:
8844 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
8845 case Intrinsic::amdgcn_image_bvh8_intersect_ray:
8847 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
8848 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
8849 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
8850 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
8851 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
8852 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
8853 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
8854 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
8855 Register Index = MI.getOperand(5).getReg();
8856 LLT I64 = LLT::integer(64);
8857 LLT IndexArgTy = MRI.getType(Index);
8858 if (IndexArgTy != I64) {
8859 auto NewIndex = IndexArgTy.isVector() ? B.buildBitcast(I64, Index)
8860 : B.buildAnyExt(I64, Index);
8861 MI.getOperand(5).setReg(NewIndex.getReg(0));
8862 }
8863 return true;
8864 }
8865 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
8866 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
8867 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
8868 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
8869 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
8870 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
8871 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
8872 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
8873 Register Index = MI.getOperand(5).getReg();
8874 LLT I32 = LLT::integer(32);
8875 if (MRI.getType(Index) != I32)
8876 MI.getOperand(5).setReg(B.buildAnyExt(I32, Index).getReg(0));
8877 return true;
8878 }
8879 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
8880 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
8881 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
8882 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
8883 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
8884 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
8885 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
8886 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
8887 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
8888 Register Index = MI.getOperand(7).getReg();
8889 LLT IdxTy = IntrID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
8890 ? LLT::integer(64)
8891 : LLT::integer(32);
8892 LLT IndexArgTy = MRI.getType(Index);
8893 if (IndexArgTy != IdxTy) {
8894 auto NewIndex = IndexArgTy.isVector() ? B.buildBitcast(IdxTy, Index)
8895 : B.buildAnyExt(IdxTy, Index);
8896 MI.getOperand(7).setReg(NewIndex.getReg(0));
8897 }
8898 return true;
8899 }
8900
8901 case Intrinsic::amdgcn_fmed3: {
8902 GISelChangeObserver &Observer = Helper.Observer;
8903
8904 // FIXME: This is to workaround the inability of tablegen match combiners to
8905 // match intrinsics in patterns.
8906 Observer.changingInstr(MI);
8907 MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_FMED3));
8908 MI.removeOperand(1);
8909 Observer.changedInstr(MI);
8910 return true;
8911 }
8912 case Intrinsic::amdgcn_readlane:
8913 case Intrinsic::amdgcn_writelane:
8914 case Intrinsic::amdgcn_readfirstlane:
8915 case Intrinsic::amdgcn_permlane16:
8916 case Intrinsic::amdgcn_permlanex16:
8917 case Intrinsic::amdgcn_permlane64:
8918 case Intrinsic::amdgcn_set_inactive:
8919 case Intrinsic::amdgcn_set_inactive_chain_arg:
8920 case Intrinsic::amdgcn_mov_dpp8:
8921 case Intrinsic::amdgcn_update_dpp:
8922 case Intrinsic::amdgcn_permlane_bcast:
8923 case Intrinsic::amdgcn_permlane_up:
8924 case Intrinsic::amdgcn_permlane_down:
8925 case Intrinsic::amdgcn_permlane_xor:
8926 return legalizeLaneOp(Helper, MI, IntrID);
8927 case Intrinsic::amdgcn_dead: {
8928 // TODO: Use poison instead of undef
8929 for (const MachineOperand &Def : MI.defs())
8930 B.buildUndef(Def);
8931 MI.eraseFromParent();
8932 return true;
8933 }
8934 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
8935 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
8936 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
8937 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8938 B.buildLoad(MI.getOperand(0), MI.getOperand(2), **MI.memoperands_begin());
8939 MI.eraseFromParent();
8940 return true;
8941 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
8942 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
8943 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
8944 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8945 B.buildStore(MI.getOperand(2), MI.getOperand(1), **MI.memoperands_begin());
8946 MI.eraseFromParent();
8947 return true;
8948 case Intrinsic::amdgcn_av_load_b8:
8949 case Intrinsic::amdgcn_av_load_b16:
8950 case Intrinsic::amdgcn_av_load_b32:
8951 case Intrinsic::amdgcn_av_load_b64:
8952 case Intrinsic::amdgcn_av_load_b128:
8953 case Intrinsic::amdgcn_av_store_b8:
8954 case Intrinsic::amdgcn_av_store_b16:
8955 case Intrinsic::amdgcn_av_store_b32:
8956 case Intrinsic::amdgcn_av_store_b64:
8957 case Intrinsic::amdgcn_av_store_b128: {
8958 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8959 if (MI.getNumExplicitDefs())
8960 B.buildLoad(MI.getOperand(0), MI.getOperand(2), **MI.memoperands_begin());
8961 else
8962 B.buildStore(MI.getOperand(2), MI.getOperand(1),
8963 **MI.memoperands_begin());
8964 MI.eraseFromParent();
8965 return true;
8966 }
8967 case Intrinsic::amdgcn_flat_load_monitor_b32:
8968 case Intrinsic::amdgcn_flat_load_monitor_b64:
8969 case Intrinsic::amdgcn_flat_load_monitor_b128:
8970 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8971 B.buildInstr(AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR)
8972 .add(MI.getOperand(0))
8973 .add(MI.getOperand(2))
8974 .addMemOperand(*MI.memoperands_begin());
8975 MI.eraseFromParent();
8976 return true;
8977 case Intrinsic::amdgcn_global_load_monitor_b32:
8978 case Intrinsic::amdgcn_global_load_monitor_b64:
8979 case Intrinsic::amdgcn_global_load_monitor_b128:
8980 assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
8981 B.buildInstr(AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR)
8982 .add(MI.getOperand(0))
8983 .add(MI.getOperand(2))
8984 .addMemOperand(*MI.memoperands_begin());
8985 MI.eraseFromParent();
8986 return true;
8987 default: {
8988 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
8990 return legalizeImageIntrinsic(MI, B, Helper.Observer, ImageDimIntr);
8991 return true;
8992 }
8993 }
8994
8995 return true;
8996}
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
unsigned NumParts
static unsigned getIntrinsicID(const SDNode *N)
const TargetLowering & TLI
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU address space definition.
uint64_t uint64_t
static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL, SelectionDAG &DAG)
static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X, SDValue Y, SDValue C, SDNodeFlags Flags=SDNodeFlags())
static bool valueIsKnownNeverF32Denorm(SDValue Src)
Return true if it's known that Src can never be an f32 denormal value.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI, SmallVectorImpl< Register > &PackedAddrs, unsigned ArgOffset, const AMDGPU::ImageDimIntrinsicInfo *Intr, bool IsA16, bool IsG16)
Turn a set of f16 typed registers in AddrRegs into a dword sized vector with f16 typed elements.
static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID)
static LLT getBufferRsrcScalarType(const LLT Ty)
static LegalityPredicate isIllegalRegisterType(const GCNSubtarget &ST, unsigned TypeIdx)
constexpr LLT F16
static MachineInstrBuilder buildExp(MachineIRBuilder &B, const DstOp &Dst, const SrcOp &Src, unsigned Flags)
static bool needsDenormHandlingF32(const MachineFunction &MF, Register Src, unsigned Flags)
constexpr std::initializer_list< LLT > AllVectors
static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx)
static LegalityPredicate isSmallOddVector(unsigned TypeIdx)
static LegalizeMutation oneMoreElement(unsigned TypeIdx)
constexpr LLT F64
static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size)
constexpr LLT V2S8
static bool allowApproxFunc(const MachineFunction &MF, unsigned Flags)
constexpr LLT V4S128
constexpr LLT S16
constexpr LLT S1
constexpr LLT V2F32
static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty, const LLT MemTy)
Return true if a load or store of the type should be lowered with a bitcast to a different type.
constexpr LLT S1024
static constexpr unsigned FPEnvModeBitField
constexpr LLT V7S64
static LegalizeMutation getScalarTypeFromMemDesc(unsigned TypeIdx)
static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size)
static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy, uint64_t AlignInBits, unsigned AddrSpace, unsigned Opcode)
Return true if we should legalize a load by widening an odd sized memory access up to the alignment.
static bool isRegisterVectorElementType(LLT EltTy)
static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx)
static LegalityPredicate isWideVec16(unsigned TypeIdx)
constexpr std::initializer_list< LLT > AllScalarTypes
static LegalityPredicate isTruncStoreToSizePowerOf2(unsigned TypeIdx)
constexpr LLT V2S16
constexpr LLT V8S16
constexpr LLT V9S32
constexpr std::initializer_list< LLT > AllS32Vectors
constexpr LLT S224
static LegalizeMutation moreElementsToNextExistingRegClass(unsigned TypeIdx)
constexpr LLT S512
static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize)
constexpr LLT MaxScalar
static Register castBufferRsrcToV4I32(Register Pointer, MachineIRBuilder &B)
Cast a buffer resource (an address space 8 pointer) into a 4xi32, which is the form in which the valu...
constexpr LLT V11S32
static bool isRegisterClassType(const GCNSubtarget &ST, LLT Ty)
constexpr LLT V6S64
constexpr LLT V2S64
static std::pair< Register, Register > emitReciprocalU64(MachineIRBuilder &B, Register Val)
static LLT getBitcastRegisterType(const LLT Ty)
static LLT getBufferRsrcRegisterType(const LLT Ty)
constexpr LLT S32
constexpr LLT V2F16
static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx)
static Register stripAnySourceMods(Register OrigSrc, MachineRegisterInfo &MRI)
constexpr LLT V8S32
constexpr LLT V2BF16
constexpr LLT S192
static LLT castBufferRsrcFromV4I32(MachineInstr &MI, MachineIRBuilder &B, MachineRegisterInfo &MRI, unsigned Idx)
Mutates IR (typicaly a load instruction) to use a <4 x s32> as the initial type of the operand idx an...
static bool replaceWithConstant(MachineIRBuilder &B, MachineInstr &MI, int64_t C)
static constexpr unsigned SPDenormModeBitField
constexpr LLT F32
static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS, bool IsLoad, bool IsAtomic)
constexpr LLT V6S32
static bool isLoadStoreSizeLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
constexpr LLT S160
static MachineInstr * verifyCFIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineInstr *&Br, MachineBasicBlock *&UncondBrTarget, bool &Negated)
constexpr LLT V4S16
constexpr LLT V2S128
constexpr LLT V10S16
static LegalityPredicate numElementsNotEven(unsigned TypeIdx)
constexpr LLT V4S32
constexpr LLT V3S32
static LLT getVGPRLoadStorePartType(LLT Ty)
constexpr LLT V6S16
constexpr std::initializer_list< LLT > AllS64Vectors
constexpr LLT S256
constexpr LLT V2F64
static bool loadStoreBitcastWorkaround(const GCNSubtarget &ST, const LLT Ty)
static void castBufferRsrcArgToV4I32(MachineInstr &MI, MachineIRBuilder &B, unsigned Idx)
constexpr LLT V4S64
static constexpr unsigned FPEnvTrapBitField
constexpr LLT V10S32
constexpr LLT V16S32
static constexpr unsigned MaxRegisterSize
static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, GLoadStore &LdSt)
Lower a whole-dword load or store on AMDGPUAS::VGPR into G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE.
constexpr LLT V7S32
constexpr LLT S96
constexpr LLT V12S16
constexpr LLT V16S64
constexpr LLT BF16
static bool isRegisterSize(const GCNSubtarget &ST, unsigned Size)
static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx)
static bool hasBufferRsrcWorkaround(const LLT Ty)
constexpr LLT V32S32
static void toggleSPDenormMode(bool Enable, MachineIRBuilder &B, const GCNSubtarget &ST, SIModeRegisterDefaults Mode)
constexpr LLT S64
constexpr std::initializer_list< LLT > AllS16Vectors
static LLT widenToNextPowerOf2(LLT Ty)
static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI)
constexpr LLT V16S16
static void convertImageAddrToPacked(MachineIRBuilder &B, MachineInstr &MI, int DimIdx, int NumVAddrs)
Convert from separate vaddr components to a single vector address register, and replace the remaining...
static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx)
constexpr LLT V5S32
constexpr LLT V5S64
constexpr LLT V3S64
static LLT getPow2VectorType(LLT Ty)
static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc, Register VIndex, Register VOffset, Register SOffset, unsigned ImmOffset, unsigned Format, unsigned AuxiliaryData, MachineMemOperand *MMO, bool IsTyped, bool HasVIndex, MachineIRBuilder &B)
constexpr LLT V8S64
static LLT getPow2ScalarType(LLT Ty)
static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx)
constexpr LLT V2S32
static bool isRegisterVectorType(LLT Ty)
constexpr LLT V12S32
constexpr LLT S128
static LegalityPredicate sizeIsMultipleOf32(unsigned TypeIdx)
static void buildTFEBufferLoad(unsigned Opc, ArrayRef< Register > ValueDsts, Register StatusDst, Register RSrc, Register VIndex, Register VOffset, Register SOffset, unsigned ImmOffset, unsigned Format, unsigned AuxiliaryData, MachineMemOperand *MMO, bool IsTyped, bool HasVIndex, MachineIRBuilder &B)
constexpr LLT S8
static bool isRegisterType(const GCNSubtarget &ST, LLT Ty)
static bool isKnownNonNull(Register Val, MachineRegisterInfo &MRI, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
Return true if the value is a known valid address, such that a null check is not necessary.
This file declares the targeting of the Machinelegalizer class for AMDGPU.
The AMDGPU TargetMachine interface definition for hw codegen targets.
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
bpf unwind lower
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static Error unsupported(const char *Str, const Triple &T)
Definition MachO.cpp:79
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
@ Enable
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
IRTranslator LLVM IR MI
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Interface for Targets to specify which operations they can successfully select and how the others sho...
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
Register Reg
Register const TargetRegisterInfo * TRI
#define R2(n)
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define T
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define P(N)
ppc ctr loops verify
R600 Clause Merge
SmallVector< MachineOperand, 4 > Cond
int64_t C1
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
int64_t C0
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
#define CH(x, y, z)
Definition SHA256.cpp:34
#define FP_DENORM_FLUSH_NONE
Definition SIDefines.h:1523
Interface definition for SIInstrInfo.
Interface definition for SIRegisterInfo.
This file defines the scope_exit class, which executes user-defined cleanup logic at scope exit.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static constexpr int Concat[]
bool legalizeConstHwRegRead(MachineInstr &MI, MachineIRBuilder &B, AMDGPU::Hwreg::Id HwReg, unsigned LowBit, unsigned Width) const
void buildMultiply(LegalizerHelper &Helper, MutableArrayRef< Register > Accum, ArrayRef< Register > Src0, ArrayRef< Register > Src1, bool UsePartialMad64_32, bool SeparateOddAlignedProducts) const
bool legalizeGlobalValue(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRTF16(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeIntrinsicTrunc(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeInsert(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeTrapHsa(LegalizerHelper &Helper, MachineInstr &MI) const
std::pair< Register, unsigned > splitBufferOffsets(MachineIRBuilder &B, Register OrigOffset) const
bool legalizeBVHIntersectRayIntrinsic(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeIsAddrSpace(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, unsigned AddrSpace) const
bool legalizeUnsignedDIV_REM(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRTF32(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeCTLZ_ZERO_POISON(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeAtomicCmpXChg(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeMul(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFFREXP(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
Register getSegmentAperture(unsigned AddrSpace, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeTrapEndpgm(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFDIV64(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizePointerAsRsrcIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
To create a buffer resource from a 64-bit pointer, mask off the upper 32 bits of the pointer and repl...
bool legalizeFlogCommon(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFrameIndexVGPR(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFExp2(MachineInstr &MI, MachineIRBuilder &B) const
void legalizeUnsignedDIV_REM32Impl(MachineIRBuilder &B, Register DstDivReg, Register DstRemReg, Register Num, Register Den) const
Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI, Register Reg, bool ImageStore=false) const
Handle register layout difference for f16 images for some subtargets.
bool legalizeCTLZ_CTTZ(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBuildVector(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeTrap(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeBufferAtomic(MachineInstr &MI, MachineIRBuilder &B, const AMDGPU::BufferIntrinsicInfo &Info) const
bool legalizeFFloor(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
AMDGPULegalizerInfo(const GCNSubtarget &ST, const GCNTargetMachine &TM)
bool legalizeFDIV32(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFMad(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeSBufferPrefetch(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFExp10Unsafe(MachineIRBuilder &B, Register Dst, Register Src, unsigned Flags) const
bool legalizeFExp(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
bool legalizeFrem(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizePreloadedArgIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const
bool legalizeStore(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
bool buildPCRelGlobalAddress(Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV, int64_t Offset, unsigned GAFlags=SIInstrInfo::MO_NONE) const
MachinePointerInfo getKernargSegmentPtrInfo(MachineFunction &MF) const
bool legalizeFDIV16(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeRsqClampIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFExpUnsafeImpl(MachineIRBuilder &B, Register Dst, Register Src, unsigned Flags, bool IsExp10) const
std::pair< Register, Register > getScaledLogInput(MachineIRBuilder &B, Register Src, unsigned Flags) const
bool legalizeFDIVFastIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool loadInputValue(Register DstReg, MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const
bool legalizeBVHDualOrBVH8IntersectRayIntrinsic(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeInsertVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFExpUnsafe(MachineIRBuilder &B, Register Dst, Register Src, unsigned Flags) const
bool legalizeFEXPF64(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeAddrSpaceCast(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeExtract(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeImplicitArgPtr(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeMinNumMaxNum(LegalizerHelper &Helper, MachineInstr &MI) const
void legalizeUnsignedDIV_REM64Impl(MachineIRBuilder &B, Register DstDivReg, Register DstRemReg, Register Num, Register Den) const
bool legalizeDebugTrap(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBufferStore(MachineInstr &MI, LegalizerHelper &Helper, const AMDGPU::BufferIntrinsicInfo &Info) const
bool legalizeFastUnsafeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeSinCos(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeCTLS(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeWaveID(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFroundeven(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeWorkGroupId(MachineInstr &MI, MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ClusterIdPV, AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV, AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const
bool legalizeSignedDIV_REM(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBufferLoad(MachineInstr &MI, LegalizerHelper &Helper, const AMDGPU::BufferIntrinsicInfo &Info) const
bool legalizeITOFP(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, bool Signed) const
bool legalizeFastUnsafeFDIV64(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFPTOI(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, bool Signed) const
bool legalizeStackSave(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFlogUnsafe(MachineIRBuilder &B, Register Dst, Register Src, bool IsLog10, unsigned Flags) const
bool legalizeKernargMemParameter(MachineInstr &MI, MachineIRBuilder &B, uint64_t Offset, Align Alignment=Align(4)) const
Legalize a value that's loaded from kernel arguments.
bool legalizeImageIntrinsic(MachineInstr &MI, MachineIRBuilder &B, GISelChangeObserver &Observer, const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr) const
Rewrite image intrinsics to use register layouts expected by the subtarget.
void buildAbsGlobalAddress(Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV, MachineRegisterInfo &MRI) const
bool legalizeGetFPEnv(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool getImplicitArgPtr(Register DstReg, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRT(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
Register getKernargParameterPtr(MachineIRBuilder &B, int64_t Offset) const
bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFPow(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFceil(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRTF64(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeExtractVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeLoad(LegalizerHelper &Helper, MachineInstr &MI) const
Register fixStoreSourceType(MachineIRBuilder &B, Register VData, LLT MemTy, bool IsFormat) const
bool legalizeLaneOp(LegalizerHelper &Helper, MachineInstr &MI, Intrinsic::ID IID) const
bool legalizeSetFPEnv(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeWorkitemIDIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, unsigned Dim, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const
void buildLoadInputValue(Register DstReg, MachineIRBuilder &B, const ArgDescriptor *Arg, const TargetRegisterClass *ArgRC, LLT ArgTy) const
bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFlog2(MachineInstr &MI, MachineIRBuilder &B) const
static bool isSupportedBitWidth(unsigned Bits)
unsigned allocateBarrierGlobal(const DataLayout &DL, const GlobalVariable &GV)
void setDynLDSAlign(const Function &F, const GlobalVariable &GV)
unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalVariable &GV)
bool isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast between SrcAS and DestAS is a noop.
const std::array< unsigned, 3 > & getDims() const
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static const fltSemantics & IEEEdouble()
Definition APFloat.h:305
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition APFloat.h:1224
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition APFloat.h:1262
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition APFloat.h:1242
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition APFloat.h:1202
an instruction to allocate memory on the stack
unsigned getAddressSpace() const
Return the address space for the allocation.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
bool isMinusOne() const
Returns true if this value is exactly -1.0.
Definition Constants.h:488
bool isOne() const
Returns true if this value is exactly +1.0.
Definition Constants.h:485
This is the shared class of boolean and integer constants.
Definition Constants.h:87
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition Constants.h:174
A debug info location.
Definition DebugLoc.h:126
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
Lightweight error class with error context and mandatory checking.
Definition Error.h:159
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
LLVM_ABI void finishedChangingAllUsesOfReg()
All instructions reported as changing by changingAllUsesOfReg() have finished being changed.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
LLVM_ABI void changingAllUsesOfReg(const MachineRegisterInfo &MRI, Register Reg)
All the instructions using the given register are being changed.
Simple wrapper observer that takes several observers, and calls each one for each event.
KnownBits getKnownBits(Register R)
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
bool hasExternalLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this global belongs to.
Definition Globals.cpp:205
LLVM_ABI uint64_t getGlobalSize(const DataLayout &DL) const
Get the size of this global variable in bytes.
Definition Globals.cpp:640
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
static constexpr LLT float64()
Get a 64-bit IEEE double value.
LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
LLT getScalarType() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isFloat() const
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
static constexpr LLT float16()
Get a 16-bit IEEE half value.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr bool isPointerOrPointerVector() const
static LLT integer(unsigned SizeInBits)
static constexpr LLT bfloat16()
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
static constexpr LLT scalarOrVector(ElementCount EC, LLT ScalarTy)
static constexpr LLT float32()
Get a 32-bit IEEE float value.
LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & maxScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at most as wide as Ty.
LegalizeRuleSet & minScalarOrElt(unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
LegalizeRuleSet & unsupportedFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & widenScalarFor(std::initializer_list< LLT > Types, LegalizeMutation Mutation)
Widen the scalar, specified in mutation, when type index 0 is any type in the given list.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & moreElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Add more elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & lowerFor(std::initializer_list< LLT > Types)
The instruction is lowered when type index 0 is any type in the given list.
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy, unsigned NumElts)
Express EltTy vectors strictly using vectors with NumElts elements (or scalars when NumElts equals 1)...
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & alwaysLegal()
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & minScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty if condition is met.
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & widenScalarToNextMultipleOf(unsigned TypeIdx, unsigned Size)
Widen the scalar to the next multiple of Size.
LLVM_ABI LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI)
LLVM_ABI void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Def by performing it with addition...
LLVM_ABI LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by replacing the value type.
LLVM_ABI LegalizeResult lowerInsert(MachineInstr &MI)
LLVM_ABI LegalizeResult lowerExtract(MachineInstr &MI)
GISelValueTracking * getValueTracking() const
LLVM_ABI LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, LLT NarrowTy)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
LLVM_ABI void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a def by inserting a G_BITCAST from ...
LLVM_ABI LegalizeResult lowerFMad(MachineInstr &MI)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LLVM_ABI void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx=0, unsigned TruncOpcode=TargetOpcode::G_TRUNC)
Legalize a single operand OpIdx of the machine instruction MI as a Def by extending the operand's typ...
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
TypeSize getValue() const
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition MCRegister.h:72
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
const AllocaInst * getObjectAllocation(int ObjectIdx) const
Return the underlying Alloca of the specified stack object if it exists.
PseudoSourceValueManager & getPSVManager() const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Helper class to build MachineInstr.
MachineFunction & getMF()
Getter for the function we currently build.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
LLT getMemoryType() const
Return the memory type of the memory reference.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
MachineBasicBlock * getMBB() const
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
void setMBB(MachineBasicBlock *MBB)
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
MutableArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition ArrayRef.h:383
LLVM_ABI const PseudoSourceValue * getConstantPool()
Return a pseudo source value referencing the constant pool.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
GCNUserSGPRUsageInfo & getUserSGPRInfo()
AMDGPU::ClusterDimsAttr getClusterDims() const
SIModeRegisterDefaults getMode() const
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const
static LLVM_READONLY const TargetRegisterClass * getSGPRClassForBitWidth(unsigned BitWidth)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void resize(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
int64_t getImm() const
Register getReg() const
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM Value Representation.
Definition Value.h:75
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
self_iterator getIterator()
Definition ilist_node.h:123
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ BUFFER_STRIDED_POINTER
Address space for 192-bit fat buffer pointers with an additional index.
@ BARRIER
Address space for modeling barrier IDs as addresses.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ VGPR
Address space for VGPRs.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
bool isFlatGlobalAddrSpace(unsigned AS)
bool isGFX12Plus(const MCSubtargetInfo &STI)
constexpr int64_t getNullPointerValue(unsigned AS)
Get the null pointer value for the given address space.
bool isGFX11(const MCSubtargetInfo &STI)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
unsigned getAMDHSACodeObjectVersion(const Module &M)
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc, bool IndexedSamp)
LLVM_READNONE constexpr bool isKernel(CallingConv::ID CC)
LLVM_READNONE constexpr bool isEntryFunctionCC(CallingConv::ID CC)
LLVM_READNONE constexpr bool isCompute(CallingConv::ID CC)
const BufferIntrinsicInfo * lookupBufferIntrinsic(unsigned Intr)
TargetExtType * isNamedBarrier(const GlobalVariable &GV)
bool isGFX11Plus(const MCSubtargetInfo &STI)
unsigned getMIROperandIdx(const MachineInstr &MI, unsigned Arg)
Convert IR argument index Arg of the intrinsic called by MI, a G_INTRINSIC* instruction,...
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned getSyntheticApertureNumber(unsigned AS)
LLVM_READNONE constexpr bool isChainCC(CallingConv::ID CC)
std::pair< Register, unsigned > getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg, GISelValueTracking *ValueTracking=nullptr, bool CheckNUW=false)
Returns base register and constant offset.
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AMDGPU_Gfx
Used for AMD graphics targets.
LLVM_ABI LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar or a vector with an element type that's wider than the ...
LLVM_ABI LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
LLVM_ABI LegalityPredicate isPointer(unsigned TypeIdx)
True iff the specified type index is a pointer (with any address space).
LLVM_ABI LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LLVM_ABI LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
LLVM_ABI LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a larger total bit size than second type index.
LLVM_ABI LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy)
True if the type index is a vector with element type EltTy.
LLVM_ABI LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the specified type indices are both the same bit size.
LLVM_ABI LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar or vector with an element type that's narrower than the...
LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type)
True iff the given type index is not the specified type.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LLVM_ABI LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
LLVM_ABI LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar that's narrower than the given size.
LLVM_ABI LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx)
Keep the same scalar or element type as TypeIdx, but take the number of elements from FromTypeIdx.
LLVM_ABI LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
LLVM_ABI LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx)
Keep the same scalar or element type as the given type index.
LLVM_ABI LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
LLVM_ABI LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
LLVM_ABI LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx)
Change the scalar size or element size to have the same scalar size as type index FromIndex.
Invariant opcodes: All instruction sets have these as their low opcodes.
constexpr double inv_pi
constexpr double ln2
constexpr double ln10
constexpr float log2ef
Definition MathExtras.h:52
constexpr double log2e
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TII, MCRegister PhysReg, const TargetRegisterClass &RC, const DebugLoc &DL, LLT RegTy=LLT())
Return a virtual register corresponding to the incoming argument register PhysReg.
Definition Utils.cpp:896
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:339
@ Offset
Definition DWP.cpp:577
LLVM_ABI Type * getTypeForLLT(LLT Ty, LLVMContext &C)
Get the type back from LLT.
Definition Utils.cpp:2140
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition Utils.cpp:679
LLVM_ABI const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
Definition Utils.cpp:487
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
@ Load
The value being inserted comes from a load (InsertElement only).
std::function< std::pair< unsigned, LLT >(const LegalityQuery &)> LegalizeMutation
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition bit.h:325
void * PointerTy
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Definition MathExtras.h:380
LLVM_ABI std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition Utils.cpp:321
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:149
std::function< bool(const LegalityQuery &)> LegalityPredicate
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
To bit_cast(const From &from) noexcept
Definition bit.h:90
@ Mul
Product of integers.
@ FMul
Product of floats.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
LLVM_ABI void eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI, LostDebugLocObserver *LocObserver=nullptr)
Definition Utils.cpp:1832
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:440
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:202
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:347
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition MathExtras.h:368
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
The argument layout of a buffer intrinsic.
static constexpr uint64_t encode(Fields... Values)
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
MCRegister getRegister() const
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ Dynamic
Denormals have unknown treatment.
static constexpr DenormalMode getPreserveSign()
static constexpr DenormalMode getIEEE()
bool isZero() const
Returns true if value is all zero.
Definition KnownBits.h:78
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
ArrayRef< LLT > Types
Matching combinators.
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
DenormalMode FP64FP16Denormals
If this is set, neither input or output denormals are flushed for both f64 and f16/v2f16 instructions...
bool IEEE
Floating point opcodes that support exception flag gathering quiet and propagate signaling NaN inputs...
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.