LLVM 24.0.0git
AMDGPUISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines an instruction selector for the AMDGPU target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUISelDAGToDAG.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPUSubtarget.h"
19#include "R600RegisterInfo.h"
20#include "SIISelLowering.h"
27#include "llvm/IR/IntrinsicsAMDGPU.h"
30
31#ifdef EXPENSIVE_CHECKS
33#include "llvm/IR/Dominators.h"
34#endif
35
36#define DEBUG_TYPE "amdgpu-isel"
37
38using namespace llvm;
39
40//===----------------------------------------------------------------------===//
41// Instruction Selector Implementation
42//===----------------------------------------------------------------------===//
43
44namespace {
45static SDValue stripBitcast(SDValue Val) {
46 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
47}
48
49// Figure out if this is really an extract of the high 16-bits of a dword.
50static bool isExtractHiElt(SDValue In, SDValue &Out) {
51 In = stripBitcast(In);
52
53 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
54 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
55 if (!Idx->isOne())
56 return false;
57 Out = In.getOperand(0);
58 return true;
59 }
60 }
61
62 if (In.getOpcode() != ISD::TRUNCATE)
63 return false;
64
65 SDValue Srl = In.getOperand(0);
66 if (Srl.getOpcode() == ISD::SRL) {
67 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
68 if (ShiftAmt->getZExtValue() == 16) {
69 SDValue Result = stripBitcast(Srl.getOperand(0));
70 // op_sel can only select the high half of a 32-bit source. If the
71 // shift source is wider (e.g. a 64-bit D16 image sample result), we
72 // cannot fold it directly. Do not assign Out before this check since
73 // the caller may pass the same SDValue for both In and Out.
74 if (Result.getValueSizeInBits() != 32)
75 return false;
76 Out = Result;
77 return true;
78 }
79 }
80 }
81
82 return false;
83}
84
85static bool isLoEltExtractHi(SDValue BV) {
86 SDValue Lo = stripBitcast(BV.getOperand(0));
87 if (Lo.getOpcode() == ISD::FNEG)
88 Lo = stripBitcast(Lo.getOperand(0));
89 SDValue Unused;
90 return isExtractHiElt(Lo, Unused);
91}
92
93static SDValue createVOP3PSrc32FromLo16(SDValue Lo, SDValue Src,
94 llvm::SelectionDAG *CurDAG,
95 const GCNSubtarget *Subtarget) {
96 if (!Subtarget->useRealTrue16Insts()) {
97 return Lo;
98 }
99
100 SDValue NewSrc;
101 SDLoc SL(Lo);
102
103 if (Lo->isDivergent()) {
104 SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
105 SL, Lo.getValueType()),
106 0);
107 const SDValue Ops[] = {
108 CurDAG->getTargetConstant(AMDGPU::VGPR_32RegClassID, SL, MVT::i32), Lo,
109 CurDAG->getTargetConstant(AMDGPU::lo16, SL, MVT::i16), Undef,
110 CurDAG->getTargetConstant(AMDGPU::hi16, SL, MVT::i16)};
111
112 NewSrc = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
113 Src.getValueType(), Ops),
114 0);
115 } else {
116 // the S_MOV is needed since the Lo could still be a VGPR16.
117 // With S_MOV, isel insert a "sgpr32 = copy vgpr16" and we reply on
118 // the fixvgpr2sgprcopy pass to legalize it
119 NewSrc = SDValue(
120 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, Src.getValueType(), Lo),
121 0);
122 }
123
124 return NewSrc;
125}
126
127// Look through operations that obscure just looking at the low 16-bits of the
128// same register.
129static SDValue stripExtractLoElt(SDValue In) {
130 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
131 SDValue Idx = In.getOperand(1);
132 if (isNullConstant(Idx) && In.getValueSizeInBits() <= 32)
133 return In.getOperand(0);
134 }
135
136 if (In.getOpcode() == ISD::TRUNCATE) {
137 SDValue Src = In.getOperand(0);
138 if (Src.getValueType().getSizeInBits() == 32)
139 return stripBitcast(Src);
140 }
141
142 return In;
143}
144
145static SDValue emitRegSequence(llvm::SelectionDAG &CurDAG, unsigned DstRegClass,
146 EVT DstTy, ArrayRef<SDValue> Elts,
147 ArrayRef<unsigned> SubRegClass,
148 const SDLoc &DL) {
149 assert(Elts.size() == SubRegClass.size() && "array size mismatch");
150 unsigned NumElts = Elts.size();
151 SmallVector<SDValue, 17> Ops(2 * NumElts + 1);
152 Ops[0] = (CurDAG.getTargetConstant(DstRegClass, DL, MVT::i32));
153 for (unsigned i = 0; i < NumElts; ++i) {
154 Ops[2 * i + 1] = Elts[i];
155 Ops[2 * i + 2] = CurDAG.getTargetConstant(SubRegClass[i], DL, MVT::i32);
156 }
157 return SDValue(
158 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, DstTy, Ops), 0);
159}
160
161} // end anonymous namespace
162
164 "AMDGPU DAG->DAG Pattern Instruction Selection", false,
165 false)
166INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysisLegacy)
168#ifdef EXPENSIVE_CHECKS
171#endif
173 "AMDGPU DAG->DAG Pattern Instruction Selection", false,
174 false)
175
176/// This pass converts a legalized DAG into a AMDGPU-specific
177// DAG, ready for instruction scheduling.
179 CodeGenOptLevel OptLevel) {
180 return new AMDGPUDAGToDAGISelLegacy(TM, OptLevel);
181}
182
186
188 Subtarget = &MF.getSubtarget<GCNSubtarget>();
189 Subtarget->checkSubtargetFeatures(MF.getFunction());
190 Mode = SIModeRegisterDefaults(MF.getFunction(), *Subtarget);
192}
193
194bool AMDGPUDAGToDAGISel::fp16SrcZerosHighBits(unsigned Opc) const {
195 // XXX - only need to list legal operations.
196 switch (Opc) {
197 case ISD::POISON:
198 return true;
199 case ISD::FADD:
200 case ISD::FSUB:
201 case ISD::FMUL:
202 case ISD::FDIV:
203 case ISD::FREM:
205 case ISD::UINT_TO_FP:
206 case ISD::SINT_TO_FP:
207 case ISD::FABS:
208 // Fabs is lowered to a bit operation, but it's an and which will clear the
209 // high bits anyway.
210 case ISD::FSQRT:
211 case ISD::FSIN:
212 case ISD::FCOS:
213 case ISD::FPOWI:
214 case ISD::FPOW:
215 case ISD::FLOG:
216 case ISD::FLOG2:
217 case ISD::FLOG10:
218 case ISD::FEXP:
219 case ISD::FEXP2:
220 case ISD::FCEIL:
221 case ISD::FTRUNC:
222 case ISD::FRINT:
223 case ISD::FNEARBYINT:
224 case ISD::FROUNDEVEN:
225 case ISD::FROUND:
226 case ISD::FFLOOR:
227 case ISD::FMINNUM:
228 case ISD::FMAXNUM:
229 case ISD::FLDEXP:
230 case AMDGPUISD::FRACT:
231 case AMDGPUISD::CLAMP:
232 case AMDGPUISD::COS_HW:
233 case AMDGPUISD::SIN_HW:
234 case AMDGPUISD::FMIN3:
235 case AMDGPUISD::FMAX3:
236 case AMDGPUISD::FMED3:
237 case AMDGPUISD::FMAD_FTZ:
238 case AMDGPUISD::RCP:
239 case AMDGPUISD::RSQ:
240 case AMDGPUISD::RCP_IFLAG:
241 // On gfx10, all 16-bit instructions preserve the high bits.
242 return Subtarget->getGeneration() <= AMDGPUSubtarget::GFX9;
243 case ISD::FP_ROUND:
244 // We may select fptrunc (fma/mad) to mad_mixlo, which does not zero the
245 // high bits on gfx9.
246 // TODO: If we had the source node we could see if the source was fma/mad
247 return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
248 case ISD::FMA:
249 case ISD::FMAD:
250 case AMDGPUISD::DIV_FIXUP:
251 return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
252 default:
253 // fcopysign, select and others may be lowered to 32-bit bit operations
254 // which don't zero the high bits.
255 return false;
256 }
257}
258
260#ifdef EXPENSIVE_CHECKS
262 LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
263 for (auto &L : LI->getLoopsInPreorder()) {
264 assert(L->isLCSSAForm(DT));
265 }
266#endif
268}
269
278
280 assert(Subtarget->d16PreservesUnusedBits());
281 MVT VT = N->getValueType(0).getSimpleVT();
282 if (VT != MVT::v2i16 && VT != MVT::v2f16)
283 return false;
284
285 SDValue Lo = N->getOperand(0);
286 SDValue Hi = N->getOperand(1);
287
288 LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi));
289
290 // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
291 // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
292 // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
293
294 // Need to check for possible indirect dependencies on the other half of the
295 // vector to avoid introducing a cycle.
296 if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) {
297 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
298
299 SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
300 SDValue Ops[] = {
301 LdHi->getChain(), LdHi->getBasePtr(), TiedIn
302 };
303
304 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
305 if (LdHi->getMemoryVT() == MVT::i8) {
306 LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
307 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
308 } else {
309 assert(LdHi->getMemoryVT() == MVT::i16);
310 }
311
312 SDValue NewLoadHi =
313 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList,
314 Ops, LdHi->getMemoryVT(),
315 LdHi->getMemOperand());
316
317 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi);
318 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1));
319 return true;
320 }
321
322 // build_vector (load ptr), hi -> load_d16_lo ptr, hi
323 // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
324 // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
325 LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo));
326 if (LdLo && Lo.hasOneUse()) {
327 SDValue TiedIn = getHi16Elt(Hi);
328 if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode()))
329 return false;
330
331 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
332 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
333 if (LdLo->getMemoryVT() == MVT::i8) {
334 LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
335 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
336 } else {
337 assert(LdLo->getMemoryVT() == MVT::i16);
338 }
339
340 TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn);
341
342 SDValue Ops[] = {
343 LdLo->getChain(), LdLo->getBasePtr(), TiedIn
344 };
345
346 SDValue NewLoadLo =
347 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList,
348 Ops, LdLo->getMemoryVT(),
349 LdLo->getMemOperand());
350
351 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo);
352 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1));
353 return true;
354 }
355
356 return false;
357}
358
360 auto *Mem = cast<MemSDNode>(N);
361 EVT VT = N->getValueType(0);
362 if (Mem->getAddressSpace() != AMDGPUAS::REGION_ADDRESS || VT.isVector() ||
363 VT.getSizeInBits() != 16)
364 return false;
365
366 SDLoc SL(N);
367 auto *Ld = dyn_cast<LoadSDNode>(N);
368 ISD::LoadExtType ExtType =
369 Ld ? Ld->getExtensionType() : cast<AtomicSDNode>(N)->getExtensionType();
370 if (ExtType == ISD::NON_EXTLOAD)
371 ExtType = ISD::EXTLOAD;
372
373 EVT MemVT = Mem->getMemoryVT().changeTypeToInteger();
374 SDValue NewLoad =
375 Ld ? CurDAG->getExtLoad(ExtType, SL, MVT::i32, Mem->getChain(),
376 Mem->getBasePtr(), MemVT, Mem->getMemOperand())
377 : CurDAG->getAtomicLoad(ExtType, SL, MemVT, MVT::i32, Mem->getChain(),
378 Mem->getBasePtr(), Mem->getMemOperand());
379
380 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, SL, MVT::i16, NewLoad);
381 SDValue Ops[] = {CurDAG->getBitcast(VT, Trunc), NewLoad.getValue(1)};
382 CurDAG->ReplaceAllUsesWith(N, Ops);
383 return true;
384}
385
387 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
388
389 bool MadeChange = false;
390 while (Position != CurDAG->allnodes_begin()) {
391 SDNode *N = &*--Position;
392 if (N->use_empty())
393 continue;
394
395 switch (N->getOpcode()) {
397 // TODO: Match load d16 from shl (extload:i16), 16
398 if (Subtarget->d16PreservesUnusedBits())
399 MadeChange |= matchLoadD16FromBuildVector(N);
400 break;
401 case ISD::LOAD:
402 case ISD::ATOMIC_LOAD:
403 if (Subtarget->useRealTrue16Insts())
404 MadeChange |= widenRegionLoad16(N);
405 break;
406 default:
407 break;
408 }
409 }
410
411 if (MadeChange) {
412 CurDAG->RemoveDeadNodes();
413 LLVM_DEBUG(dbgs() << "After PreProcess:\n";
414 CurDAG->dump(););
415 }
416}
417
418bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N) const {
419 if (N->isUndef())
420 return true;
421
422 const SIInstrInfo *TII = Subtarget->getInstrInfo();
424 return TII->isInlineConstant(C->getAPIntValue());
425
427 return TII->isInlineConstant(C->getValueAPF());
428
429 return false;
430}
431
432/// Determine the register class for \p OpNo
433/// \returns The register class of the virtual register that will be used for
434/// the given operand number \OpNo or NULL if the register class cannot be
435/// determined.
436const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
437 unsigned OpNo) const {
438 if (!N->isMachineOpcode()) {
439 if (N->getOpcode() == ISD::CopyToReg) {
440 Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
441 if (Reg.isVirtual()) {
443 return MRI.getRegClass(Reg);
444 }
445
446 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
447 return TRI->getPhysRegBaseClass(Reg);
448 }
449
450 return nullptr;
451 }
452
453 switch (N->getMachineOpcode()) {
454 default: {
455 const SIInstrInfo *TII = Subtarget->getInstrInfo();
456 const MCInstrDesc &Desc = TII->get(N->getMachineOpcode());
457 unsigned OpIdx = Desc.getNumDefs() + OpNo;
458 if (OpIdx >= Desc.getNumOperands())
459 return nullptr;
460
461 int16_t RegClass = TII->getOpRegClassID(Desc.operands()[OpIdx]);
462 if (RegClass == -1)
463 return nullptr;
464
465 return Subtarget->getRegisterInfo()->getRegClass(RegClass);
466 }
467 case AMDGPU::REG_SEQUENCE: {
468 unsigned RCID = N->getConstantOperandVal(0);
469 const TargetRegisterClass *SuperRC =
470 Subtarget->getRegisterInfo()->getRegClass(RCID);
471
472 SDValue SubRegOp = N->getOperand(OpNo + 1);
473 unsigned SubRegIdx = SubRegOp->getAsZExtVal();
474 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
475 SubRegIdx);
476 }
477 }
478}
479
480SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
481 SDValue Glue) const {
483 Ops.push_back(NewChain); // Replace the chain.
484 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
485 Ops.push_back(N->getOperand(i));
486
487 Ops.push_back(Glue);
488 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
489}
490
491SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
492 const SITargetLowering& Lowering =
493 *static_cast<const SITargetLowering*>(getTargetLowering());
494
495 assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
496
497 SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val);
498 return glueCopyToOp(N, M0, M0.getValue(1));
499}
500
501SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
502 unsigned AS = cast<MemSDNode>(N)->getAddressSpace();
503 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
504 if (Subtarget->ldsRequiresM0Init())
505 return glueCopyToM0(
506 N, CurDAG->getSignedTargetConstant(-1, SDLoc(N), MVT::i32));
507 } else if (AS == AMDGPUAS::REGION_ADDRESS) {
508 MachineFunction &MF = CurDAG->getMachineFunction();
509 unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
510 return
511 glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32));
512 }
513 return N;
514}
515
516MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
517 EVT VT) const {
518 SDNode *Lo = CurDAG->getMachineNode(
519 AMDGPU::S_MOV_B32, DL, MVT::i32,
520 CurDAG->getTargetConstant(Lo_32(Imm), DL, MVT::i32));
521 SDNode *Hi = CurDAG->getMachineNode(
522 AMDGPU::S_MOV_B32, DL, MVT::i32,
523 CurDAG->getTargetConstant(Hi_32(Imm), DL, MVT::i32));
524 const SDValue Ops[] = {
525 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
526 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
527 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
528
529 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops);
530}
531
532SDNode *AMDGPUDAGToDAGISel::packConstantV2I16(const SDNode *N,
533 SelectionDAG &DAG) const {
534 // TODO: Handle undef as zero
535
536 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
537 uint32_t LHSVal, RHSVal;
538 if (getConstantValue(N->getOperand(0), LHSVal) &&
539 getConstantValue(N->getOperand(1), RHSVal)) {
540 SDLoc SL(N);
541 uint32_t K = (LHSVal & 0xffff) | (RHSVal << 16);
542 return DAG.getMachineNode(
543 isVGPRImm(N) ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32, SL,
544 N->getValueType(0), DAG.getTargetConstant(K, SL, MVT::i32));
545 }
546
547 return nullptr;
548}
549
550void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
551 EVT VT = N->getValueType(0);
552 unsigned NumVectorElts = VT.getVectorNumElements();
553 EVT EltVT = VT.getVectorElementType();
554 SDLoc DL(N);
555 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
556
557 if (NumVectorElts == 1) {
558 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
559 RegClass);
560 return;
561 }
562
563 bool IsGCN = CurDAG->getSubtarget().getTargetTriple().isAMDGCN();
564 if (IsGCN && Subtarget->has64BitLiterals() && VT.getSizeInBits() == 64 &&
565 CurDAG->isConstantValueOfAnyType(SDValue(N, 0))) {
566 uint64_t C = 0;
567 bool AllConst = true;
568 unsigned EltSize = EltVT.getSizeInBits();
569 for (unsigned I = 0; I < NumVectorElts; ++I) {
570 SDValue Op = N->getOperand(I);
571 if (Op.isUndef()) {
572 AllConst = false;
573 break;
574 }
575 uint64_t Val;
577 Val = CF->getValueAPF().bitcastToAPInt().getZExtValue();
578 } else
579 Val = cast<ConstantSDNode>(Op)->getZExtValue();
580 C |= Val << (EltSize * I);
581 }
582 if (AllConst) {
583 SDValue CV = CurDAG->getTargetConstant(C, DL, MVT::i64);
584 MachineSDNode *Copy =
585 CurDAG->getMachineNode(AMDGPU::S_MOV_B64_IMM_PSEUDO, DL, VT, CV);
586 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, VT, SDValue(Copy, 0),
587 RegClass);
588 return;
589 }
590 }
591
592 assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
593 "supported yet");
594 // 32 = Max Num Vector Elements
595 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
596 // 1 = Vector Register Class
597 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
598
599 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
600 bool IsRegSeq = true;
601 unsigned NOps = N->getNumOperands();
602 unsigned EltSizeInRegs = EltVT.getSizeInBits() / 32;
603 assert(IsGCN || EltSizeInRegs == 1);
604 for (unsigned i = 0; i < NOps; i++) {
605 // XXX: Why is this here?
606 if (isa<RegisterSDNode>(N->getOperand(i))) {
607 IsRegSeq = false;
608 break;
609 }
610 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(
611 i * EltSizeInRegs, EltSizeInRegs)
613 RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
614 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32);
615 }
616 if (NOps != NumVectorElts) {
617 // Fill in the missing undef elements if this was a scalar_to_vector.
618 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
619 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
620 DL, EltVT);
621 for (unsigned i = NOps; i < NumVectorElts; ++i) {
622 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(
623 i * EltSizeInRegs, EltSizeInRegs)
625 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
626 RegSeqArgs[1 + (2 * i) + 1] =
627 CurDAG->getTargetConstant(Sub, DL, MVT::i32);
628 }
629 }
630
631 if (!IsRegSeq)
632 SelectCode(N);
633 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
634}
635
637 EVT VT = N->getValueType(0);
638 EVT EltVT = VT.getVectorElementType();
639
640 // TODO: Handle 16-bit element vectors with even aligned masks.
641 if (!Subtarget->hasPkMovB32() || !EltVT.bitsEq(MVT::i32) ||
642 VT.getVectorNumElements() != 2) {
643 SelectCode(N);
644 return;
645 }
646
647 auto *SVN = cast<ShuffleVectorSDNode>(N);
648
649 SDValue Src0 = SVN->getOperand(0);
650 SDValue Src1 = SVN->getOperand(1);
651 ArrayRef<int> Mask = SVN->getMask();
652 SDLoc DL(N);
653
654 assert(Src0.getValueType().getVectorNumElements() == 2 && Mask.size() == 2 &&
655 Mask[0] < 4 && Mask[1] < 4);
656
657 SDValue VSrc0 = Mask[0] < 2 ? Src0 : Src1;
658 SDValue VSrc1 = Mask[1] < 2 ? Src0 : Src1;
659 unsigned Src0SubReg = Mask[0] & 1 ? AMDGPU::sub1 : AMDGPU::sub0;
660 unsigned Src1SubReg = Mask[1] & 1 ? AMDGPU::sub1 : AMDGPU::sub0;
661
662 if (Mask[0] < 0) {
663 Src0SubReg = Src1SubReg;
664 MachineSDNode *ImpDef =
665 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
666 VSrc0 = SDValue(ImpDef, 0);
667 }
668
669 if (Mask[1] < 0) {
670 Src1SubReg = Src0SubReg;
671 MachineSDNode *ImpDef =
672 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
673 VSrc1 = SDValue(ImpDef, 0);
674 }
675
676 // SGPR case needs to lower to copies.
677 //
678 // Also use subregister extract when we can directly blend the registers with
679 // a simple subregister copy.
680 //
681 // TODO: Maybe we should fold this out earlier
682 if (N->isDivergent() && Src0SubReg == AMDGPU::sub1 &&
683 Src1SubReg == AMDGPU::sub0) {
684 // The low element of the result always comes from src0.
685 // The high element of the result always comes from src1.
686 // op_sel selects the high half of src0.
687 // op_sel_hi selects the high half of src1.
688
689 unsigned Src0OpSel =
690 Src0SubReg == AMDGPU::sub1 ? SISrcMods::OP_SEL_0 : SISrcMods::NONE;
691 unsigned Src1OpSel =
692 Src1SubReg == AMDGPU::sub1 ? SISrcMods::OP_SEL_0 : SISrcMods::NONE;
693
694 // Enable op_sel_hi to avoid printing it. This should have no effect on the
695 // result.
696 Src0OpSel |= SISrcMods::OP_SEL_1;
697 Src1OpSel |= SISrcMods::OP_SEL_1;
698
699 SDValue Src0OpSelVal = CurDAG->getTargetConstant(Src0OpSel, DL, MVT::i32);
700 SDValue Src1OpSelVal = CurDAG->getTargetConstant(Src1OpSel, DL, MVT::i32);
701 SDValue ZeroMods = CurDAG->getTargetConstant(0, DL, MVT::i32);
702
703 CurDAG->SelectNodeTo(N, AMDGPU::V_PK_MOV_B32, N->getVTList(),
704 {Src0OpSelVal, VSrc0, Src1OpSelVal, VSrc1,
705 ZeroMods, // clamp
706 ZeroMods, // op_sel
707 ZeroMods, // op_sel_hi
708 ZeroMods, // neg_lo
709 ZeroMods}); // neg_hi
710 return;
711 }
712
713 SDValue ResultElt0 =
714 CurDAG->getTargetExtractSubreg(Src0SubReg, DL, EltVT, VSrc0);
715 SDValue ResultElt1 =
716 CurDAG->getTargetExtractSubreg(Src1SubReg, DL, EltVT, VSrc1);
717
718 const SDValue Ops[] = {
719 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
720 ResultElt0, CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
721 ResultElt1, CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
722 CurDAG->SelectNodeTo(N, TargetOpcode::REG_SEQUENCE, VT, Ops);
723}
724
726 unsigned int Opc = N->getOpcode();
727 if (N->isMachineOpcode()) {
728 N->setNodeId(-1);
729 return; // Already selected.
730 }
731
732 // isa<MemSDNode> almost works but is slightly too permissive for some DS
733 // intrinsics.
734 if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N)) {
735 N = glueCopyToM0LDSInit(N);
736 SelectCode(N);
737 return;
738 }
739
740 switch (Opc) {
741 default:
742 break;
743 case ISD::UADDO_CARRY:
744 case ISD::USUBO_CARRY:
745 if (N->getValueType(0) == MVT::i64) {
746 SelectAddcSubbI64(N);
747 return;
748 }
749
750 if (N->getValueType(0) != MVT::i32)
751 break;
752
753 SelectAddcSubb(N);
754 return;
755 case ISD::UADDO:
756 case ISD::USUBO: {
757 if (N->getValueType(0) == MVT::i64) {
758 SelectAddcSubbI64(N);
759 return;
760 }
761
762 SelectUADDO_USUBO(N);
763 return;
764 }
765 case AMDGPUISD::FMUL_W_CHAIN: {
766 SelectFMUL_W_CHAIN(N);
767 return;
768 }
769 case AMDGPUISD::FMA_W_CHAIN: {
770 SelectFMA_W_CHAIN(N);
771 return;
772 }
773
775 case ISD::BUILD_VECTOR: {
776 EVT VT = N->getValueType(0);
777 unsigned NumVectorElts = VT.getVectorNumElements();
778 if (VT.getScalarSizeInBits() == 16) {
779 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
780 if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
781 ReplaceNode(N, Packed);
782 return;
783 }
784 }
785
786 break;
787 }
788
789 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
790 EVT EltTy = VT.getVectorElementType();
791 assert(EltTy.bitsEq(MVT::i32) || EltTy.bitsEq(MVT::i64));
792 unsigned VecInBits = NumVectorElts * EltTy.getScalarSizeInBits();
793 const TargetRegisterClass *RegClass =
794 N->isDivergent() ? TRI->getDefaultVectorSuperClassForBitWidth(VecInBits)
796
797 SelectBuildVector(N, RegClass->getID());
798 return;
799 }
802 return;
803 case ISD::BUILD_PAIR: {
804 SDValue RC, SubReg0, SubReg1;
805 SDLoc DL(N);
806 if (N->getValueType(0) == MVT::i128) {
807 RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32);
808 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
809 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
810 } else if (N->getValueType(0) == MVT::i64) {
811 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
812 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
813 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
814 } else {
815 llvm_unreachable("Unhandled value type for BUILD_PAIR");
816 }
817 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
818 N->getOperand(1), SubReg1 };
819 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
820 N->getValueType(0), Ops));
821 return;
822 }
823
824 case ISD::Constant:
825 case ISD::ConstantFP: {
826 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N) ||
827 Subtarget->has64BitLiterals())
828 break;
829
830 uint64_t Imm;
832 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
834 break;
835 } else {
837 Imm = C->getZExtValue();
839 break;
840 }
841
842 SDLoc DL(N);
843 ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0)));
844 return;
845 }
846 case AMDGPUISD::BFE_I32:
847 case AMDGPUISD::BFE_U32: {
848 // There is a scalar version available, but unlike the vector version which
849 // has a separate operand for the offset and width, the scalar version packs
850 // the width and offset into a single operand. Try to move to the scalar
851 // version if the offsets are constant, so that we can try to keep extended
852 // loads of kernel arguments in SGPRs.
853
854 // TODO: Technically we could try to pattern match scalar bitshifts of
855 // dynamic values, but it's probably not useful.
857 if (!Offset)
858 break;
859
860 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
861 if (!Width)
862 break;
863
864 bool Signed = Opc == AMDGPUISD::BFE_I32;
865
866 uint32_t OffsetVal = Offset->getZExtValue();
867 uint32_t WidthVal = Width->getZExtValue();
868
869 ReplaceNode(N, getBFE32(Signed, SDLoc(N), N->getOperand(0), OffsetVal,
870 WidthVal));
871 return;
872 }
873 case AMDGPUISD::DIV_SCALE: {
874 SelectDIV_SCALE(N);
875 return;
876 }
879 SelectMAD_64_32(N);
880 return;
881 }
882 case ISD::SMUL_LOHI:
883 case ISD::UMUL_LOHI:
884 return SelectMUL_LOHI(N);
885 case ISD::CopyToReg: {
887 *static_cast<const SITargetLowering*>(getTargetLowering());
888 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
889 break;
890 }
891 case ISD::AND:
892 case ISD::SRL:
893 case ISD::SRA:
895 if (N->getValueType(0) != MVT::i32)
896 break;
897
898 SelectS_BFE(N);
899 return;
900 case ISD::BRCOND:
901 SelectBRCOND(N);
902 return;
903 case ISD::FP_EXTEND:
904 SelectFP_EXTEND(N);
905 return;
906 case AMDGPUISD::CVT_PKRTZ_F16_F32:
907 case AMDGPUISD::CVT_PKNORM_I16_F32:
908 case AMDGPUISD::CVT_PKNORM_U16_F32:
909 case AMDGPUISD::CVT_PK_U16_U32:
910 case AMDGPUISD::CVT_PK_I16_I32: {
911 // Hack around using a legal type if f16 is illegal.
912 if (N->getValueType(0) == MVT::i32) {
913 MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
914 N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT),
915 { N->getOperand(0), N->getOperand(1) });
916 SelectCode(N);
917 return;
918 }
919
920 break;
921 }
923 SelectINTRINSIC_W_CHAIN(N);
924 return;
925 }
927 SelectINTRINSIC_WO_CHAIN(N);
928 return;
929 }
930 case ISD::INTRINSIC_VOID: {
931 SelectINTRINSIC_VOID(N);
932 return;
933 }
935 SelectWAVE_ADDRESS(N);
936 return;
937 }
938 case ISD::STACKRESTORE: {
939 SelectSTACKRESTORE(N);
940 return;
941 }
942 case ISD::WRITE_REGISTER: {
943 SelectWRITE_REGISTER(N);
944 return;
945 }
946 }
947
948 SelectCode(N);
949}
950
952 if (!Subtarget->hasSDWA())
953 return false;
954
955 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
956 EVT VT = cast<VTSDNode>(N->getOperand(1))->getVT();
957 return VT.getScalarSizeInBits() == 8 || VT.getScalarSizeInBits() == 16;
958 }
959
960 if (N->getOpcode() == ISD::AND)
961 if (auto *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1)))
962 return RHS->getZExtValue() == 0xFF || RHS->getZExtValue() == 0xFFFF;
963
964 if (N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL)
965 if (auto *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1)))
966 return (RHS->getZExtValue() % 8) == 0;
967
968 return false;
969}
970
971bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
972 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
973 const Instruction *Term = BB->getTerminator();
974 return Term->getMetadata("amdgpu.uniform") ||
975 Term->getMetadata("structurizecfg.uniform");
976}
977
978bool AMDGPUDAGToDAGISel::isUnneededShiftMask(const SDNode *N,
979 unsigned ShAmtBits) const {
980 assert(N->getOpcode() == ISD::AND);
981
982 const APInt &RHS = N->getConstantOperandAPInt(1);
983 if (RHS.countr_one() >= ShAmtBits)
984 return true;
985
986 const APInt &LHSKnownZeros = CurDAG->computeKnownBits(N->getOperand(0)).Zero;
987 return (LHSKnownZeros | RHS).countr_one() >= ShAmtBits;
988}
989
991 SDValue &N0, SDValue &N1) {
992 if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
994 // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
995 // (i64 (bitcast (v2i32 (build_vector
996 // (or (extract_vector_elt V, 0), OFFSET),
997 // (extract_vector_elt V, 1)))))
998 SDValue Lo = Addr.getOperand(0).getOperand(0);
999 if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) {
1000 SDValue BaseLo = Lo.getOperand(0);
1001 SDValue BaseHi = Addr.getOperand(0).getOperand(1);
1002 // Check that split base (Lo and Hi) are extracted from the same one.
1003 if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1004 BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1005 BaseLo.getOperand(0) == BaseHi.getOperand(0) &&
1006 // Lo is statically extracted from index 0.
1007 isa<ConstantSDNode>(BaseLo.getOperand(1)) &&
1008 BaseLo.getConstantOperandVal(1) == 0 &&
1009 // Hi is statically extracted from index 0.
1010 isa<ConstantSDNode>(BaseHi.getOperand(1)) &&
1011 BaseHi.getConstantOperandVal(1) == 1) {
1012 N0 = BaseLo.getOperand(0).getOperand(0);
1013 N1 = Lo.getOperand(1);
1014 return true;
1015 }
1016 }
1017 }
1018 return false;
1019}
1020
1021bool AMDGPUDAGToDAGISel::isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
1022 SDValue &RHS) const {
1023 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1024 LHS = Addr.getOperand(0);
1025 RHS = Addr.getOperand(1);
1026 return true;
1027 }
1028
1031 return true;
1032 }
1033
1034 return false;
1035}
1036
1038 return "AMDGPU DAG->DAG Pattern Instruction Selection";
1039}
1040
1044
1049 .getManager();
1050 auto &F = MF.getFunction();
1051 // UniformityInfoAnalysis is optional in generic dag isel,
1052 // AMDGPUISelDAGToDAGPass requires it, calculate it explicitly.
1053 FAM.getResult<UniformityInfoAnalysis>(F);
1054#ifdef EXPENSIVE_CHECKS
1055 DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
1056 LoopInfo &LI = FAM.getResult<LoopAnalysis>(F);
1057 for (auto &L : LI.getLoopsInPreorder())
1058 assert(L->isLCSSAForm(DT) && "Loop is not in LCSSA form!");
1059#endif
1060 return SelectionDAGISelPass::run(MF, MFAM);
1061}
1062
1063//===----------------------------------------------------------------------===//
1064// Complex Patterns
1065//===----------------------------------------------------------------------===//
1066
1067bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
1068 SDValue &Offset) {
1069 return false;
1070}
1071
1072bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
1073 SDValue &Offset) {
1075 SDLoc DL(Addr);
1076
1077 if ((C = dyn_cast<ConstantSDNode>(Addr))) {
1078 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
1079 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
1080 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
1081 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
1082 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
1083 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
1084 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
1085 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
1086 Base = Addr.getOperand(0);
1087 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
1088 } else {
1089 Base = Addr;
1090 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1091 }
1092
1093 return true;
1094}
1095
1096SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
1097 const SDLoc &DL) const {
1098 SDNode *Mov = CurDAG->getMachineNode(
1099 AMDGPU::S_MOV_B32, DL, MVT::i32,
1100 CurDAG->getTargetConstant(Val, DL, MVT::i32));
1101 return SDValue(Mov, 0);
1102}
1103
1104void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
1105 SDValue LHS = N->getOperand(0);
1106 SDValue RHS = N->getOperand(1);
1107 SDValue CI = N->getOperand(2);
1108
1109 if (N->isDivergent()) {
1110 unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::V_ADDC_U32_e64
1111 : AMDGPU::V_SUBB_U32_e64;
1112 CurDAG->SelectNodeTo(
1113 N, Opc, N->getVTList(),
1114 {LHS, RHS, CI,
1115 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1116 } else {
1117 unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::S_ADD_CO_PSEUDO
1118 : AMDGPU::S_SUB_CO_PSEUDO;
1119 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI});
1120 }
1121}
1122
1123void AMDGPUDAGToDAGISel::SelectAddcSubbI64(SDNode *N) {
1124 SDLoc DL(N);
1125 SDValue LHS = N->getOperand(0);
1126 SDValue RHS = N->getOperand(1);
1127
1128 unsigned Opcode = N->getOpcode();
1129 bool ConsumeCarry = Opcode == ISD::UADDO_CARRY || Opcode == ISD::USUBO_CARRY;
1130 bool IsAdd = Opcode == ISD::UADDO || Opcode == ISD::UADDO_CARRY;
1131
1132 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1133 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1134
1135 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
1136 MVT::i32, LHS, Sub0);
1137 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
1138 MVT::i32, LHS, Sub1);
1139
1140 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
1141 MVT::i32, RHS, Sub0);
1142 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
1143 MVT::i32, RHS, Sub1);
1144
1145 SDVTList VTList = CurDAG->getVTList(MVT::i32, N->getValueType(1));
1146
1147 static const unsigned NoCarryOpcMap[2][2] = {
1148 {AMDGPU::S_USUBO_PSEUDO, AMDGPU::S_UADDO_PSEUDO},
1149 {AMDGPU::V_SUB_CO_U32_e64, AMDGPU::V_ADD_CO_U32_e64}};
1150 static const unsigned CarryOpcMap[2][2] = {
1151 {AMDGPU::S_SUB_CO_PSEUDO, AMDGPU::S_ADD_CO_PSEUDO},
1152 {AMDGPU::V_SUBB_U32_e64, AMDGPU::V_ADDC_U32_e64}};
1153
1154 bool IsVALU = N->isDivergent();
1155
1156 unsigned NoCarryOpc = NoCarryOpcMap[IsVALU][IsAdd];
1157 unsigned CarryOpc = CarryOpcMap[IsVALU][IsAdd];
1158 SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1159
1160 SDNode *AddLo;
1161 if (!ConsumeCarry) {
1162 if (IsVALU) {
1163 SDValue Args[] = {SDValue(Lo0, 0), SDValue(Lo1, 0), Clamp};
1164 AddLo = CurDAG->getMachineNode(NoCarryOpc, DL, VTList, Args);
1165 } else {
1166 SDValue Args[] = {SDValue(Lo0, 0), SDValue(Lo1, 0)};
1167 AddLo = CurDAG->getMachineNode(NoCarryOpc, DL, VTList, Args);
1168 }
1169 } else {
1170 if (IsVALU) {
1171 SDValue Args[] = {SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2),
1172 Clamp};
1173 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
1174 } else {
1175 SDValue Args[] = {SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2)};
1176 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
1177 }
1178 }
1179
1180 SDNode *AddHi;
1181 if (IsVALU) {
1182 SDValue Args[] = {SDValue(Hi0, 0), SDValue(Hi1, 0), SDValue(AddLo, 1),
1183 Clamp};
1184 AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
1185 } else {
1186 SDValue Args[] = {SDValue(Hi0, 0), SDValue(Hi1, 0), SDValue(AddLo, 1)};
1187 AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
1188 }
1189
1190 unsigned RC = IsVALU ? AMDGPU::VReg_64RegClassID : AMDGPU::SReg_64RegClassID;
1191 SDValue RegSequenceArgs[] = {CurDAG->getTargetConstant(RC, DL, MVT::i32),
1192 SDValue(AddLo, 0), Sub0, SDValue(AddHi, 0),
1193 Sub1};
1194 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1195 MVT::i64, RegSequenceArgs);
1196
1197 ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1));
1198 ReplaceNode(N, RegSequence);
1199}
1200
1201void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
1202 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
1203 // carry out despite the _i32 name. These were renamed in VI to _U32.
1204 // FIXME: We should probably rename the opcodes here.
1205 bool IsAdd = N->getOpcode() == ISD::UADDO;
1206 bool IsVALU = N->isDivergent();
1207
1208 for (SDNode::user_iterator UI = N->user_begin(), E = N->user_end(); UI != E;
1209 ++UI)
1210 if (UI.getUse().getResNo() == 1) {
1211 if (UI->isMachineOpcode()) {
1212 if (UI->getMachineOpcode() !=
1213 (IsAdd ? AMDGPU::S_ADD_CO_PSEUDO : AMDGPU::S_SUB_CO_PSEUDO)) {
1214 IsVALU = true;
1215 break;
1216 }
1217 } else {
1218 if (UI->getOpcode() != (IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY)) {
1219 IsVALU = true;
1220 break;
1221 }
1222 }
1223 }
1224
1225 if (IsVALU) {
1226 unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
1227
1228 CurDAG->SelectNodeTo(
1229 N, Opc, N->getVTList(),
1230 {N->getOperand(0), N->getOperand(1),
1231 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1232 } else {
1233 unsigned Opc = IsAdd ? AMDGPU::S_UADDO_PSEUDO : AMDGPU::S_USUBO_PSEUDO;
1234
1235 CurDAG->SelectNodeTo(N, Opc, N->getVTList(),
1236 {N->getOperand(0), N->getOperand(1)});
1237 }
1238}
1239
1240void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
1241 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod
1242 SDValue Ops[10];
1243
1244 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
1245 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1246 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
1247 Ops[8] = N->getOperand(0);
1248 Ops[9] = N->getOperand(4);
1249
1250 // If there are no source modifiers, prefer fmac over fma because it can use
1251 // the smaller VOP2 encoding.
1252 bool UseFMAC = Subtarget->hasDLInsts() &&
1253 cast<ConstantSDNode>(Ops[0])->isZero() &&
1254 cast<ConstantSDNode>(Ops[2])->isZero() &&
1255 cast<ConstantSDNode>(Ops[4])->isZero();
1256 unsigned Opcode = UseFMAC ? AMDGPU::V_FMAC_F32_e64 : AMDGPU::V_FMA_F32_e64;
1257 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), Ops);
1258}
1259
1260void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
1261 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod
1262 SDValue Ops[8];
1263
1264 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
1265 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1266 Ops[6] = N->getOperand(0);
1267 Ops[7] = N->getOperand(3);
1268
1269 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
1270}
1271
1272// We need to handle this here because tablegen doesn't support matching
1273// instructions with multiple outputs.
1274void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
1275 EVT VT = N->getValueType(0);
1276
1277 assert(VT == MVT::f32 || VT == MVT::f64);
1278
1279 unsigned Opc
1280 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64_e64 : AMDGPU::V_DIV_SCALE_F32_e64;
1281
1282 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
1283 // omod
1284 SDValue Ops[8];
1285 SelectVOP3BMods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
1286 SelectVOP3BMods(N->getOperand(1), Ops[3], Ops[2]);
1287 SelectVOP3BMods(N->getOperand(2), Ops[5], Ops[4]);
1288 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1289}
1290
1291// We need to handle this here because tablegen doesn't support matching
1292// instructions with multiple outputs.
1293void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
1294 SDLoc SL(N);
1295 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
1296 unsigned Opc;
1297 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() && !N->hasAnyUseOfValue(1);
1298 if (Subtarget->hasMADIntraFwdBug())
1299 Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1300 : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1301 else if (UseNoCarry)
1302 Opc = Signed ? AMDGPU::V_MAD_NC_I64_I32_e64 : AMDGPU::V_MAD_NC_U64_U32_e64;
1303 else
1304 Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1305
1306 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1307 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1308 Clamp };
1309
1310 if (UseNoCarry) {
1311 MachineSDNode *Mad = CurDAG->getMachineNode(Opc, SL, MVT::i64, Ops);
1312 ReplaceUses(SDValue(N, 0), SDValue(Mad, 0));
1313 CurDAG->RemoveDeadNode(N);
1314 return;
1315 }
1316
1317 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1318}
1319
1320// We need to handle this here because tablegen doesn't support matching
1321// instructions with multiple outputs.
1322void AMDGPUDAGToDAGISel::SelectMUL_LOHI(SDNode *N) {
1323 SDLoc SL(N);
1324 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
1325 SDVTList VTList;
1326 unsigned Opc;
1327 if (Subtarget->hasMadNC64_32Insts()) {
1328 VTList = CurDAG->getVTList(MVT::i64);
1329 Opc = Signed ? AMDGPU::V_MAD_NC_I64_I32_e64 : AMDGPU::V_MAD_NC_U64_U32_e64;
1330 } else {
1331 VTList = CurDAG->getVTList(MVT::i64, MVT::i1);
1332 if (Subtarget->hasMADIntraFwdBug()) {
1333 Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1334 : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1335 } else {
1336 Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1337 }
1338 }
1339
1340 SDValue Zero = CurDAG->getTargetConstant(0, SL, MVT::i64);
1341 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1342 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Zero, Clamp};
1343 SDNode *Mad = CurDAG->getMachineNode(Opc, SL, VTList, Ops);
1344 if (!SDValue(N, 0).use_empty()) {
1345 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32);
1346 SDNode *Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1347 MVT::i32, SDValue(Mad, 0), Sub0);
1348 ReplaceUses(SDValue(N, 0), SDValue(Lo, 0));
1349 }
1350 if (!SDValue(N, 1).use_empty()) {
1351 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32);
1352 SDNode *Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1353 MVT::i32, SDValue(Mad, 0), Sub1);
1354 ReplaceUses(SDValue(N, 1), SDValue(Hi, 0));
1355 }
1356 CurDAG->RemoveDeadNode(N);
1357}
1358
1359bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const {
1360 if (!isUInt<16>(Offset))
1361 return false;
1362
1363 if (!Base || Subtarget->hasUsableDSOffset() ||
1364 Subtarget->unsafeDSOffsetFoldingEnabled())
1365 return true;
1366
1367 // On Southern Islands instruction with a negative base value and an offset
1368 // don't seem to work.
1369 return CurDAG->SignBitIsZero(Base);
1370}
1371
1372bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1373 SDValue &Offset) const {
1374 SDLoc DL(Addr);
1375 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1376 SDValue N0 = Addr.getOperand(0);
1377 SDValue N1 = Addr.getOperand(1);
1378 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1379 if (isDSOffsetLegal(N0, C1->getSExtValue())) {
1380 // (add n0, c0)
1381 Base = N0;
1382 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1383 return true;
1384 }
1385 } else if (Addr.getOpcode() == ISD::SUB) {
1386 // sub C, x -> add (sub 0, x), C
1387 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1388 int64_t ByteOffset = C->getSExtValue();
1389 if (isDSOffsetLegal(SDValue(), ByteOffset)) {
1390 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1391
1392 // XXX - This is kind of hacky. Create a dummy sub node so we can check
1393 // the known bits in isDSOffsetLegal. We need to emit the selected node
1394 // here, so this is thrown away.
1395 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1396 Zero, Addr.getOperand(1));
1397
1398 if (isDSOffsetLegal(Sub, ByteOffset)) {
1400 Opnds.push_back(Zero);
1401 Opnds.push_back(Addr.getOperand(1));
1402
1403 // FIXME: Select to VOP3 version for with-carry.
1404 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1405 if (Subtarget->hasAddNoCarryInsts()) {
1406 SubOp = AMDGPU::V_SUB_U32_e64;
1407 Opnds.push_back(
1408 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1409 }
1410
1411 MachineSDNode *MachineSub =
1412 CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1413
1414 Base = SDValue(MachineSub, 0);
1415 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
1416 return true;
1417 }
1418 }
1419 }
1420 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1421 // If we have a constant address, prefer to put the constant into the
1422 // offset. This can save moves to load the constant address since multiple
1423 // operations can share the zero base address register, and enables merging
1424 // into read2 / write2 instructions.
1425
1426 SDLoc DL(Addr);
1427
1428 if (isDSOffsetLegal(SDValue(), CAddr->getZExtValue())) {
1429 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1430 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1431 DL, MVT::i32, Zero);
1432 Base = SDValue(MovZero, 0);
1433 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1434 return true;
1435 }
1436 }
1437
1438 // default case
1439 Base = Addr;
1440 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1441 return true;
1442}
1443
1444bool AMDGPUDAGToDAGISel::isDSOffset2Legal(SDValue Base, unsigned Offset0,
1445 unsigned Offset1,
1446 unsigned Size) const {
1447 if (Offset0 % Size != 0 || Offset1 % Size != 0)
1448 return false;
1449 if (!isUInt<8>(Offset0 / Size) || !isUInt<8>(Offset1 / Size))
1450 return false;
1451
1452 if (!Base || Subtarget->hasUsableDSOffset() ||
1453 Subtarget->unsafeDSOffsetFoldingEnabled())
1454 return true;
1455
1456 // On Southern Islands instruction with a negative base value and an offset
1457 // don't seem to work.
1458 return CurDAG->SignBitIsZero(Base);
1459}
1460
1461// Return whether the operation has NoUnsignedWrap property.
1462static bool isNoUnsignedWrap(SDValue Addr) {
1463 return (Addr.getOpcode() == ISD::ADD &&
1464 Addr->getFlags().hasNoUnsignedWrap()) ||
1465 Addr->getOpcode() == ISD::OR;
1466}
1467
1468// Check that the base address of flat scratch load/store in the form of `base +
1469// offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
1470// requirement). We always treat the first operand as the base address here.
1471bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegal(SDValue Addr) const {
1472 if (isNoUnsignedWrap(Addr))
1473 return true;
1474
1475 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
1476 // values.
1477 if (Subtarget->hasSignedScratchOffsets())
1478 return true;
1479
1480 auto LHS = Addr.getOperand(0);
1481 auto RHS = Addr.getOperand(1);
1482
1483 // If the immediate offset is negative and within certain range, the base
1484 // address cannot also be negative. If the base is also negative, the sum
1485 // would be either negative or much larger than the valid range of scratch
1486 // memory a thread can access.
1487 ConstantSDNode *ImmOp = nullptr;
1488 if (Addr.getOpcode() == ISD::ADD && (ImmOp = dyn_cast<ConstantSDNode>(RHS))) {
1489 if (ImmOp->getSExtValue() < 0 && ImmOp->getSExtValue() > -0x40000000)
1490 return true;
1491 }
1492
1493 return CurDAG->SignBitIsZero(LHS);
1494}
1495
1496// Check address value in SGPR/VGPR are legal for flat scratch in the form
1497// of: SGPR + VGPR.
1498bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSV(SDValue Addr) const {
1499 if (isNoUnsignedWrap(Addr))
1500 return true;
1501
1502 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
1503 // values.
1504 if (Subtarget->hasSignedScratchOffsets())
1505 return true;
1506
1507 auto LHS = Addr.getOperand(0);
1508 auto RHS = Addr.getOperand(1);
1509 return CurDAG->SignBitIsZero(RHS) && CurDAG->SignBitIsZero(LHS);
1510}
1511
1512// Check address value in SGPR/VGPR are legal for flat scratch in the form
1513// of: SGPR + VGPR + Imm.
1514bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSVImm(SDValue Addr) const {
1515 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
1516 // values.
1517 if (AMDGPU::isGFX12Plus(*Subtarget))
1518 return true;
1519
1520 auto Base = Addr.getOperand(0);
1521 auto *RHSImm = cast<ConstantSDNode>(Addr.getOperand(1));
1522 // If the immediate offset is negative and within certain range, the base
1523 // address cannot also be negative. If the base is also negative, the sum
1524 // would be either negative or much larger than the valid range of scratch
1525 // memory a thread can access.
1526 if (isNoUnsignedWrap(Base) &&
1527 (isNoUnsignedWrap(Addr) ||
1528 (RHSImm->getSExtValue() < 0 && RHSImm->getSExtValue() > -0x40000000)))
1529 return true;
1530
1531 auto LHS = Base.getOperand(0);
1532 auto RHS = Base.getOperand(1);
1533 return CurDAG->SignBitIsZero(RHS) && CurDAG->SignBitIsZero(LHS);
1534}
1535
1536// TODO: If offset is too big, put low 16-bit into offset.
1537bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1538 SDValue &Offset0,
1539 SDValue &Offset1) const {
1540 return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 4);
1541}
1542
1543bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base,
1544 SDValue &Offset0,
1545 SDValue &Offset1) const {
1546 return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 8);
1547}
1548
1549bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
1550 SDValue &Offset0, SDValue &Offset1,
1551 unsigned Size) const {
1552 SDLoc DL(Addr);
1553
1554 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1555 SDValue N0 = Addr.getOperand(0);
1556 SDValue N1 = Addr.getOperand(1);
1557 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1558 unsigned OffsetValue0 = C1->getZExtValue();
1559 unsigned OffsetValue1 = OffsetValue0 + Size;
1560
1561 // (add n0, c0)
1562 if (isDSOffset2Legal(N0, OffsetValue0, OffsetValue1, Size)) {
1563 Base = N0;
1564 Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i32);
1565 Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i32);
1566 return true;
1567 }
1568 } else if (Addr.getOpcode() == ISD::SUB) {
1569 // sub C, x -> add (sub 0, x), C
1570 if (const ConstantSDNode *C =
1572 unsigned OffsetValue0 = C->getZExtValue();
1573 unsigned OffsetValue1 = OffsetValue0 + Size;
1574
1575 if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1576 SDLoc DL(Addr);
1577 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1578
1579 // XXX - This is kind of hacky. Create a dummy sub node so we can check
1580 // the known bits in isDSOffsetLegal. We need to emit the selected node
1581 // here, so this is thrown away.
1582 SDValue Sub =
1583 CurDAG->getNode(ISD::SUB, DL, MVT::i32, Zero, Addr.getOperand(1));
1584
1585 if (isDSOffset2Legal(Sub, OffsetValue0, OffsetValue1, Size)) {
1587 Opnds.push_back(Zero);
1588 Opnds.push_back(Addr.getOperand(1));
1589 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1590 if (Subtarget->hasAddNoCarryInsts()) {
1591 SubOp = AMDGPU::V_SUB_U32_e64;
1592 Opnds.push_back(
1593 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1594 }
1595
1596 MachineSDNode *MachineSub = CurDAG->getMachineNode(
1597 SubOp, DL, MVT::getIntegerVT(Size * 8), Opnds);
1598
1599 Base = SDValue(MachineSub, 0);
1600 Offset0 =
1601 CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i32);
1602 Offset1 =
1603 CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i32);
1604 return true;
1605 }
1606 }
1607 }
1608 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1609 unsigned OffsetValue0 = CAddr->getZExtValue();
1610 unsigned OffsetValue1 = OffsetValue0 + Size;
1611
1612 if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1613 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1614 MachineSDNode *MovZero =
1615 CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, Zero);
1616 Base = SDValue(MovZero, 0);
1617 Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i32);
1618 Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i32);
1619 return true;
1620 }
1621 }
1622
1623 // default case
1624
1625 Base = Addr;
1626 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i32);
1627 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i32);
1628 return true;
1629}
1630
1631bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr,
1632 SDValue &SOffset, SDValue &Offset,
1633 SDValue &Offen, SDValue &Idxen,
1634 SDValue &Addr64) const {
1635 // Subtarget prefers to use flat instruction
1636 // FIXME: This should be a pattern predicate and not reach here
1637 if (Subtarget->useFlatForGlobal())
1638 return false;
1639
1640 Addr = Expand32BitAddress(Addr);
1641 SDLoc DL(Addr);
1642
1643 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1644 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1645 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1646 SOffset = Subtarget->hasRestrictedSOffset()
1647 ? CurDAG->getRegister(AMDGPU::SGPR_NULL, MVT::i32)
1648 : CurDAG->getTargetConstant(0, DL, MVT::i32);
1649
1650 ConstantSDNode *C1 = nullptr;
1651 SDValue N0 = Addr;
1652 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1654 if (isUInt<32>(C1->getZExtValue()))
1655 N0 = Addr.getOperand(0);
1656 else
1657 C1 = nullptr;
1658 }
1659
1660 if (N0->isAnyAdd()) {
1661 // (add N2, N3) -> addr64, or
1662 // (add (add N2, N3), C1) -> addr64
1663 SDValue N2 = N0.getOperand(0);
1664 SDValue N3 = N0.getOperand(1);
1665 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1666
1667 if (N2->isDivergent()) {
1668 if (N3->isDivergent()) {
1669 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1670 // addr64, and construct the resource from a 0 address.
1671 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1672 VAddr = N0;
1673 } else {
1674 // N2 is divergent, N3 is not.
1675 Ptr = N3;
1676 VAddr = N2;
1677 }
1678 } else {
1679 // N2 is not divergent.
1680 Ptr = N2;
1681 VAddr = N3;
1682 }
1683 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1684 } else if (N0->isDivergent()) {
1685 // N0 is divergent. Use it as the addr64, and construct the resource from a
1686 // 0 address.
1687 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1688 VAddr = N0;
1689 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1690 } else {
1691 // N0 -> offset, or
1692 // (N0 + C1) -> offset
1693 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1694 Ptr = N0;
1695 }
1696
1697 if (!C1) {
1698 // No offset.
1699 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1700 return true;
1701 }
1702
1703 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1704 if (TII->isLegalMUBUFImmOffset(C1->getZExtValue())) {
1705 // Legal offset for instruction.
1706 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
1707 return true;
1708 }
1709
1710 // Illegal offset, store it in soffset.
1711 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1712 SOffset =
1713 SDValue(CurDAG->getMachineNode(
1714 AMDGPU::S_MOV_B32, DL, MVT::i32,
1715 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1716 0);
1717 return true;
1718}
1719
1720bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1721 SDValue &VAddr, SDValue &SOffset,
1722 SDValue &Offset) const {
1723 SDValue Ptr, Offen, Idxen, Addr64;
1724
1725 // addr64 bit was removed for volcanic islands.
1726 // FIXME: This should be a pattern predicate and not reach here
1727 if (!Subtarget->hasAddr64())
1728 return false;
1729
1730 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1731 return false;
1732
1733 ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1734 if (C->getSExtValue()) {
1735 SDLoc DL(Addr);
1736
1737 const SITargetLowering& Lowering =
1738 *static_cast<const SITargetLowering*>(getTargetLowering());
1739
1740 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1741 return true;
1742 }
1743
1744 return false;
1745}
1746
1747std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1748 SDLoc DL(N);
1749
1750 auto *FI = dyn_cast<FrameIndexSDNode>(N);
1751 SDValue TFI =
1752 FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N;
1753
1754 // We rebase the base address into an absolute stack address and hence
1755 // use constant 0 for soffset. This value must be retained until
1756 // frame elimination and eliminateFrameIndex will choose the appropriate
1757 // frame register if need be.
1758 return std::pair(TFI, CurDAG->getTargetConstant(0, DL, MVT::i32));
1759}
1760
1761bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1762 SDValue Addr, SDValue &Rsrc,
1763 SDValue &VAddr, SDValue &SOffset,
1764 SDValue &ImmOffset) const {
1765
1766 SDLoc DL(Addr);
1767 MachineFunction &MF = CurDAG->getMachineFunction();
1768 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1769
1770 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1771
1772 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1773 int64_t Imm = CAddr->getSExtValue();
1774 const int64_t NullPtr =
1776 // Don't fold null pointer.
1777 if (Imm != NullPtr) {
1778 const int64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(*Subtarget);
1779 SDValue HighBits =
1780 CurDAG->getTargetConstant(Imm & ~MaxOffset, DL, MVT::i32);
1781 MachineSDNode *MovHighBits = CurDAG->getMachineNode(
1782 AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits);
1783 VAddr = SDValue(MovHighBits, 0);
1784
1785 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1786 ImmOffset = CurDAG->getTargetConstant(Imm & MaxOffset, DL, MVT::i32);
1787 return true;
1788 }
1789 }
1790
1791 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1792 // (add n0, c1)
1793
1794 SDValue N0 = Addr.getOperand(0);
1796
1797 // Offsets in vaddr must be positive if range checking is enabled.
1798 //
1799 // The total computation of vaddr + soffset + offset must not overflow. If
1800 // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1801 // overflowing.
1802 //
1803 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1804 // always perform a range check. If a negative vaddr base index was used,
1805 // this would fail the range check. The overall address computation would
1806 // compute a valid address, but this doesn't happen due to the range
1807 // check. For out-of-bounds MUBUF loads, a 0 is returned.
1808 //
1809 // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1810 // MUBUF vaddr, but not on older subtargets which can only do this if the
1811 // sign bit is known 0.
1812 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1813 if (TII->isLegalMUBUFImmOffset(C1) &&
1814 (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1815 CurDAG->SignBitIsZero(N0))) {
1816 std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1817 ImmOffset = CurDAG->getTargetConstant(C1, DL, MVT::i32);
1818 return true;
1819 }
1820 }
1821
1822 // (node)
1823 std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1824 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1825 return true;
1826}
1827
1828static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val) {
1829 if (Val.getOpcode() != ISD::CopyFromReg)
1830 return false;
1831 auto Reg = cast<RegisterSDNode>(Val.getOperand(1))->getReg();
1832 if (!Reg.isPhysical())
1833 return false;
1834 const auto *RC = TRI.getPhysRegBaseClass(Reg);
1835 return RC && TRI.isSGPRClass(RC);
1836}
1837
1838bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1839 SDValue Addr,
1840 SDValue &SRsrc,
1841 SDValue &SOffset,
1842 SDValue &Offset) const {
1843 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
1844 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1845 MachineFunction &MF = CurDAG->getMachineFunction();
1846 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1847 SDLoc DL(Addr);
1848
1849 // CopyFromReg <sgpr>
1850 if (IsCopyFromSGPR(*TRI, Addr)) {
1851 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1852 SOffset = Addr;
1853 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1854 return true;
1855 }
1856
1857 ConstantSDNode *CAddr;
1858 if (Addr.getOpcode() == ISD::ADD) {
1859 // Add (CopyFromReg <sgpr>) <constant>
1860 CAddr = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
1861 if (!CAddr || !TII->isLegalMUBUFImmOffset(CAddr->getZExtValue()))
1862 return false;
1863 if (!IsCopyFromSGPR(*TRI, Addr.getOperand(0)))
1864 return false;
1865
1866 SOffset = Addr.getOperand(0);
1867 } else if ((CAddr = dyn_cast<ConstantSDNode>(Addr)) &&
1868 TII->isLegalMUBUFImmOffset(CAddr->getZExtValue())) {
1869 // <constant>
1870 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1871 } else {
1872 return false;
1873 }
1874
1875 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1876
1877 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i32);
1878 return true;
1879}
1880
1881bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1882 SDValue &SOffset, SDValue &Offset
1883 ) const {
1884 SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1885 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1886
1887 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1888 return false;
1889
1890 if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1891 !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1892 !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1893 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1894 maskTrailingOnes<uint64_t>(32); // Size
1895 SDLoc DL(Addr);
1896
1897 const SITargetLowering& Lowering =
1898 *static_cast<const SITargetLowering*>(getTargetLowering());
1899
1900 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1901 return true;
1902 }
1903 return false;
1904}
1905
1906bool AMDGPUDAGToDAGISel::SelectBUFSOffset(SDValue ByteOffsetNode,
1907 SDValue &SOffset) const {
1908 if (Subtarget->hasRestrictedSOffset() && isNullConstant(ByteOffsetNode)) {
1909 SOffset = CurDAG->getRegister(AMDGPU::SGPR_NULL, MVT::i32);
1910 return true;
1911 }
1912
1913 SOffset = ByteOffsetNode;
1914 return true;
1915}
1916
1917// Find a load or store from corresponding pattern root.
1918// Roots may be build_vector, bitconvert or their combinations.
1921 if (MemSDNode *MN = dyn_cast<MemSDNode>(N))
1922 return MN;
1924 for (SDValue V : N->op_values())
1925 if (MemSDNode *MN =
1927 return MN;
1928 llvm_unreachable("cannot find MemSDNode in the pattern!");
1929}
1930
1931bool AMDGPUDAGToDAGISel::SelectFlatOffsetImpl(
1932 SDNode *N, SDValue Addr, SDValue &VAddr, SDValue &Offset,
1933 AMDGPU::FlatAddrSpace FlatVariant) const {
1935 int64_t OffsetVal = 0;
1936
1937 unsigned AS = findMemSDNode(N)->getAddressSpace();
1939 Addr = Expand32BitAddress(Addr);
1940
1941 bool CanHaveFlatSegmentOffsetBug =
1942 Subtarget->hasFlatSegmentOffsetBug() &&
1943 FlatVariant == FlatAddrSpace::FLAT &&
1945
1946 if (Subtarget->hasFlatInstOffsets() && !CanHaveFlatSegmentOffsetBug) {
1947 SDValue N0, N1;
1948 if (isBaseWithConstantOffset64(Addr, N0, N1) &&
1949 (FlatVariant != FlatAddrSpace::FlatScratch ||
1950 isFlatScratchBaseLegal(Addr))) {
1951 int64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
1952
1953 // Adding the offset to the base address in a FLAT instruction must not
1954 // change the memory aperture in which the address falls. Therefore we can
1955 // only fold offsets from inbounds GEPs into FLAT instructions.
1956 bool IsInBounds =
1957 Addr.getOpcode() == ISD::PTRADD && Addr->getFlags().hasInBounds();
1958 if (COffsetVal == 0 || FlatVariant != FlatAddrSpace::FLAT || IsInBounds) {
1959 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1960 if (TII->isLegalFLATOffset(COffsetVal, AS, FlatVariant)) {
1961 Addr = N0;
1962 OffsetVal = COffsetVal;
1963 } else {
1964 // If the offset doesn't fit, put the low bits into the offset field
1965 // and add the rest.
1966 //
1967 // For a FLAT instruction the hardware decides whether to access
1968 // global/scratch/shared memory based on the high bits of vaddr,
1969 // ignoring the offset field, so we have to ensure that when we add
1970 // remainder to vaddr it still points into the same underlying object.
1971 // The easiest way to do that is to make sure that we split the offset
1972 // into two pieces that are both >= 0 or both <= 0.
1973
1974 SDLoc DL(N);
1975 uint64_t RemainderOffset;
1976
1977 std::tie(OffsetVal, RemainderOffset) =
1978 TII->splitFlatOffset(COffsetVal, AS, FlatVariant);
1979
1980 SDValue AddOffsetLo =
1981 getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1982 SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1983
1984 if (Addr.getValueType().getSizeInBits() == 32) {
1986 Opnds.push_back(N0);
1987 Opnds.push_back(AddOffsetLo);
1988 unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1989 if (Subtarget->hasAddNoCarryInsts()) {
1990 AddOp = AMDGPU::V_ADD_U32_e64;
1991 Opnds.push_back(Clamp);
1992 }
1993 Addr =
1994 SDValue(CurDAG->getMachineNode(AddOp, DL, MVT::i32, Opnds), 0);
1995 } else {
1996 // TODO: Should this try to use a scalar add pseudo if the base
1997 // address is uniform and saddr is usable?
1998 SDValue Sub0 =
1999 CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
2000 SDValue Sub1 =
2001 CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
2002
2003 SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
2004 DL, MVT::i32, N0, Sub0);
2005 SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
2006 DL, MVT::i32, N0, Sub1);
2007
2008 SDValue AddOffsetHi =
2009 getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
2010
2011 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
2012
2013 SDNode *Add =
2014 CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64, DL, VTs,
2015 {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
2016
2017 SDNode *Addc = CurDAG->getMachineNode(
2018 AMDGPU::V_ADDC_U32_e64, DL, VTs,
2019 {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
2020
2021 SDValue RegSequenceArgs[] = {
2022 CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL,
2023 MVT::i32),
2024 SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
2025
2026 Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
2027 MVT::i64, RegSequenceArgs),
2028 0);
2029 }
2030 }
2031 }
2032 }
2033 }
2034
2035 VAddr = Addr;
2036 Offset = CurDAG->getSignedTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2037 return true;
2038}
2039
2040bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, SDValue Addr,
2041 SDValue &VAddr,
2042 SDValue &Offset) const {
2043 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
2045}
2046
2047bool AMDGPUDAGToDAGISel::SelectGlobalOffset(SDNode *N, SDValue Addr,
2048 SDValue &VAddr,
2049 SDValue &Offset) const {
2050 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
2052}
2053
2054bool AMDGPUDAGToDAGISel::SelectScratchOffset(SDNode *N, SDValue Addr,
2055 SDValue &VAddr,
2056 SDValue &Offset) const {
2057 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
2059}
2060
2061// If this matches *_extend i32:x, return x
2062// Otherwise if the value is I32 returns x.
2064 const SelectionDAG *DAG) {
2065 if (Op.getValueType() == MVT::i32)
2066 return Op;
2067
2068 if (Op.getOpcode() != (IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND) &&
2069 Op.getOpcode() != ISD::ANY_EXTEND &&
2070 !(Op.getOpcode() == (IsSigned ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND) &&
2071 DAG->SignBitIsZero(Op.getOperand(0))))
2072 return SDValue();
2073
2074 SDValue ExtSrc = Op.getOperand(0);
2075 return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
2076}
2077
2078// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
2079// or (64-bit SGPR base) + (sext vgpr offset) + sext(imm offset)
2080bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
2081 SDValue &SAddr, SDValue &VOffset,
2082 SDValue &Offset, bool &ScaleOffset,
2083 bool NeedIOffset) const {
2085 int64_t ImmOffset = 0;
2086 ScaleOffset = false;
2087
2088 Addr = Expand32BitAddress(Addr);
2089
2090 // Match the immediate offset first, which canonically is moved as low as
2091 // possible.
2092
2093 SDValue LHS, RHS;
2094 if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
2095 int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
2096 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2097
2098 if (NeedIOffset &&
2099 TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::GLOBAL_ADDRESS,
2100 FlatAddrSpace::FlatGlobal)) {
2101 Addr = LHS;
2102 ImmOffset = COffsetVal;
2103 } else if (!LHS->isDivergent()) {
2104 if (COffsetVal != 0) {
2105 SDLoc SL(N);
2106 // saddr + large_offset -> saddr +
2107 // (voffset = large_offset & ~MaxOffset) +
2108 // (large_offset & MaxOffset);
2109 int64_t SplitImmOffset = 0, RemainderOffset = COffsetVal;
2110 if (NeedIOffset) {
2111 std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
2112 COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, FlatAddrSpace::FlatGlobal);
2113 }
2114
2115 if (Subtarget->hasSignedGVSOffset() ? isInt<32>(RemainderOffset)
2116 : isUInt<32>(RemainderOffset)) {
2117 SDNode *VMov = CurDAG->getMachineNode(
2118 AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
2119 CurDAG->getTargetConstant(Lo_32(RemainderOffset), SDLoc(),
2120 MVT::i32));
2121 VOffset = SDValue(VMov, 0);
2122 SAddr = LHS;
2123 Offset = CurDAG->getSignedTargetConstant(SplitImmOffset, SDLoc(),
2124 MVT::i32);
2125 return true;
2126 }
2127 }
2128
2129 // We are adding a 64 bit SGPR and a constant. If constant bus limit
2130 // is 1 we would need to perform 1 or 2 extra moves for each half of
2131 // the constant and it is better to do a scalar add and then issue a
2132 // single VALU instruction to materialize zero. Otherwise it is less
2133 // instructions to perform VALU adds with immediates or inline literals.
2134 unsigned NumLiterals =
2135 !TII->isInlineConstant(APInt(32, Lo_32(COffsetVal))) +
2136 !TII->isInlineConstant(APInt(32, Hi_32(COffsetVal)));
2137 if (Subtarget->getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals)
2138 return false;
2139 }
2140 }
2141
2142 // Match the variable offset.
2143 if (Addr->isAnyAdd()) {
2144 LHS = Addr.getOperand(0);
2145
2146 if (!LHS->isDivergent()) {
2147 // add (i64 sgpr), (*_extend (i32 vgpr))
2148 RHS = Addr.getOperand(1);
2149 ScaleOffset = SelectScaleOffset(N, RHS, Subtarget->hasSignedGVSOffset());
2150 if (SDValue ExtRHS = matchExtFromI32orI32(
2151 RHS, Subtarget->hasSignedGVSOffset(), CurDAG)) {
2152 SAddr = LHS;
2153 VOffset = ExtRHS;
2154 }
2155 }
2156
2157 RHS = Addr.getOperand(1);
2158 if (!SAddr && !RHS->isDivergent()) {
2159 // add (*_extend (i32 vgpr)), (i64 sgpr)
2160 ScaleOffset = SelectScaleOffset(N, LHS, Subtarget->hasSignedGVSOffset());
2161 if (SDValue ExtLHS = matchExtFromI32orI32(
2162 LHS, Subtarget->hasSignedGVSOffset(), CurDAG)) {
2163 SAddr = RHS;
2164 VOffset = ExtLHS;
2165 }
2166 }
2167
2168 if (SAddr) {
2169 Offset = CurDAG->getSignedTargetConstant(ImmOffset, SDLoc(), MVT::i32);
2170 return true;
2171 }
2172 }
2173
2174 if (Subtarget->hasScaleOffset() &&
2175 (Addr.getOpcode() == (Subtarget->hasSignedGVSOffset()
2178 (Addr.getOpcode() == AMDGPUISD::MAD_U64_U32 &&
2179 CurDAG->SignBitIsZero(Addr.getOperand(0)))) &&
2180 Addr.getOperand(0)->isDivergent() &&
2182 !Addr.getOperand(2)->isDivergent()) {
2183 // mad_u64_u32 (i32 vgpr), (i32 c), (i64 sgpr)
2184 unsigned Size =
2185 (unsigned)cast<MemSDNode>(N)->getMemoryVT().getFixedSizeInBits() / 8;
2186 ScaleOffset = Addr.getConstantOperandVal(1) == Size;
2187 if (ScaleOffset) {
2188 SAddr = Addr.getOperand(2);
2189 VOffset = Addr.getOperand(0);
2190 Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i32);
2191 return true;
2192 }
2193 }
2194
2195 if (Addr->isDivergent() || Addr.isUndef() || isa<ConstantSDNode>(Addr))
2196 return false;
2197
2198 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
2199 // moves required to copy a 64-bit SGPR to VGPR.
2200 SAddr = Addr;
2201 SDNode *VMov =
2202 CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, SDLoc(Addr), MVT::i32,
2203 CurDAG->getTargetConstant(0, SDLoc(), MVT::i32));
2204 VOffset = SDValue(VMov, 0);
2205 Offset = CurDAG->getSignedTargetConstant(ImmOffset, SDLoc(), MVT::i32);
2206 return true;
2207}
2208
2209bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
2210 SDValue &SAddr, SDValue &VOffset,
2211 SDValue &Offset,
2212 SDValue &CPol) const {
2213 bool ScaleOffset;
2214 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset, ScaleOffset))
2215 return false;
2216
2217 CPol = CurDAG->getTargetConstant(ScaleOffset ? AMDGPU::CPol::SCAL : 0,
2218 SDLoc(), MVT::i32);
2219 return true;
2220}
2221
2222bool AMDGPUDAGToDAGISel::SelectGlobalSAddrCPol(SDNode *N, SDValue Addr,
2223 SDValue &SAddr, SDValue &VOffset,
2224 SDValue &Offset,
2225 SDValue &CPol) const {
2226 bool ScaleOffset;
2227 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset, ScaleOffset))
2228 return false;
2229
2230 // We are assuming CPol is always the last operand of the intrinsic.
2231 auto PassedCPol =
2232 N->getConstantOperandVal(N->getNumOperands() - 1) & ~AMDGPU::CPol::SCAL;
2233 CPol = CurDAG->getTargetConstant(
2234 (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | PassedCPol, SDLoc(), MVT::i32);
2235 return true;
2236}
2237
2238bool AMDGPUDAGToDAGISel::SelectGlobalSAddrCPolM0(SDNode *N, SDValue Addr,
2239 SDValue &SAddr,
2240 SDValue &VOffset,
2241 SDValue &Offset,
2242 SDValue &CPol) const {
2243 bool ScaleOffset;
2244 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset, ScaleOffset))
2245 return false;
2246
2247 // We are assuming CPol is second from last operand of the intrinsic.
2248 auto PassedCPol =
2249 N->getConstantOperandVal(N->getNumOperands() - 2) & ~AMDGPU::CPol::SCAL;
2250 CPol = CurDAG->getTargetConstant(
2251 (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | PassedCPol, SDLoc(), MVT::i32);
2252 return true;
2253}
2254
2255bool AMDGPUDAGToDAGISel::SelectGlobalSAddrGLC(SDNode *N, SDValue Addr,
2256 SDValue &SAddr, SDValue &VOffset,
2257 SDValue &Offset,
2258 SDValue &CPol) const {
2259 bool ScaleOffset;
2260 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset, ScaleOffset))
2261 return false;
2262
2263 unsigned CPolVal = (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | AMDGPU::CPol::GLC;
2264 CPol = CurDAG->getTargetConstant(CPolVal, SDLoc(), MVT::i32);
2265 return true;
2266}
2267
2268bool AMDGPUDAGToDAGISel::SelectGlobalSAddrNoIOffset(SDNode *N, SDValue Addr,
2269 SDValue &SAddr,
2270 SDValue &VOffset,
2271 SDValue &CPol) const {
2272 bool ScaleOffset;
2273 SDValue DummyOffset;
2274 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, DummyOffset, ScaleOffset,
2275 false))
2276 return false;
2277
2278 // We are assuming CPol is always the last operand of the intrinsic.
2279 auto PassedCPol =
2280 N->getConstantOperandVal(N->getNumOperands() - 1) & ~AMDGPU::CPol::SCAL;
2281 CPol = CurDAG->getTargetConstant(
2282 (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | PassedCPol, SDLoc(), MVT::i32);
2283 return true;
2284}
2285
2286bool AMDGPUDAGToDAGISel::SelectGlobalSAddrNoIOffsetM0(SDNode *N, SDValue Addr,
2287 SDValue &SAddr,
2288 SDValue &VOffset,
2289 SDValue &CPol) const {
2290 bool ScaleOffset;
2291 SDValue DummyOffset;
2292 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, DummyOffset, ScaleOffset,
2293 false))
2294 return false;
2295
2296 // We are assuming CPol is second from last operand of the intrinsic.
2297 auto PassedCPol =
2298 N->getConstantOperandVal(N->getNumOperands() - 2) & ~AMDGPU::CPol::SCAL;
2299 CPol = CurDAG->getTargetConstant(
2300 (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | PassedCPol, SDLoc(), MVT::i32);
2301 return true;
2302}
2303
2305 if (auto *FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
2306 SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
2307 } else if (SAddr.getOpcode() == ISD::ADD &&
2309 // Materialize this into a scalar move for scalar address to avoid
2310 // readfirstlane.
2311 auto *FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
2312 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
2313 FI->getValueType(0));
2314 SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, SDLoc(SAddr),
2315 MVT::i32, TFI, SAddr.getOperand(1)),
2316 0);
2317 }
2318
2319 return SAddr;
2320}
2321
2322// Match (32-bit SGPR base) + sext(imm offset)
2323bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *Parent, SDValue Addr,
2324 SDValue &SAddr,
2325 SDValue &Offset) const {
2327 if (Addr->isDivergent())
2328 return false;
2329
2330 SDLoc DL(Addr);
2331
2332 int64_t COffsetVal = 0;
2333
2334 if (CurDAG->isBaseWithConstantOffset(Addr) && isFlatScratchBaseLegal(Addr)) {
2335 COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2336 SAddr = Addr.getOperand(0);
2337 } else {
2338 SAddr = Addr;
2339 }
2340
2341 SAddr = SelectSAddrFI(CurDAG, SAddr);
2342
2343 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2344
2345 if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS,
2346 FlatAddrSpace::FlatScratch)) {
2347 int64_t SplitImmOffset, RemainderOffset;
2348 std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
2349 COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, FlatAddrSpace::FlatScratch);
2350
2351 COffsetVal = SplitImmOffset;
2352
2353 SDValue AddOffset =
2355 ? getMaterializedScalarImm32(Lo_32(RemainderOffset), DL)
2356 : CurDAG->getSignedTargetConstant(RemainderOffset, DL, MVT::i32);
2357 SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, DL, MVT::i32,
2358 SAddr, AddOffset),
2359 0);
2360 }
2361
2362 Offset = CurDAG->getSignedTargetConstant(COffsetVal, DL, MVT::i32);
2363
2364 return true;
2365}
2366
2367// Check whether the flat scratch SVS swizzle bug affects this access.
2368bool AMDGPUDAGToDAGISel::checkFlatScratchSVSSwizzleBug(
2369 SDValue VAddr, SDValue SAddr, uint64_t ImmOffset) const {
2370 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
2371 return false;
2372
2373 // The bug affects the swizzling of SVS accesses if there is any carry out
2374 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
2375 // voffset to (soffset + inst_offset).
2376 KnownBits VKnown = CurDAG->computeKnownBits(VAddr);
2377 KnownBits SKnown =
2378 KnownBits::add(CurDAG->computeKnownBits(SAddr),
2379 KnownBits::makeConstant(APInt(32, ImmOffset,
2380 /*isSigned=*/true)));
2381 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
2383 return (VMax & 3) + (SMax & 3) >= 4;
2384}
2385
2386bool AMDGPUDAGToDAGISel::SelectScratchSVAddr(SDNode *N, SDValue Addr,
2387 SDValue &VAddr, SDValue &SAddr,
2388 SDValue &Offset,
2389 SDValue &CPol) const {
2390 int64_t ImmOffset = 0;
2391
2392 SDValue LHS, RHS;
2393 SDValue OrigAddr = Addr;
2394 if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
2395 int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
2396 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2397
2398 if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS,
2400 Addr = LHS;
2401 ImmOffset = COffsetVal;
2402 } else if (!LHS->isDivergent() && COffsetVal > 0) {
2403 SDLoc SL(N);
2404 // saddr + large_offset -> saddr + (vaddr = large_offset & ~MaxOffset) +
2405 // (large_offset & MaxOffset);
2406 int64_t SplitImmOffset, RemainderOffset;
2407 std::tie(SplitImmOffset, RemainderOffset) =
2408 TII->splitFlatOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS,
2410
2411 if (isUInt<32>(RemainderOffset)) {
2412 SDNode *VMov = CurDAG->getMachineNode(
2413 AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
2414 CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
2415 VAddr = SDValue(VMov, 0);
2416 SAddr = LHS;
2417 if (!isFlatScratchBaseLegal(Addr))
2418 return false;
2419 if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, SplitImmOffset))
2420 return false;
2421 Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i32);
2422 CPol = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2423 return true;
2424 }
2425 }
2426 }
2427
2428 if (Addr.getOpcode() != ISD::ADD)
2429 return false;
2430
2431 LHS = Addr.getOperand(0);
2432 RHS = Addr.getOperand(1);
2433
2434 if (!LHS->isDivergent() && RHS->isDivergent()) {
2435 SAddr = LHS;
2436 VAddr = RHS;
2437 } else if (!RHS->isDivergent() && LHS->isDivergent()) {
2438 SAddr = RHS;
2439 VAddr = LHS;
2440 } else {
2441 return false;
2442 }
2443
2444 if (OrigAddr != Addr) {
2445 if (!isFlatScratchBaseLegalSVImm(OrigAddr))
2446 return false;
2447 } else {
2448 if (!isFlatScratchBaseLegalSV(OrigAddr))
2449 return false;
2450 }
2451
2452 if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, ImmOffset))
2453 return false;
2454 SAddr = SelectSAddrFI(CurDAG, SAddr);
2455 Offset = CurDAG->getSignedTargetConstant(ImmOffset, SDLoc(), MVT::i32);
2456
2457 bool ScaleOffset = SelectScaleOffset(N, VAddr, true /* IsSigned */);
2458 CPol = CurDAG->getTargetConstant(ScaleOffset ? AMDGPU::CPol::SCAL : 0,
2459 SDLoc(), MVT::i32);
2460 return true;
2461}
2462
2463// For unbuffered smem loads, it is illegal for the Immediate Offset to be
2464// negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
2465// Handle the case where the Immediate Offset + SOffset is negative.
2466bool AMDGPUDAGToDAGISel::isSOffsetLegalWithImmOffset(SDValue *SOffset,
2467 bool Imm32Only,
2468 bool IsBuffer,
2469 int64_t ImmOffset) const {
2470 if (!IsBuffer && !Imm32Only && ImmOffset < 0 &&
2471 AMDGPU::hasSMRDSignedImmOffset(*Subtarget)) {
2472 KnownBits SKnown = CurDAG->computeKnownBits(*SOffset);
2473 if (ImmOffset + SKnown.getMinValue().getSExtValue() < 0)
2474 return false;
2475 }
2476
2477 return true;
2478}
2479
2480// Given \p Offset and load node \p N check if an \p Offset is a multiple of
2481// the load byte size. If it is update \p Offset to a pre-scaled value and
2482// return true.
2483bool AMDGPUDAGToDAGISel::SelectScaleOffset(SDNode *N, SDValue &Offset,
2484 bool IsSigned, bool IsSMEM) const {
2485 bool ScaleOffset = false;
2486 if (!Subtarget->hasScaleOffset() || !Offset)
2487 return false;
2488
2489 unsigned Size =
2490 (unsigned)cast<MemSDNode>(N)->getMemoryVT().getFixedSizeInBits() / 8;
2491
2492 SDValue Off = Offset;
2493 if (SDValue Ext = matchExtFromI32orI32(Offset, IsSigned, CurDAG))
2494 Off = Ext;
2495
2496 if (isPowerOf2_32(Size) && Off.getOpcode() == ISD::SHL) {
2497 if (auto *C = dyn_cast<ConstantSDNode>(Off.getOperand(1)))
2498 ScaleOffset = C->getZExtValue() == Log2_32(Size);
2499 } else if (Offset.getOpcode() == ISD::MUL ||
2500 (IsSigned && Offset.getOpcode() == AMDGPUISD::MUL_I24) ||
2501 Offset.getOpcode() == AMDGPUISD::MUL_U24 ||
2502 (Offset.isMachineOpcode() &&
2503 Offset.getMachineOpcode() ==
2504 (IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO
2505 : AMDGPU::S_MUL_U64_U32_PSEUDO))) {
2506 if (auto *C = dyn_cast<ConstantSDNode>(Offset.getOperand(1)))
2507 ScaleOffset = C->getZExtValue() == Size;
2508 }
2509
2510 if (!ScaleOffset)
2511 return false;
2512
2513 // SMEM scales the offset in 32 bits, other instructions in 64 bits.
2514 unsigned Bits = Log2_32_Ceil(Size);
2515 if (IsSMEM) {
2516 if (Off.getValueType() == MVT::i64) {
2517 KnownBits Known = CurDAG->computeKnownBits(Off.getOperand(0));
2518 if (Known.countMaxActiveBits() + Bits > 32)
2519 return false;
2520 }
2521 } else if (Off != Offset) {
2522 SDNodeFlags Flags = Off->getFlags();
2523 SDValue Idx = Off.getOperand(0);
2524 if (IsSigned && !Flags.hasNoSignedWrap() &&
2525 CurDAG->ComputeNumSignBits(Idx) <= Bits)
2526 return false;
2527 if (!IsSigned && !Flags.hasNoUnsignedWrap() &&
2528 CurDAG->computeKnownBits(Idx).countMinLeadingZeros() < Bits)
2529 return false;
2530 }
2531
2532 Offset = Off.getOperand(0);
2533 return true;
2534}
2535
2536// Match an immediate (if Offset is not null) or an SGPR (if SOffset is
2537// not null) offset. If Imm32Only is true, match only 32-bit immediate
2538// offsets available on CI.
2539bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDNode *N, SDValue ByteOffsetNode,
2540 SDValue *SOffset, SDValue *Offset,
2541 bool Imm32Only, bool IsBuffer,
2542 bool HasSOffset, int64_t ImmOffset,
2543 bool *ScaleOffset) const {
2544 assert((!SOffset || !Offset) &&
2545 "Cannot match both soffset and offset at the same time!");
2546
2547 if (ScaleOffset) {
2548 assert(N && SOffset);
2549
2550 *ScaleOffset = SelectScaleOffset(N, ByteOffsetNode, false /* IsSigned */,
2551 true /* IsSMEM */);
2552 }
2553
2554 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
2555 if (!C) {
2556 if (!SOffset)
2557 return false;
2558
2559 if (ByteOffsetNode.getValueType().isScalarInteger() &&
2560 ByteOffsetNode.getValueType().getSizeInBits() == 32) {
2561 *SOffset = ByteOffsetNode;
2562 return isSOffsetLegalWithImmOffset(SOffset, Imm32Only, IsBuffer,
2563 ImmOffset);
2564 }
2565 if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
2566 if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
2567 *SOffset = ByteOffsetNode.getOperand(0);
2568 return isSOffsetLegalWithImmOffset(SOffset, Imm32Only, IsBuffer,
2569 ImmOffset);
2570 }
2571 }
2572 return false;
2573 }
2574
2575 SDLoc SL(ByteOffsetNode);
2576
2577 // GFX9 and GFX10 have signed byte immediate offsets. The immediate
2578 // offset for S_BUFFER instructions is unsigned.
2579 int64_t ByteOffset = IsBuffer ? C->getZExtValue() : C->getSExtValue();
2580 std::optional<int64_t> EncodedOffset = AMDGPU::getSMRDEncodedOffset(
2581 *Subtarget, ByteOffset, IsBuffer, HasSOffset);
2582 if (EncodedOffset && Offset && !Imm32Only) {
2583 *Offset = CurDAG->getSignedTargetConstant(*EncodedOffset, SL, MVT::i32);
2584 return true;
2585 }
2586
2587 // SGPR and literal offsets are unsigned.
2588 if (ByteOffset < 0)
2589 return false;
2590
2591 EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
2592 if (EncodedOffset && Offset && Imm32Only) {
2593 *Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
2594 return true;
2595 }
2596
2597 if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset))
2598 return false;
2599
2600 if (SOffset) {
2601 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
2602 *SOffset = SDValue(
2603 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0);
2604 return true;
2605 }
2606
2607 return false;
2608}
2609
2610SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
2611 if (Addr.getValueType() != MVT::i32)
2612 return Addr;
2613
2614 // Zero-extend a 32-bit address.
2615 SDLoc SL(Addr);
2616
2617 const MachineFunction &MF = CurDAG->getMachineFunction();
2618 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2619 unsigned AddrHiVal = Info->get32BitAddressHighBits();
2620 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
2621
2622 const SDValue Ops[] = {
2623 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
2624 Addr,
2625 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
2626 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
2627 0),
2628 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
2629 };
2630
2631 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
2632 Ops), 0);
2633}
2634
2635// Match a base and an immediate (if Offset is not null) or an SGPR (if
2636// SOffset is not null) or an immediate+SGPR offset. If Imm32Only is
2637// true, match only 32-bit immediate offsets available on CI.
2638bool AMDGPUDAGToDAGISel::SelectSMRDBaseOffset(SDNode *N, SDValue Addr,
2639 SDValue &SBase, SDValue *SOffset,
2640 SDValue *Offset, bool Imm32Only,
2641 bool IsBuffer, bool HasSOffset,
2642 int64_t ImmOffset,
2643 bool *ScaleOffset) const {
2644 if (SOffset && Offset) {
2645 assert(!Imm32Only && !IsBuffer);
2646 SDValue B;
2647
2648 if (!SelectSMRDBaseOffset(N, Addr, B, nullptr, Offset, false, false, true))
2649 return false;
2650
2651 int64_t ImmOff = 0;
2652 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(*Offset))
2653 ImmOff = C->getSExtValue();
2654
2655 return SelectSMRDBaseOffset(N, B, SBase, SOffset, nullptr, false, false,
2656 true, ImmOff, ScaleOffset);
2657 }
2658
2659 // A 32-bit (address + offset) should not cause unsigned 32-bit integer
2660 // wraparound, because s_load instructions perform the addition in 64 bits.
2661 if (Addr.getValueType() == MVT::i32 && Addr.getOpcode() == ISD::ADD &&
2662 !Addr->getFlags().hasNoUnsignedWrap())
2663 return false;
2664
2665 SDValue N0, N1;
2666 // Extract the base and offset if possible.
2667 if (Addr->isAnyAdd() || CurDAG->isADDLike(Addr)) {
2668 N0 = Addr.getOperand(0);
2669 N1 = Addr.getOperand(1);
2670 } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
2671 assert(N0 && N1 && isa<ConstantSDNode>(N1));
2672 }
2673 if (!N0 || !N1)
2674 return false;
2675
2676 if (SelectSMRDOffset(N, N1, SOffset, Offset, Imm32Only, IsBuffer, HasSOffset,
2677 ImmOffset, ScaleOffset)) {
2678 SBase = N0;
2679 return true;
2680 }
2681 if (SelectSMRDOffset(N, N0, SOffset, Offset, Imm32Only, IsBuffer, HasSOffset,
2682 ImmOffset, ScaleOffset)) {
2683 SBase = N1;
2684 return true;
2685 }
2686 return false;
2687}
2688
2689bool AMDGPUDAGToDAGISel::SelectSMRD(SDNode *N, SDValue Addr, SDValue &SBase,
2690 SDValue *SOffset, SDValue *Offset,
2691 bool Imm32Only, bool *ScaleOffset) const {
2692 if (SelectSMRDBaseOffset(N, Addr, SBase, SOffset, Offset, Imm32Only,
2693 /* IsBuffer */ false, /* HasSOffset */ false,
2694 /* ImmOffset */ 0, ScaleOffset)) {
2695 SBase = Expand32BitAddress(SBase);
2696 return true;
2697 }
2698
2699 if (Addr.getValueType() == MVT::i32 && Offset && !SOffset) {
2700 SBase = Expand32BitAddress(Addr);
2701 *Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
2702 return true;
2703 }
2704
2705 return false;
2706}
2707
2708bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
2709 SDValue &Offset) const {
2710 return SelectSMRD(/* N */ nullptr, Addr, SBase, /* SOffset */ nullptr,
2711 &Offset);
2712}
2713
2714bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
2715 SDValue &Offset) const {
2716 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2717 return SelectSMRD(/* N */ nullptr, Addr, SBase, /* SOffset */ nullptr,
2718 &Offset, /* Imm32Only */ true);
2719}
2720
2721bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDNode *N, SDValue Addr, SDValue &SBase,
2722 SDValue &SOffset, SDValue &CPol) const {
2723 bool ScaleOffset;
2724 if (!SelectSMRD(N, Addr, SBase, &SOffset, /* Offset */ nullptr,
2725 /* Imm32Only */ false, &ScaleOffset))
2726 return false;
2727
2728 CPol = CurDAG->getTargetConstant(ScaleOffset ? AMDGPU::CPol::SCAL : 0,
2729 SDLoc(N), MVT::i32);
2730 return true;
2731}
2732
2733bool AMDGPUDAGToDAGISel::SelectSMRDSgprImm(SDNode *N, SDValue Addr,
2734 SDValue &SBase, SDValue &SOffset,
2735 SDValue &Offset,
2736 SDValue &CPol) const {
2737 bool ScaleOffset;
2738 if (!SelectSMRD(N, Addr, SBase, &SOffset, &Offset, false, &ScaleOffset))
2739 return false;
2740
2741 CPol = CurDAG->getTargetConstant(ScaleOffset ? AMDGPU::CPol::SCAL : 0,
2742 SDLoc(N), MVT::i32);
2743 return true;
2744}
2745
2746bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue N, SDValue &Offset) const {
2747 return SelectSMRDOffset(/* N */ nullptr, N, /* SOffset */ nullptr, &Offset,
2748 /* Imm32Only */ false, /* IsBuffer */ true);
2749}
2750
2751bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue N,
2752 SDValue &Offset) const {
2753 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2754 return SelectSMRDOffset(/* N */ nullptr, N, /* SOffset */ nullptr, &Offset,
2755 /* Imm32Only */ true, /* IsBuffer */ true);
2756}
2757
2758bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
2759 SDValue &Offset) const {
2760 // Match the (soffset + offset) pair as a 32-bit register base and
2761 // an immediate offset.
2762 return N.getValueType() == MVT::i32 &&
2763 SelectSMRDBaseOffset(/* N */ nullptr, N, /* SBase */ SOffset,
2764 /* SOffset*/ nullptr, &Offset,
2765 /* Imm32Only */ false, /* IsBuffer */ true);
2766}
2767
2768bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
2769 SDValue &Base,
2770 SDValue &Offset) const {
2771 SDLoc DL(Index);
2772
2773 if (CurDAG->isBaseWithConstantOffset(Index)) {
2774 SDValue N0 = Index.getOperand(0);
2775 SDValue N1 = Index.getOperand(1);
2776 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
2777
2778 // (add n0, c0)
2779 // Don't peel off the offset (c0) if doing so could possibly lead
2780 // the base (n0) to be negative.
2781 // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
2782 if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) ||
2783 (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
2784 Base = N0;
2785 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
2786 return true;
2787 }
2788 }
2789
2790 if (isa<ConstantSDNode>(Index))
2791 return false;
2792
2793 Base = Index;
2794 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2795 return true;
2796}
2797
2798SDNode *AMDGPUDAGToDAGISel::getBFE32(bool IsSigned, const SDLoc &DL,
2799 SDValue Val, uint32_t Offset,
2800 uint32_t Width) {
2801 if (Val->isDivergent()) {
2802 unsigned Opcode = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2803 SDValue Off = CurDAG->getTargetConstant(Offset, DL, MVT::i32);
2804 SDValue W = CurDAG->getTargetConstant(Width, DL, MVT::i32);
2805
2806 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, Off, W);
2807 }
2808 unsigned Opcode = IsSigned ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2809 // Transformation function, pack the offset and width of a BFE into
2810 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
2811 // source, bits [5:0] contain the offset and bits [22:16] the width.
2812 uint32_t PackedVal = Offset | (Width << 16);
2813 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
2814
2815 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
2816}
2817
2818void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
2819 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
2820 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
2821 // Predicate: 0 < b <= c < 32
2822
2823 const SDValue &Shl = N->getOperand(0);
2824 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
2825 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2826
2827 if (B && C) {
2828 uint32_t BVal = B->getZExtValue();
2829 uint32_t CVal = C->getZExtValue();
2830
2831 if (0 < BVal && BVal <= CVal && CVal < 32) {
2832 bool Signed = N->getOpcode() == ISD::SRA;
2833 ReplaceNode(N, getBFE32(Signed, SDLoc(N), Shl.getOperand(0), CVal - BVal,
2834 32 - CVal));
2835 return;
2836 }
2837 }
2838 SelectCode(N);
2839}
2840
2841void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
2842 switch (N->getOpcode()) {
2843 case ISD::AND:
2844 if (N->getOperand(0).getOpcode() == ISD::SRL) {
2845 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
2846 // Predicate: isMask(mask)
2847 const SDValue &Srl = N->getOperand(0);
2848 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
2849 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
2850
2851 if (Shift && Mask) {
2852 uint32_t ShiftVal = Shift->getZExtValue();
2853 uint32_t MaskVal = Mask->getZExtValue();
2854
2855 if (isMask_32(MaskVal)) {
2856 uint32_t WidthVal = llvm::popcount(MaskVal);
2857 ReplaceNode(N, getBFE32(false, SDLoc(N), Srl.getOperand(0), ShiftVal,
2858 WidthVal));
2859 return;
2860 }
2861 }
2862 }
2863 break;
2864 case ISD::SRL:
2865 if (N->getOperand(0).getOpcode() == ISD::AND) {
2866 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
2867 // Predicate: isMask(mask >> b)
2868 const SDValue &And = N->getOperand(0);
2869 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
2870 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
2871
2872 if (Shift && Mask) {
2873 uint32_t ShiftVal = Shift->getZExtValue();
2874 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
2875
2876 if (isMask_32(MaskVal)) {
2877 uint32_t WidthVal = llvm::popcount(MaskVal);
2878 ReplaceNode(N, getBFE32(false, SDLoc(N), And.getOperand(0), ShiftVal,
2879 WidthVal));
2880 return;
2881 }
2882 }
2883 } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
2884 SelectS_BFEFromShifts(N);
2885 return;
2886 }
2887 break;
2888 case ISD::SRA:
2889 if (N->getOperand(0).getOpcode() == ISD::SHL) {
2890 SelectS_BFEFromShifts(N);
2891 return;
2892 }
2893 break;
2894
2896 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
2897 SDValue Src = N->getOperand(0);
2898 if (Src.getOpcode() != ISD::SRL)
2899 break;
2900
2901 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
2902 if (!Amt)
2903 break;
2904
2905 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2906 ReplaceNode(N, getBFE32(true, SDLoc(N), Src.getOperand(0),
2907 Amt->getZExtValue(), Width));
2908 return;
2909 }
2910 }
2911
2912 SelectCode(N);
2913}
2914
2915bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
2916 assert(N->getOpcode() == ISD::BRCOND);
2917 if (!N->hasOneUse())
2918 return false;
2919
2920 SDValue Cond = N->getOperand(1);
2921 if (Cond.getOpcode() == ISD::CopyToReg)
2922 Cond = Cond.getOperand(2);
2923
2924 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
2925 return false;
2926
2927 MVT VT = Cond.getOperand(0).getSimpleValueType();
2928 if (VT == MVT::i32)
2929 return true;
2930
2931 if (VT == MVT::i64) {
2932 ISD::CondCode CC = Cond.getOperandCondCode(2);
2933 return (CC == ISD::SETEQ || CC == ISD::SETNE) &&
2934 Subtarget->hasScalarCompareEq64();
2935 }
2936
2937 if ((VT == MVT::f16 || VT == MVT::f32) && Subtarget->hasSALUFloatInsts())
2938 return true;
2939
2940 return false;
2941}
2942
2943static SDValue combineBallotPattern(SDValue VCMP, bool &Negate) {
2944 assert(VCMP->getOpcode() == AMDGPUISD::SETCC);
2945 // Special case for amdgcn.ballot:
2946 // %Cond = i1 (and/or combination of i1 ISD::SETCCs)
2947 // %VCMP = i(WaveSize) AMDGPUISD::SETCC (ext %Cond), 0, setne/seteq
2948 // =>
2949 // Use i1 %Cond value instead of i(WaveSize) %VCMP.
2950 // This is possible because divergent ISD::SETCC is selected as V_CMP and
2951 // Cond becomes a i(WaveSize) full mask value.
2952 // Note that ballot doesn't use SETEQ condition but its easy to support it
2953 // here for completeness, so in this case Negate is set true on return.
2954 auto VCMP_CC = VCMP.getOperandCondCode(2);
2955 if ((VCMP_CC == ISD::SETEQ || VCMP_CC == ISD::SETNE) &&
2956 isNullConstant(VCMP.getOperand(1))) {
2957
2958 auto Cond = VCMP.getOperand(0);
2959 if (ISD::isExtOpcode(Cond->getOpcode())) // Skip extension.
2960 Cond = Cond.getOperand(0);
2961
2962 if (isBoolSGPR(Cond)) {
2963 Negate = VCMP_CC == ISD::SETEQ;
2964 return Cond;
2965 }
2966 }
2967 return SDValue();
2968}
2969
2970void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
2971 SDValue Cond = N->getOperand(1);
2972
2973 if (Cond.isUndef()) {
2974 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
2975 N->getOperand(2), N->getOperand(0));
2976 return;
2977 }
2978
2979 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
2980
2981 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
2982 bool AndExec = !UseSCCBr;
2983 bool Negate = false;
2984
2985 if (Cond.getOpcode() == ISD::SETCC &&
2986 Cond->getOperand(0)->getOpcode() == AMDGPUISD::SETCC) {
2987 SDValue VCMP = Cond->getOperand(0);
2988 auto CC = Cond->getOperandCondCode(2);
2989 if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
2990 isNullConstant(Cond->getOperand(1)) &&
2991 // We may encounter ballot.i64 in wave32 mode on -O0.
2992 VCMP.getValueType().getSizeInBits() == Subtarget->getWavefrontSize()) {
2993 // %VCMP = i(WaveSize) AMDGPUISD::SETCC ...
2994 // %C = i1 ISD::SETCC %VCMP, 0, setne/seteq
2995 // BRCOND i1 %C, %BB
2996 // =>
2997 // %VCMP = i(WaveSize) AMDGPUISD::SETCC ...
2998 // VCC = COPY i(WaveSize) %VCMP
2999 // S_CBRANCH_VCCNZ/VCCZ %BB
3000 Negate = CC == ISD::SETEQ;
3001 bool NegatedBallot = false;
3002 if (auto BallotCond = combineBallotPattern(VCMP, NegatedBallot)) {
3003 Cond = BallotCond;
3004 UseSCCBr = !BallotCond->isDivergent();
3005 Negate = Negate ^ NegatedBallot;
3006 } else {
3007 // TODO: don't use SCC here assuming that AMDGPUISD::SETCC is always
3008 // selected as V_CMP, but this may change for uniform condition.
3009 Cond = VCMP;
3010 UseSCCBr = false;
3011 }
3012 }
3013 // Cond is either V_CMP resulted from AMDGPUISD::SETCC or a combination of
3014 // V_CMPs resulted from ballot or ballot has uniform condition and SCC is
3015 // used.
3016 AndExec = false;
3017 }
3018
3019 unsigned BrOp =
3020 UseSCCBr ? (Negate ? AMDGPU::S_CBRANCH_SCC0 : AMDGPU::S_CBRANCH_SCC1)
3021 : (Negate ? AMDGPU::S_CBRANCH_VCCZ : AMDGPU::S_CBRANCH_VCCNZ);
3022 Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
3023 SDLoc SL(N);
3024
3025 if (AndExec) {
3026 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not
3027 // analyzed what generates the vcc value, so we do not know whether vcc
3028 // bits for disabled lanes are 0. Thus we need to mask out bits for
3029 // disabled lanes.
3030 //
3031 // For the case that we select S_CBRANCH_SCC1 and it gets
3032 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
3033 // SIInstrInfo::moveToVALU which inserts the S_AND).
3034 //
3035 // We could add an analysis of what generates the vcc value here and omit
3036 // the S_AND when is unnecessary. But it would be better to add a separate
3037 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
3038 // catches both cases.
3039 Cond = SDValue(
3040 CurDAG->getMachineNode(
3041 Subtarget->isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64, SL,
3042 MVT::i1,
3043 CurDAG->getRegister(Subtarget->isWave32() ? AMDGPU::EXEC_LO
3044 : AMDGPU::EXEC,
3045 MVT::i1),
3046 Cond),
3047 0);
3048 }
3049
3050 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
3051 CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
3052 N->getOperand(2), // Basic Block
3053 VCC.getValue(0));
3054}
3055
3056void AMDGPUDAGToDAGISel::SelectFP_EXTEND(SDNode *N) {
3057 if (Subtarget->hasSALUFloatInsts() && N->getValueType(0) == MVT::f32 &&
3058 !N->isDivergent()) {
3059 SDValue Src = N->getOperand(0);
3060 if (Src.getValueType() == MVT::f16) {
3061 if (isExtractHiElt(Src, Src)) {
3062 CurDAG->SelectNodeTo(N, AMDGPU::S_CVT_HI_F32_F16, N->getVTList(),
3063 {Src});
3064 return;
3065 }
3066 }
3067 }
3068
3069 SelectCode(N);
3070}
3071
3072void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
3073 // The address is assumed to be uniform, so if it ends up in a VGPR, it will
3074 // be copied to an SGPR with readfirstlane.
3075 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
3076 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
3077
3078 SDValue Chain = N->getOperand(0);
3079 SDValue Ptr = N->getOperand(2);
3080 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
3081 MachineMemOperand *MMO = M->getMemOperand();
3082 bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
3083
3085 if (CurDAG->isBaseWithConstantOffset(Ptr)) {
3086 SDValue PtrBase = Ptr.getOperand(0);
3087 SDValue PtrOffset = Ptr.getOperand(1);
3088
3089 const APInt &OffsetVal = PtrOffset->getAsAPIntVal();
3090 if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue())) {
3091 N = glueCopyToM0(N, PtrBase);
3092 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
3093 }
3094 }
3095
3096 if (!Offset) {
3097 N = glueCopyToM0(N, Ptr);
3098 Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
3099 }
3100
3101 SDValue Ops[] = {
3102 Offset,
3103 CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
3104 Chain,
3105 N->getOperand(N->getNumOperands() - 1) // New glue
3106 };
3107
3108 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
3109 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
3110}
3111
3112// We need to handle this here because tablegen doesn't support matching
3113// instructions with multiple outputs.
3114void AMDGPUDAGToDAGISel::SelectDSBvhStackIntrinsic(SDNode *N, unsigned IntrID) {
3115 unsigned Opc;
3116 switch (IntrID) {
3117 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
3118 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
3119 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
3120 break;
3121 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
3122 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
3123 break;
3124 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
3125 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
3126 break;
3127 }
3128 SDValue Ops[] = {N->getOperand(2), N->getOperand(3), N->getOperand(4),
3129 N->getOperand(5), N->getOperand(0)};
3130
3131 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
3132 MachineMemOperand *MMO = M->getMemOperand();
3133 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
3134 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
3135}
3136
3137void AMDGPUDAGToDAGISel::SelectTensorLoadStore(SDNode *N, unsigned IntrID) {
3138 bool IsLoad = IntrID == Intrinsic::amdgcn_tensor_load_to_lds;
3139 unsigned Opc =
3140 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3141
3142 SmallVector<SDValue, 7> TensorOps;
3143 // First two groups
3144 TensorOps.push_back(N->getOperand(2)); // D# group 0
3145 TensorOps.push_back(N->getOperand(3)); // D# group 1
3146
3147 // Use _D2 version if both group 2 and 3 are zero-initialized.
3148 SDValue Group2 = N->getOperand(4);
3149 SDValue Group3 = N->getOperand(5);
3150 if (ISD::isBuildVectorAllZeros(Group2) &&
3152 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3153 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3154 } else { // Has at least 4 groups
3155 TensorOps.push_back(Group2); // D# group 2
3156 TensorOps.push_back(Group3); // D# group 3
3157 }
3158
3159 // TODO: Handle the fifth group: N->getOperand(6), which is silently ignored
3160 // for now because all existing targets only support up to 4 groups.
3161 TensorOps.push_back(CurDAG->getTargetConstant(0, SDLoc(N), MVT::i1)); // r128
3162 TensorOps.push_back(N->getOperand(7)); // cache policy
3163 TensorOps.push_back(N->getOperand(0)); // chain
3164
3165 (void)CurDAG->SelectNodeTo(N, Opc, MVT::Other, TensorOps);
3166}
3167
3168static unsigned gwsIntrinToOpcode(unsigned IntrID) {
3169 switch (IntrID) {
3170 case Intrinsic::amdgcn_ds_gws_init:
3171 return AMDGPU::DS_GWS_INIT;
3172 case Intrinsic::amdgcn_ds_gws_barrier:
3173 return AMDGPU::DS_GWS_BARRIER;
3174 case Intrinsic::amdgcn_ds_gws_sema_v:
3175 return AMDGPU::DS_GWS_SEMA_V;
3176 case Intrinsic::amdgcn_ds_gws_sema_br:
3177 return AMDGPU::DS_GWS_SEMA_BR;
3178 case Intrinsic::amdgcn_ds_gws_sema_p:
3179 return AMDGPU::DS_GWS_SEMA_P;
3180 case Intrinsic::amdgcn_ds_gws_sema_release_all:
3181 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
3182 default:
3183 llvm_unreachable("not a gws intrinsic");
3184 }
3185}
3186
3187void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
3188 if (!Subtarget->hasGWS() ||
3189 (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
3190 !Subtarget->hasGWSSemaReleaseAll())) {
3191 // Let this error.
3192 SelectCode(N);
3193 return;
3194 }
3195
3196 // Chain, intrinsic ID, vsrc, offset
3197 const bool HasVSrc = N->getNumOperands() == 4;
3198 assert(HasVSrc || N->getNumOperands() == 3);
3199
3200 SDLoc SL(N);
3201 SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
3202 int ImmOffset = 0;
3203 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
3204 MachineMemOperand *MMO = M->getMemOperand();
3205
3206 // Don't worry if the offset ends up in a VGPR. Only one lane will have
3207 // effect, so SIFixSGPRCopies will validly insert readfirstlane.
3208
3209 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
3210 // offset field) % 64. Some versions of the programming guide omit the m0
3211 // part, or claim it's from offset 0.
3212 if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
3213 // If we have a constant offset, try to use the 0 in m0 as the base.
3214 // TODO: Look into changing the default m0 initialization value. If the
3215 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
3216 // the immediate offset.
3217 glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32));
3218 ImmOffset = ConstOffset->getZExtValue();
3219 } else {
3220 if (CurDAG->isBaseWithConstantOffset(BaseOffset)) {
3221 ImmOffset = BaseOffset.getConstantOperandVal(1);
3222 BaseOffset = BaseOffset.getOperand(0);
3223 }
3224
3225 // Prefer to do the shift in an SGPR since it should be possible to use m0
3226 // as the result directly. If it's already an SGPR, it will be eliminated
3227 // later.
3228 SDNode *SGPROffset
3229 = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
3230 BaseOffset);
3231 // Shift to offset in m0
3232 SDNode *M0Base
3233 = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
3234 SDValue(SGPROffset, 0),
3235 CurDAG->getTargetConstant(16, SL, MVT::i32));
3236 glueCopyToM0(N, SDValue(M0Base, 0));
3237 }
3238
3239 SDValue Chain = N->getOperand(0);
3240 SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
3241
3242 const unsigned Opc = gwsIntrinToOpcode(IntrID);
3243
3244 const MCInstrDesc &InstrDesc = TII->get(Opc);
3245 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
3246
3247 const TargetRegisterClass *DataRC = TII->getRegClass(InstrDesc, Data0Idx);
3248
3250 if (HasVSrc) {
3251 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3252
3253 SDValue Data = N->getOperand(2);
3254 MVT DataVT = Data.getValueType().getSimpleVT();
3255 if (TRI->isTypeLegalForClass(*DataRC, DataVT)) {
3256 // Normal 32-bit case.
3257 Ops.push_back(N->getOperand(2));
3258 } else {
3259 // Operand is really 32-bits, but requires 64-bit alignment, so use the
3260 // even aligned 64-bit register class.
3261 const SDValue RegSeqOps[] = {
3262 CurDAG->getTargetConstant(DataRC->getID(), SL, MVT::i32), Data,
3263 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
3264 SDValue(
3265 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL, MVT::i32),
3266 0),
3267 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32)};
3268
3269 Ops.push_back(SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
3270 SL, MVT::v2i32, RegSeqOps),
3271 0));
3272 }
3273 }
3274
3275 Ops.push_back(OffsetField);
3276 Ops.push_back(Chain);
3277
3278 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
3279 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
3280}
3281
3282void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) {
3283 if (Subtarget->getLDSBankCount() != 16) {
3284 // This is a single instruction with a pattern.
3285 SelectCode(N);
3286 return;
3287 }
3288
3289 SDLoc DL(N);
3290
3291 // This requires 2 instructions. It is possible to write a pattern to support
3292 // this, but the generated isel emitter doesn't correctly deal with multiple
3293 // output instructions using the same physical register input. The copy to m0
3294 // is incorrectly placed before the second instruction.
3295 //
3296 // TODO: Match source modifiers.
3297 //
3298 // def : Pat <
3299 // (int_amdgcn_interp_p1_f16
3300 // (VOP3Mods f32:$src0, i32:$src0_modifiers),
3301 // (i32 timm:$attrchan), (i32 timm:$attr),
3302 // (i1 timm:$high), M0),
3303 // (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr,
3304 // timm:$attrchan, 0,
3305 // (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> {
3306 // let Predicates = [has16BankLDS];
3307 // }
3308
3309 // 16 bank LDS
3310 SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0,
3311 N->getOperand(5), SDValue());
3312
3313 SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other);
3314
3315 SDNode *InterpMov =
3316 CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, {
3317 CurDAG->getTargetConstant(2, DL, MVT::i32), // P0
3318 N->getOperand(3), // Attr
3319 N->getOperand(2), // Attrchan
3320 ToM0.getValue(1) // In glue
3321 });
3322
3323 SDNode *InterpP1LV =
3324 CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, {
3325 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
3326 N->getOperand(1), // Src0
3327 N->getOperand(3), // Attr
3328 N->getOperand(2), // Attrchan
3329 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
3330 SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high
3331 N->getOperand(4), // high
3332 CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp
3333 CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod
3334 SDValue(InterpMov, 1)
3335 });
3336
3337 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0));
3338}
3339
3340void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
3341 unsigned IntrID = N->getConstantOperandVal(1);
3342 switch (IntrID) {
3343 case Intrinsic::amdgcn_ds_append:
3344 case Intrinsic::amdgcn_ds_consume: {
3345 if (N->getValueType(0) != MVT::i32)
3346 break;
3347 SelectDSAppendConsume(N, IntrID);
3348 return;
3349 }
3350 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
3351 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
3352 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
3353 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
3354 SelectDSBvhStackIntrinsic(N, IntrID);
3355 return;
3356 case Intrinsic::amdgcn_init_whole_wave:
3357 CurDAG->getMachineFunction()
3358 .getInfo<SIMachineFunctionInfo>()
3359 ->setInitWholeWave();
3360 break;
3361 }
3362
3363 SelectCode(N);
3364}
3365
3366void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
3367 unsigned IntrID = N->getConstantOperandVal(0);
3368 unsigned Opcode = AMDGPU::INSTRUCTION_LIST_END;
3369 SDNode *ConvGlueNode = N->getGluedNode();
3370 if (ConvGlueNode) {
3371 // FIXME: Possibly iterate over multiple glue nodes?
3372 assert(ConvGlueNode->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
3373 ConvGlueNode = ConvGlueNode->getOperand(0).getNode();
3374 ConvGlueNode =
3375 CurDAG->getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE, {},
3376 MVT::Glue, SDValue(ConvGlueNode, 0));
3377 } else {
3378 ConvGlueNode = nullptr;
3379 }
3380 switch (IntrID) {
3381 case Intrinsic::amdgcn_wqm:
3382 Opcode = AMDGPU::WQM;
3383 break;
3384 case Intrinsic::amdgcn_softwqm:
3385 Opcode = AMDGPU::SOFT_WQM;
3386 break;
3387 case Intrinsic::amdgcn_wwm:
3388 case Intrinsic::amdgcn_strict_wwm:
3389 Opcode = AMDGPU::STRICT_WWM;
3390 break;
3391 case Intrinsic::amdgcn_strict_wqm:
3392 Opcode = AMDGPU::STRICT_WQM;
3393 break;
3394 case Intrinsic::amdgcn_interp_p1_f16:
3395 SelectInterpP1F16(N);
3396 return;
3397 case Intrinsic::amdgcn_permlane16_swap:
3398 case Intrinsic::amdgcn_permlane32_swap: {
3399 if ((IntrID == Intrinsic::amdgcn_permlane16_swap &&
3400 !Subtarget->hasPermlane16Swap()) ||
3401 (IntrID == Intrinsic::amdgcn_permlane32_swap &&
3402 !Subtarget->hasPermlane32Swap())) {
3403 SelectCode(N); // Hit the default error
3404 return;
3405 }
3406
3407 Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
3408 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
3409 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
3410
3411 SmallVector<SDValue, 4> NewOps(N->op_begin() + 1, N->op_end());
3412 if (ConvGlueNode)
3413 NewOps.push_back(SDValue(ConvGlueNode, 0));
3414
3415 bool FI = N->getConstantOperandVal(3);
3416 NewOps[2] = CurDAG->getTargetConstant(
3417 FI ? AMDGPU::DPP::DPP_FI_1 : AMDGPU::DPP::DPP_FI_0, SDLoc(), MVT::i32);
3418
3419 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), NewOps);
3420 return;
3421 }
3422 default:
3423 SelectCode(N);
3424 break;
3425 }
3426
3427 if (Opcode != AMDGPU::INSTRUCTION_LIST_END) {
3428 SDValue Src = N->getOperand(1);
3429 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src});
3430 }
3431
3432 if (ConvGlueNode) {
3433 SmallVector<SDValue, 4> NewOps(N->ops());
3434 NewOps.push_back(SDValue(ConvGlueNode, 0));
3435 CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), NewOps);
3436 }
3437}
3438
3439void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
3440 unsigned IntrID = N->getConstantOperandVal(1);
3441 switch (IntrID) {
3442 case Intrinsic::amdgcn_ds_gws_init:
3443 case Intrinsic::amdgcn_ds_gws_barrier:
3444 case Intrinsic::amdgcn_ds_gws_sema_v:
3445 case Intrinsic::amdgcn_ds_gws_sema_br:
3446 case Intrinsic::amdgcn_ds_gws_sema_p:
3447 case Intrinsic::amdgcn_ds_gws_sema_release_all:
3448 SelectDS_GWS(N, IntrID);
3449 return;
3450 case Intrinsic::amdgcn_tensor_load_to_lds:
3451 case Intrinsic::amdgcn_tensor_store_from_lds:
3452 SelectTensorLoadStore(N, IntrID);
3453 return;
3454 default:
3455 break;
3456 }
3457
3458 SelectCode(N);
3459}
3460
3461void AMDGPUDAGToDAGISel::SelectWAVE_ADDRESS(SDNode *N) {
3462 SDValue Log2WaveSize =
3463 CurDAG->getTargetConstant(Subtarget->getWavefrontSizeLog2(), SDLoc(N), MVT::i32);
3464 CurDAG->SelectNodeTo(N, AMDGPU::S_LSHR_B32, N->getVTList(),
3465 {N->getOperand(0), Log2WaveSize});
3466}
3467
3468void AMDGPUDAGToDAGISel::SelectSTACKRESTORE(SDNode *N) {
3469 SDValue SrcVal = N->getOperand(1);
3470 if (SrcVal.getValueType() != MVT::i32) {
3471 SelectCode(N); // Emit default error
3472 return;
3473 }
3474
3475 SDValue CopyVal;
3476 Register SP = TLI->getStackPointerRegisterToSaveRestore();
3477 SDLoc SL(N);
3478
3479 if (SrcVal.getOpcode() == AMDGPUISD::WAVE_ADDRESS) {
3480 CopyVal = SrcVal.getOperand(0);
3481 } else {
3482 SDValue Log2WaveSize = CurDAG->getTargetConstant(
3483 Subtarget->getWavefrontSizeLog2(), SL, MVT::i32);
3484
3485 if (N->isDivergent()) {
3486 SrcVal = SDValue(CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL,
3487 MVT::i32, SrcVal),
3488 0);
3489 }
3490
3491 CopyVal = SDValue(CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
3492 {SrcVal, Log2WaveSize}),
3493 0);
3494 }
3495
3496 SDValue CopyToSP = CurDAG->getCopyToReg(N->getOperand(0), SL, SP, CopyVal);
3497 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), CopyToSP);
3498}
3499
3500void AMDGPUDAGToDAGISel::SelectWRITE_REGISTER(SDNode *N) {
3501 const MDString *RegStr = cast<MDString>(
3502 cast<MDNodeSDNode>(N->getOperand(1))->getMD()->getOperand(0));
3503 SDValue SrcVal = N->getOperand(2);
3504 EVT VT = SrcVal.getValueType();
3505 Register Reg = TLI->getRegisterByName(RegStr->getString().data(),
3507 CurDAG->getMachineFunction());
3508 if (!Reg) {
3509 const Function &Fn = CurDAG->getMachineFunction().getFunction();
3510 Fn.getContext().diagnose(DiagnosticInfoGenericWithLoc(
3511 "invalid register \"" + Twine(RegStr->getString()) +
3512 "\" for llvm.write_register",
3513 Fn, N->getDebugLoc()));
3514 ReplaceUses(SDValue(N, 0), N->getOperand(0));
3515 CurDAG->RemoveDeadNode(N);
3516 return;
3517 }
3518
3519 // Speculatively insert a readfirstlane in case the source value ends up in a
3520 // VGPR, which hopefully will fold away if not.
3521 SDLoc SL(N);
3522 SDValue CopyVal;
3523 if (isa<ConstantSDNode>(SrcVal)) {
3524 CopyVal = SrcVal;
3525 } else if (VT == MVT::i32) {
3526 CopyVal = SDValue(CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL,
3527 MVT::i32, SrcVal),
3528 0);
3529 } else {
3530 assert(VT == MVT::i64);
3531 SDValue Lo =
3532 CurDAG->getTargetExtractSubreg(AMDGPU::sub0, SL, MVT::i32, SrcVal);
3533 SDValue Hi =
3534 CurDAG->getTargetExtractSubreg(AMDGPU::sub1, SL, MVT::i32, SrcVal);
3535 Lo = SDValue(
3536 CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32, Lo),
3537 0);
3538 Hi = SDValue(
3539 CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32, Hi),
3540 0);
3541 CopyVal = emitRegSequence(*CurDAG, AMDGPU::SReg_64RegClassID, VT, {Lo, Hi},
3542 {AMDGPU::sub0, AMDGPU::sub1}, SL);
3543 }
3544
3545 SDValue CopyToReg = CurDAG->getCopyToReg(N->getOperand(0), SL, Reg, CopyVal);
3546 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), CopyToReg);
3547}
3548
3549bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
3550 unsigned &Mods,
3551 bool IsCanonicalizing,
3552 bool AllowAbs) const {
3553 Mods = SISrcMods::NONE;
3554 Src = In;
3555
3556 if (Src.getOpcode() == ISD::FNEG) {
3557 Mods |= SISrcMods::NEG;
3558 Src = Src.getOperand(0);
3559 } else if (Src.getOpcode() == ISD::FSUB && IsCanonicalizing) {
3560 // Fold fsub [+-]0 into fneg. This may not have folded depending on the
3561 // denormal mode, but we're implicitly canonicalizing in a source operand.
3562 auto *LHS = dyn_cast<ConstantFPSDNode>(Src.getOperand(0));
3563 if (LHS && LHS->isZero()) {
3564 Mods |= SISrcMods::NEG;
3565 Src = Src.getOperand(1);
3566 }
3567 }
3568
3569 if (AllowAbs && Src.getOpcode() == ISD::FABS) {
3570 Mods |= SISrcMods::ABS;
3571 Src = Src.getOperand(0);
3572 }
3573
3574 if (Mods != SISrcMods::NONE)
3575 return true;
3576
3577 // Convert various sign-bit masks on integers to src mods. Currently disabled
3578 // for 16-bit types as the codegen replaces the operand without adding a
3579 // srcmod. This is intentionally finding the cases where we are performing
3580 // float neg and abs on int types, the goal is not to obtain two's complement
3581 // neg or abs. Limit converison to select operands via the nonCanonalizing
3582 // pattern.
3583 // TODO: Add 16-bit support.
3584 if (IsCanonicalizing)
3585 return true;
3586
3587 // v2i32 xor/or/and are legal. A vselect using these instructions as operands
3588 // is scalarised into two selects with EXTRACT_VECTOR_ELT operands. Peek
3589 // through the extract to the bitwise op.
3590 SDValue PeekSrc =
3591 Src->getOpcode() == ISD::EXTRACT_VECTOR_ELT ? Src->getOperand(0) : Src;
3592 // Convert various sign-bit masks to src mods. Currently disabled for 16-bit
3593 // types as the codegen replaces the operand without adding a srcmod.
3594 // This is intentionally finding the cases where we are performing float neg
3595 // and abs on int types, the goal is not to obtain two's complement neg or
3596 // abs.
3597 // TODO: Add 16-bit support.
3598 unsigned Opc = PeekSrc.getOpcode();
3599 EVT VT = Src.getValueType();
3600 if ((Opc != ISD::AND && Opc != ISD::OR && Opc != ISD::XOR) ||
3601 (VT != MVT::i32 && VT != MVT::v2i32 && VT != MVT::i64))
3602 return true;
3603
3604 ConstantSDNode *CRHS = isConstOrConstSplat(PeekSrc->getOperand(1));
3605 if (!CRHS)
3606 return true;
3607
3608 auto ReplaceSrc = [&]() -> SDValue {
3609 if (Src->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
3610 return Src.getOperand(0);
3611
3612 SDValue LHS = PeekSrc->getOperand(0);
3613 SDValue Index = Src->getOperand(1);
3614 return CurDAG->getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Src),
3615 Src.getValueType(), LHS, Index);
3616 };
3617
3618 // Recognise Srcmods:
3619 // (xor a, 0x80000000) or v2i32 (xor a, {0x80000000,0x80000000}) as NEG.
3620 // (and a, 0x7fffffff) or v2i32 (and a, {0x7fffffff,0x7fffffff}) as ABS.
3621 // (or a, 0x80000000) or v2i32 (or a, {0x80000000,0x80000000}) as NEG+ABS
3622 // SrcModifiers.
3623 if (Opc == ISD::XOR && CRHS->getAPIntValue().isSignMask()) {
3624 Mods |= SISrcMods::NEG;
3625 Src = ReplaceSrc();
3626 } else if (Opc == ISD::AND && AllowAbs &&
3627 CRHS->getAPIntValue().isMaxSignedValue()) {
3628 Mods |= SISrcMods::ABS;
3629 Src = ReplaceSrc();
3630 } else if (Opc == ISD::OR && AllowAbs && CRHS->getAPIntValue().isSignMask()) {
3632 Src = ReplaceSrc();
3633 }
3634
3635 return true;
3636}
3637
3638bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
3639 SDValue &SrcMods) const {
3640 unsigned Mods;
3641 if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/true,
3642 /*AllowAbs=*/true)) {
3643 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3644 return true;
3645 }
3646
3647 return false;
3648}
3649
3650bool AMDGPUDAGToDAGISel::SelectVOP3ModsNonCanonicalizing(
3651 SDValue In, SDValue &Src, SDValue &SrcMods) const {
3652 unsigned Mods;
3653 if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/false,
3654 /*AllowAbs=*/true)) {
3655 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3656 return true;
3657 }
3658
3659 return false;
3660}
3661
3662bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src,
3663 SDValue &SrcMods) const {
3664 unsigned Mods;
3665 if (SelectVOP3ModsImpl(In, Src, Mods,
3666 /*IsCanonicalizing=*/true,
3667 /*AllowAbs=*/false)) {
3668 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3669 return true;
3670 }
3671
3672 return false;
3673}
3674
3675bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
3676 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
3677 return false;
3678
3679 Src = In;
3680 return true;
3681}
3682
3683bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const {
3685 if (isExtractHiElt(In, Unused))
3686 return false;
3687
3688 return SelectVOP3NoMods(In, Src);
3689}
3690
3691bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(SDValue In, SDValue &Src,
3692 SDValue &SrcMods,
3693 bool OpSel) const {
3694 unsigned Mods;
3695 if (SelectVOP3ModsImpl(In, Src, Mods,
3696 /*IsCanonicalizing=*/true,
3697 /*AllowAbs=*/false)) {
3698 if (OpSel)
3699 Mods |= SISrcMods::OP_SEL_0;
3700 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3701 return true;
3702 }
3703
3704 return false;
3705}
3706
3707bool AMDGPUDAGToDAGISel::SelectVINTERPMods(SDValue In, SDValue &Src,
3708 SDValue &SrcMods) const {
3709 return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ false);
3710}
3711
3712bool AMDGPUDAGToDAGISel::SelectVINTERPModsHi(SDValue In, SDValue &Src,
3713 SDValue &SrcMods) const {
3714 return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ true);
3715}
3716
3717bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
3718 SDValue &SrcMods, SDValue &Clamp,
3719 SDValue &Omod) const {
3720 SDLoc DL(In);
3721 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
3722 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
3723
3724 return SelectVOP3Mods(In, Src, SrcMods);
3725}
3726
3727bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src,
3728 SDValue &SrcMods, SDValue &Clamp,
3729 SDValue &Omod) const {
3730 SDLoc DL(In);
3731 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
3732 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
3733
3734 return SelectVOP3BMods(In, Src, SrcMods);
3735}
3736
3737bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
3738 SDValue &Clamp, SDValue &Omod) const {
3739 Src = In;
3740
3741 SDLoc DL(In);
3742 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
3743 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
3744
3745 return true;
3746}
3747
3748bool AMDGPUDAGToDAGISel::SelectVOP3PModsPkFmaOpSel0(SDValue In, SDValue &Src,
3749 SDValue &SrcMods) const {
3750 return SelectVOP3PMods(In, Src, SrcMods, false,
3751 Subtarget->hasPkF32InvalidOpSel() &&
3752 In.getValueType() == MVT::v2f32);
3753}
3754
3755bool AMDGPUDAGToDAGISel::SelectVOP3PModsPkAddMulOpSel0(SDValue In, SDValue &Src,
3756 SDValue &SrcMods) const {
3757 if (!SelectVOP3PMods(In, Src, SrcMods))
3758 return false;
3759
3760 return !Subtarget->hasPkF32InvalidOpSel() ||
3761 In.getValueType() != MVT::v2f32 ||
3762 (cast<ConstantSDNode>(SrcMods)->getZExtValue() &
3763 SISrcMods::OP_SEL_0) == 0;
3764}
3765
3766bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
3767 SDValue &SrcMods, bool IsDOT,
3768 bool NoOpSel0) const {
3769 unsigned Mods = SISrcMods::NONE;
3770 Src = In;
3771
3772 // TODO: Handle G_FSUB 0 as fneg
3773 if (Src.getOpcode() == ISD::FNEG) {
3775 Src = Src.getOperand(0);
3776 }
3777
3778 // 64-bit VOP3P instructions do not have OPSEL or ABS.
3779 bool HasOpSel = Src.getValueSizeInBits() != 128;
3780
3781 if (Src.getOpcode() == ISD::BUILD_VECTOR && Src.getNumOperands() == 2 &&
3782 (!IsDOT || !Subtarget->hasDOTOpSelHazard()) &&
3783 (!NoOpSel0 || !isLoEltExtractHi(Src))) {
3784 unsigned VecMods = Mods;
3785
3786 SDValue Lo = stripBitcast(Src.getOperand(0));
3787 SDValue Hi = stripBitcast(Src.getOperand(1));
3788
3789 if (Lo.getOpcode() == ISD::FNEG) {
3790 Lo = stripBitcast(Lo.getOperand(0));
3791 Mods ^= SISrcMods::NEG;
3792 }
3793
3794 if (Hi.getOpcode() == ISD::FNEG) {
3795 Hi = stripBitcast(Hi.getOperand(0));
3796 Mods ^= SISrcMods::NEG_HI;
3797 }
3798
3799 if (HasOpSel) {
3800 if (isExtractHiElt(Lo, Lo))
3801 Mods |= SISrcMods::OP_SEL_0;
3802
3803 if (isExtractHiElt(Hi, Hi))
3804 Mods |= SISrcMods::OP_SEL_1;
3805 }
3806
3807 unsigned VecSize = Src.getValueSizeInBits();
3808 Lo = stripExtractLoElt(Lo);
3809 Hi = stripExtractLoElt(Hi);
3810
3811 if (Lo.getValueSizeInBits() > VecSize) {
3812 Lo = CurDAG->getTargetExtractSubreg(
3813 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
3814 MVT::getIntegerVT(VecSize), Lo);
3815 }
3816
3817 if (Hi.getValueSizeInBits() > VecSize) {
3818 Hi = CurDAG->getTargetExtractSubreg(
3819 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
3820 MVT::getIntegerVT(VecSize), Hi);
3821 }
3822
3823 assert(Lo.getValueSizeInBits() <= VecSize &&
3824 Hi.getValueSizeInBits() <= VecSize);
3825
3826 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
3827 // Really a scalar input. Just select from the low half of the register to
3828 // avoid packing.
3829
3830 if (VecSize == Lo.getValueSizeInBits()) {
3831 Src = Lo;
3832 } else if (VecSize == 32) {
3833 Src = createVOP3PSrc32FromLo16(Lo, Src, CurDAG, Subtarget);
3834 } else {
3835 assert((Lo.getValueSizeInBits() == 32 && VecSize == 64) ||
3836 (Lo.getValueSizeInBits() == 64 && VecSize == 128));
3837
3838 SDLoc SL(In);
3840 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL,
3841 Lo.getValueType()), 0);
3842 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3843 // <2 x 64> instructions do not have OPSEL and also replicate low 64
3844 // bits of a scalar input into high 64 bits. Use VGPRs in this case.
3845 // TODO: This fact can be exploited but we need to set proper OPSEL for
3846 // codegen folding purposes. It will not affect a final instruction.
3847 auto RC = Lo->isDivergent() ? TRI->getVGPRClassForBitWidth(VecSize)
3848 : TRI->getSGPRClassForBitWidth(VecSize);
3849 unsigned NumRegs = Lo.getValueSizeInBits() == 32 ? 1 : 2;
3850 const SDValue Ops[] = {
3851 CurDAG->getTargetConstant(RC->getID(), SL, MVT::i32), Lo,
3852 CurDAG->getTargetConstant(TRI->getSubRegFromChannel(0, NumRegs), SL,
3853 MVT::i32),
3854 // For packed 64-bit ops without OPSEL support, a later pass will
3855 // optimize the splat sgpr patterns to save registers.
3856 HasOpSel ? Undef : Lo,
3857 CurDAG->getTargetConstant(
3858 TRI->getSubRegFromChannel(NumRegs, NumRegs), SL, MVT::i32)};
3859
3860 Src = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
3861 Src.getValueType(), Ops), 0);
3862 // Check that both op_sel_0 and op_sel_1 are zero.
3864 }
3865 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3866 return true;
3867 }
3868
3869 if (VecSize == 64 && Lo == Hi && isa<ConstantFPSDNode>(Lo)) {
3870 uint64_t Lit = cast<ConstantFPSDNode>(Lo)->getValueAPF()
3871 .bitcastToAPInt().getZExtValue();
3872 if (AMDGPU::isInlinableLiteral32(Lit, Subtarget->hasInv2PiInlineImm())) {
3873 Src = CurDAG->getTargetConstant(Lit, SDLoc(In), MVT::i64);
3874 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3875 return true;
3876 }
3877 }
3878
3879 Mods = VecMods;
3880 } else if (Src.getOpcode() == ISD::VECTOR_SHUFFLE &&
3881 Src.getNumOperands() == 2) {
3882
3883 // TODO: We should repeat the build_vector source check above for the
3884 // vector_shuffle for negates and casts of individual elements.
3885
3886 assert(Src.getValueSizeInBits() != 128 &&
3887 "<2 x 64> VECTOR_SHUFFLE should not be legal.");
3888
3889 auto *SVN = cast<ShuffleVectorSDNode>(Src);
3890 ArrayRef<int> Mask = SVN->getMask();
3891
3892 if (Mask[0] < 2 && Mask[1] < 2 && (!NoOpSel0 || Mask[0] != 1)) {
3893 // src1 should be undef.
3894 SDValue ShuffleSrc = SVN->getOperand(0);
3895
3896 if (ShuffleSrc.getOpcode() == ISD::FNEG) {
3897 ShuffleSrc = ShuffleSrc.getOperand(0);
3899 }
3900
3901 if (Mask[0] == 1)
3902 Mods |= SISrcMods::OP_SEL_0;
3903 if (Mask[1] == 1)
3904 Mods |= SISrcMods::OP_SEL_1;
3905
3906 Src = ShuffleSrc;
3907 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3908 return true;
3909 }
3910 }
3911
3912 // Packed instructions do not have abs modifiers.
3913 Mods |= SISrcMods::OP_SEL_1;
3914
3915 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3916 return true;
3917}
3918
3919bool AMDGPUDAGToDAGISel::SelectVOP3PModsDOT(SDValue In, SDValue &Src,
3920 SDValue &SrcMods) const {
3921 return SelectVOP3PMods(In, Src, SrcMods, true);
3922}
3923
3924bool AMDGPUDAGToDAGISel::SelectVOP3PNoModsDOT(SDValue In, SDValue &Src) const {
3925 SDValue SrcTmp, SrcModsTmp;
3926 SelectVOP3PMods(In, SrcTmp, SrcModsTmp, true);
3927 if (cast<ConstantSDNode>(SrcModsTmp)->getZExtValue() == SISrcMods::OP_SEL_1) {
3928 Src = SrcTmp;
3929 return true;
3930 }
3931
3932 return false;
3933}
3934
3935bool AMDGPUDAGToDAGISel::SelectVOP3PModsF32(SDValue In, SDValue &Src,
3936 SDValue &SrcMods) const {
3937 SelectVOP3Mods(In, Src, SrcMods);
3938 unsigned Mods = SISrcMods::OP_SEL_1;
3939 Mods |= cast<ConstantSDNode>(SrcMods)->getZExtValue();
3940 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3941 return true;
3942}
3943
3944bool AMDGPUDAGToDAGISel::SelectVOP3PNoModsF32(SDValue In, SDValue &Src) const {
3945 SDValue SrcTmp, SrcModsTmp;
3946 SelectVOP3PModsF32(In, SrcTmp, SrcModsTmp);
3947 if (cast<ConstantSDNode>(SrcModsTmp)->getZExtValue() == SISrcMods::OP_SEL_1) {
3948 Src = SrcTmp;
3949 return true;
3950 }
3951
3952 return false;
3953}
3954
3955bool AMDGPUDAGToDAGISel::SelectWMMAOpSelVOP3PMods(SDValue In,
3956 SDValue &Src) const {
3957 const ConstantSDNode *C = cast<ConstantSDNode>(In);
3958 assert(C->getAPIntValue().getBitWidth() == 1 && "expected i1 value");
3959
3960 unsigned Mods = SISrcMods::OP_SEL_1;
3961 unsigned SrcVal = C->getZExtValue();
3962 if (SrcVal == 1)
3963 Mods |= SISrcMods::OP_SEL_0;
3964
3965 Src = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3966 return true;
3967}
3968
3970AMDGPUDAGToDAGISel::buildRegSequence32(SmallVectorImpl<SDValue> &Elts,
3971 const SDLoc &DL) const {
3972 unsigned DstRegClass;
3973 EVT DstTy;
3974 switch (Elts.size()) {
3975 case 8:
3976 DstRegClass = AMDGPU::VReg_256RegClassID;
3977 DstTy = MVT::v8i32;
3978 break;
3979 case 4:
3980 DstRegClass = AMDGPU::VReg_128RegClassID;
3981 DstTy = MVT::v4i32;
3982 break;
3983 case 2:
3984 DstRegClass = AMDGPU::VReg_64RegClassID;
3985 DstTy = MVT::v2i32;
3986 break;
3987 default:
3988 llvm_unreachable("unhandled Reg sequence size");
3989 }
3990
3992 Ops.push_back(CurDAG->getTargetConstant(DstRegClass, DL, MVT::i32));
3993 for (unsigned i = 0; i < Elts.size(); ++i) {
3994 Ops.push_back(Elts[i]);
3995 Ops.push_back(CurDAG->getTargetConstant(
3997 }
3998 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, DstTy, Ops);
3999}
4000
4002AMDGPUDAGToDAGISel::buildRegSequence16(SmallVectorImpl<SDValue> &Elts,
4003 const SDLoc &DL) const {
4004 SmallVector<SDValue, 8> PackedElts;
4005 assert("unhandled Reg sequence size" &&
4006 (Elts.size() == 8 || Elts.size() == 16));
4007
4008 // Pack 16-bit elements in pairs into 32-bit register. If both elements are
4009 // unpacked from 32-bit source use it, otherwise pack them using v_perm.
4010 for (unsigned i = 0; i < Elts.size(); i += 2) {
4011 SDValue LoSrc = stripExtractLoElt(stripBitcast(Elts[i]));
4012 SDValue HiSrc;
4013 if (isExtractHiElt(Elts[i + 1], HiSrc) && LoSrc == HiSrc) {
4014 PackedElts.push_back(HiSrc);
4015 } else {
4016 if (Subtarget->useRealTrue16Insts()) {
4017 // FIXME-TRUE16. For now pack VGPR_32 for 16-bit source before
4018 // passing to v_perm_b32. Eventually we should use replace v_perm_b32
4019 // by reg_sequence.
4021 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::i16),
4022 0);
4023 Elts[i] =
4024 emitRegSequence(*CurDAG, AMDGPU::VGPR_32RegClassID, MVT::i32,
4025 {Elts[i], Undef}, {AMDGPU::lo16, AMDGPU::hi16}, DL);
4026 Elts[i + 1] = emitRegSequence(*CurDAG, AMDGPU::VGPR_32RegClassID,
4027 MVT::i32, {Elts[i + 1], Undef},
4028 {AMDGPU::lo16, AMDGPU::hi16}, DL);
4029 }
4030 SDValue PackLoLo = CurDAG->getTargetConstant(0x05040100, DL, MVT::i32);
4031 MachineSDNode *Packed =
4032 CurDAG->getMachineNode(AMDGPU::V_PERM_B32_e64, DL, MVT::i32,
4033 {Elts[i + 1], Elts[i], PackLoLo});
4034 PackedElts.push_back(SDValue(Packed, 0));
4035 }
4036 }
4037 return buildRegSequence32(PackedElts, DL);
4038}
4039
4041AMDGPUDAGToDAGISel::buildRegSequence(SmallVectorImpl<SDValue> &Elts,
4042 const SDLoc &DL,
4043 unsigned ElementSize) const {
4044 if (ElementSize == 16)
4045 return buildRegSequence16(Elts, DL);
4046 if (ElementSize == 32)
4047 return buildRegSequence32(Elts, DL);
4048 llvm_unreachable("Unhandled element size");
4049}
4050
4051void AMDGPUDAGToDAGISel::selectWMMAModsNegAbs(unsigned ModOpcode,
4052 unsigned &Mods,
4054 SDValue &Src, const SDLoc &DL,
4055 unsigned ElementSize) const {
4056 if (ModOpcode == ISD::FNEG) {
4057 Mods |= SISrcMods::NEG;
4058 // Check if all elements also have abs modifier
4059 SmallVector<SDValue, 8> NegAbsElts;
4060 for (auto El : Elts) {
4061 if (El.getOpcode() != ISD::FABS)
4062 break;
4063 NegAbsElts.push_back(El->getOperand(0));
4064 }
4065 if (Elts.size() != NegAbsElts.size()) {
4066 // Neg
4067 Src = SDValue(buildRegSequence(Elts, DL, ElementSize), 0);
4068 } else {
4069 // Neg and Abs
4070 Mods |= SISrcMods::NEG_HI;
4071 Src = SDValue(buildRegSequence(NegAbsElts, DL, ElementSize), 0);
4072 }
4073 } else {
4074 assert(ModOpcode == ISD::FABS);
4075 // Abs
4076 Mods |= SISrcMods::NEG_HI;
4077 Src = SDValue(buildRegSequence(Elts, DL, ElementSize), 0);
4078 }
4079}
4080
4081// Check all f16 elements for modifiers while looking through b32 and v2b16
4082// build vector, stop if element does not satisfy ModifierCheck.
4083static void
4085 std::function<bool(SDValue)> ModifierCheck) {
4086 for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
4087 if (auto *F16Pair =
4088 dyn_cast<BuildVectorSDNode>(stripBitcast(BV->getOperand(i)))) {
4089 for (unsigned i = 0; i < F16Pair->getNumOperands(); ++i) {
4090 SDValue ElF16 = stripBitcast(F16Pair->getOperand(i));
4091 if (!ModifierCheck(ElF16))
4092 break;
4093 }
4094 }
4095 }
4096}
4097
4098bool AMDGPUDAGToDAGISel::SelectWMMAModsF16Neg(SDValue In, SDValue &Src,
4099 SDValue &SrcMods) const {
4100 Src = In;
4101 unsigned Mods = SISrcMods::OP_SEL_1;
4102
4103 // mods are on f16 elements
4104 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
4106
4107 checkWMMAElementsModifiersF16(BV, [&](SDValue Element) -> bool {
4108 if (Element.getOpcode() != ISD::FNEG)
4109 return false;
4110 EltsF16.push_back(Element.getOperand(0));
4111 return true;
4112 });
4113
4114 // All elements have neg modifier
4115 if (BV->getNumOperands() * 2 == EltsF16.size()) {
4116 Src = SDValue(buildRegSequence16(EltsF16, SDLoc(In)), 0);
4117 Mods |= SISrcMods::NEG;
4118 Mods |= SISrcMods::NEG_HI;
4119 }
4120 }
4121
4122 // mods are on v2f16 elements
4123 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
4124 SmallVector<SDValue, 8> EltsV2F16;
4125 for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
4126 SDValue ElV2f16 = stripBitcast(BV->getOperand(i));
4127 // Based on first element decide which mod we match, neg or abs
4128 if (ElV2f16.getOpcode() != ISD::FNEG)
4129 break;
4130 EltsV2F16.push_back(ElV2f16.getOperand(0));
4131 }
4132
4133 // All pairs of elements have neg modifier
4134 if (BV->getNumOperands() == EltsV2F16.size()) {
4135 Src = SDValue(buildRegSequence32(EltsV2F16, SDLoc(In)), 0);
4136 Mods |= SISrcMods::NEG;
4137 Mods |= SISrcMods::NEG_HI;
4138 }
4139 }
4140
4141 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4142 return true;
4143}
4144
4145bool AMDGPUDAGToDAGISel::SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
4146 SDValue &SrcMods) const {
4147 Src = In;
4148 unsigned Mods = SISrcMods::OP_SEL_1;
4149 unsigned ModOpcode;
4150
4151 // mods are on f16 elements
4152 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
4154 checkWMMAElementsModifiersF16(BV, [&](SDValue ElF16) -> bool {
4155 // Based on first element decide which mod we match, neg or abs
4156 if (EltsF16.empty())
4157 ModOpcode = (ElF16.getOpcode() == ISD::FNEG) ? ISD::FNEG : ISD::FABS;
4158 if (ElF16.getOpcode() != ModOpcode)
4159 return false;
4160 EltsF16.push_back(ElF16.getOperand(0));
4161 return true;
4162 });
4163
4164 // All elements have ModOpcode modifier
4165 if (BV->getNumOperands() * 2 == EltsF16.size())
4166 selectWMMAModsNegAbs(ModOpcode, Mods, EltsF16, Src, SDLoc(In), 16);
4167 }
4168
4169 // mods are on v2f16 elements
4170 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
4171 SmallVector<SDValue, 8> EltsV2F16;
4172
4173 for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
4174 SDValue ElV2f16 = stripBitcast(BV->getOperand(i));
4175 // Based on first element decide which mod we match, neg or abs
4176 if (EltsV2F16.empty())
4177 ModOpcode = (ElV2f16.getOpcode() == ISD::FNEG) ? ISD::FNEG : ISD::FABS;
4178 if (ElV2f16->getOpcode() != ModOpcode)
4179 break;
4180 EltsV2F16.push_back(ElV2f16->getOperand(0));
4181 }
4182
4183 // All elements have ModOpcode modifier
4184 if (BV->getNumOperands() == EltsV2F16.size())
4185 selectWMMAModsNegAbs(ModOpcode, Mods, EltsV2F16, Src, SDLoc(In), 32);
4186 }
4187
4188 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4189 return true;
4190}
4191
4192bool AMDGPUDAGToDAGISel::SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
4193 SDValue &SrcMods) const {
4194 Src = In;
4195 unsigned Mods = SISrcMods::OP_SEL_1;
4197
4198 if (auto *BV = dyn_cast<BuildVectorSDNode>(stripBitcast(In))) {
4199 assert(BV->getNumOperands() > 0);
4200 // Based on first element decide which mod we match, neg or abs
4201 SDValue ElF32 = stripBitcast(BV->getOperand(0));
4202 unsigned ModOpcode =
4203 (ElF32.getOpcode() == ISD::FNEG) ? ISD::FNEG : ISD::FABS;
4204 for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
4205 SDValue ElF32 = stripBitcast(BV->getOperand(i));
4206 if (ElF32.getOpcode() != ModOpcode)
4207 break;
4208 EltsF32.push_back(ElF32.getOperand(0));
4209 }
4210
4211 // All elements had ModOpcode modifier
4212 if (BV->getNumOperands() == EltsF32.size())
4213 selectWMMAModsNegAbs(ModOpcode, Mods, EltsF32, Src, SDLoc(In), 32);
4214 }
4215
4216 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4217 return true;
4218}
4219
4220bool AMDGPUDAGToDAGISel::SelectWMMAVISrc(SDValue In, SDValue &Src) const {
4221 if (auto *BV = dyn_cast<BuildVectorSDNode>(In)) {
4222 BitVector UndefElements;
4223 if (SDValue Splat = BV->getSplatValue(&UndefElements))
4224 if (isInlineImmediate(Splat.getNode())) {
4225 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Splat)) {
4226 unsigned Imm = C->getAPIntValue().getSExtValue();
4227 Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32);
4228 return true;
4229 }
4230 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Splat)) {
4231 unsigned Imm = C->getValueAPF().bitcastToAPInt().getSExtValue();
4232 Src = CurDAG->getTargetConstant(Imm, SDLoc(In), MVT::i32);
4233 return true;
4234 }
4235 llvm_unreachable("unhandled Constant node");
4236 }
4237 }
4238
4239 // 16 bit splat
4240 SDValue SplatSrc32 = stripBitcast(In);
4241 if (auto *SplatSrc32BV = dyn_cast<BuildVectorSDNode>(SplatSrc32))
4242 if (SDValue Splat32 = SplatSrc32BV->getSplatValue()) {
4243 SDValue SplatSrc16 = stripBitcast(Splat32);
4244 if (auto *SplatSrc16BV = dyn_cast<BuildVectorSDNode>(SplatSrc16))
4245 if (SDValue Splat = SplatSrc16BV->getSplatValue()) {
4246 const SIInstrInfo *TII = Subtarget->getInstrInfo();
4247 std::optional<APInt> RawValue;
4248 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Splat))
4249 RawValue = C->getValueAPF().bitcastToAPInt();
4250 else if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Splat))
4251 RawValue = C->getAPIntValue();
4252
4253 if (RawValue.has_value()) {
4254 EVT VT = In.getValueType().getScalarType();
4255 if (VT.getSimpleVT() == MVT::f16 || VT.getSimpleVT() == MVT::bf16) {
4256 APFloat FloatVal(VT.getSimpleVT() == MVT::f16
4259 RawValue.value());
4260 if (TII->isInlineConstant(FloatVal)) {
4261 Src = CurDAG->getTargetConstant(RawValue.value(), SDLoc(In),
4262 MVT::i16);
4263 return true;
4264 }
4265 } else if (VT.getSimpleVT() == MVT::i16) {
4266 if (TII->isInlineConstant(RawValue.value())) {
4267 Src = CurDAG->getTargetConstant(RawValue.value(), SDLoc(In),
4268 MVT::i16);
4269 return true;
4270 }
4271 } else
4272 llvm_unreachable("unknown 16-bit type");
4273 }
4274 }
4275 }
4276
4277 // Currently f64 immediate vectors are represented as vectors of v2i32, with
4278 // different lo and hi 32-bit values even though double values are splated.
4279 // So we have to manually compare to determine whether it is splated.
4280 if (CurDAG->isConstantIntBuildVectorOrConstantInt(SplatSrc32)) {
4281 int64_t Imm64 = 0;
4282 for (unsigned i = 0; i < SplatSrc32->getNumOperands(); i += 2) {
4283 auto Lo32 = cast<ConstantSDNode>(SplatSrc32->getOperand(i));
4284 auto Hi32 = cast<ConstantSDNode>(SplatSrc32->getOperand(i + 1));
4285 int64_t LoImm = Lo32->getAPIntValue().getSExtValue();
4286 int64_t HiImm = Hi32->getAPIntValue().getSExtValue();
4287 int64_t Imm64I = (HiImm << 32) + LoImm;
4288 if (i == 0) {
4289 if (!isInlineImmediate(APInt(64, Imm64I)))
4290 return false;
4291 Imm64 = Imm64I;
4292 } else if (Imm64I != Imm64)
4293 return false;
4294 } // end for
4295
4296 Src = CurDAG->getTargetConstant(Imm64, SDLoc(In), MVT::i64);
4297 return true;
4298 }
4299
4300 return false;
4301}
4302
4303bool AMDGPUDAGToDAGISel::SelectSWMMACIndex8(SDValue In, SDValue &Src,
4304 SDValue &IndexKey) const {
4305 unsigned Key = 0;
4306 Src = In;
4307
4308 if (In.getOpcode() == ISD::SRL) {
4309 const llvm::SDValue &ShiftSrc = In.getOperand(0);
4310 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(In.getOperand(1));
4311 if (ShiftSrc.getValueType().getSizeInBits() == 32 && ShiftAmt &&
4312 ShiftAmt->getZExtValue() % 8 == 0) {
4313 Key = ShiftAmt->getZExtValue() / 8;
4314 Src = ShiftSrc;
4315 }
4316 }
4317
4318 IndexKey = CurDAG->getTargetConstant(Key, SDLoc(In), MVT::i32);
4319 return true;
4320}
4321
4322bool AMDGPUDAGToDAGISel::SelectSWMMACIndex16(SDValue In, SDValue &Src,
4323 SDValue &IndexKey) const {
4324 unsigned Key = 0;
4325 Src = In;
4326
4327 if (In.getOpcode() == ISD::SRL) {
4328 const llvm::SDValue &ShiftSrc = In.getOperand(0);
4329 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(In.getOperand(1));
4330 if (ShiftSrc.getValueType().getSizeInBits() == 32 && ShiftAmt &&
4331 ShiftAmt->getZExtValue() == 16) {
4332 Key = 1;
4333 Src = ShiftSrc;
4334 }
4335 }
4336
4337 IndexKey = CurDAG->getTargetConstant(Key, SDLoc(In), MVT::i32);
4338 return true;
4339}
4340
4341bool AMDGPUDAGToDAGISel::SelectSWMMACIndex32(SDValue In, SDValue &Src,
4342 SDValue &IndexKey) const {
4343 unsigned Key = 0;
4344 Src = In;
4345
4346 SDValue InI32;
4347
4348 if (In.getOpcode() == ISD::ANY_EXTEND || In.getOpcode() == ISD::ZERO_EXTEND) {
4349 const SDValue &ExtendSrc = In.getOperand(0);
4350 if (ExtendSrc.getValueSizeInBits() == 32)
4351 InI32 = ExtendSrc;
4352 } else if (In->getOpcode() == ISD::BITCAST) {
4353 const SDValue &CastSrc = In.getOperand(0);
4354 if (CastSrc.getOpcode() == ISD::BUILD_VECTOR &&
4355 CastSrc.getOperand(0).getValueSizeInBits() == 32) {
4356 ConstantSDNode *Zero = dyn_cast<ConstantSDNode>(CastSrc.getOperand(1));
4357 if (Zero && Zero->getZExtValue() == 0)
4358 InI32 = CastSrc.getOperand(0);
4359 }
4360 }
4361
4362 if (InI32 && InI32.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
4363 const SDValue &ExtractVecEltSrc = InI32.getOperand(0);
4364 ConstantSDNode *EltIdx = dyn_cast<ConstantSDNode>(InI32.getOperand(1));
4365 if (ExtractVecEltSrc.getValueSizeInBits() == 64 && EltIdx &&
4366 EltIdx->getZExtValue() == 1) {
4367 Key = 1;
4368 Src = ExtractVecEltSrc;
4369 }
4370 }
4371
4372 IndexKey = CurDAG->getTargetConstant(Key, SDLoc(In), MVT::i32);
4373 return true;
4374}
4375
4376bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
4377 SDValue &SrcMods) const {
4378 unsigned Mods = SISrcMods::NONE;
4379 Src = In;
4380 if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 &&
4381 isExtractHiElt(Src, Src))
4382 Mods |= SISrcMods::OP_SEL_0;
4383 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4384 return true;
4385}
4386
4387bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
4388 SDValue &SrcMods) const {
4389 unsigned Mods;
4390 if (!SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/true,
4391 /*AllowAbs=*/true))
4392 return false;
4393
4394 if (!Subtarget->useRealTrue16Insts() && In.getValueSizeInBits() == 16 &&
4395 isExtractHiElt(Src, Src))
4396 Mods |= SISrcMods::OP_SEL_0;
4397
4398 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4399 return true;
4400}
4401
4402// Match lowered fpext from bf16 to f32. This is a bit operation extending
4403// a 16-bit value with 16-bit of zeroes at LSB:
4404//
4405// 1. (f32 (bitcast (build_vector (i16 0), (i16 (bitcast bf16:val)))))
4406// 2. (f32 (bitcast (and i32:val, 0xffff0000))) -> IsExtractHigh = true
4407// 3. (f32 (bitcast (shl i32:va, 16) -> IsExtractHigh = false
4408static SDValue matchBF16FPExtendLike(SDValue Op, bool &IsExtractHigh) {
4409 if (Op.getValueType() != MVT::f32 || Op.getOpcode() != ISD::BITCAST)
4410 return SDValue();
4411 Op = Op.getOperand(0);
4412
4413 IsExtractHigh = false;
4414 if (Op.getValueType() == MVT::v2i16 && Op.getOpcode() == ISD::BUILD_VECTOR) {
4415 auto Low16 = dyn_cast<ConstantSDNode>(Op.getOperand(0));
4416 if (!Low16 || !Low16->isZero())
4417 return SDValue();
4418 Op = stripBitcast(Op.getOperand(1));
4419 if (Op.getValueType() != MVT::bf16)
4420 return SDValue();
4421 return Op;
4422 }
4423
4424 if (Op.getValueType() != MVT::i32)
4425 return SDValue();
4426
4427 if (Op.getOpcode() == ISD::AND) {
4428 if (auto Mask = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
4429 if (Mask->getZExtValue() == 0xffff0000) {
4430 IsExtractHigh = true;
4431 return Op.getOperand(0);
4432 }
4433 }
4434 return SDValue();
4435 }
4436
4437 if (Op.getOpcode() == ISD::SHL) {
4438 if (auto Amt = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
4439 if (Amt->getZExtValue() == 16)
4440 return Op.getOperand(0);
4441 }
4442 }
4443
4444 return SDValue();
4445}
4446
4447// The return value is not whether the match is possible (which it always is),
4448// but whether or not it a conversion is really used.
4449bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
4450 unsigned &Mods,
4451 MVT VT) const {
4452 Mods = 0;
4453 SelectVOP3ModsImpl(In, Src, Mods);
4454
4455 bool IsExtractHigh = false;
4456 if (Src.getOpcode() == ISD::FP_EXTEND &&
4457 Src.getOperand(0).getValueType() == VT) {
4458 Src = Src.getOperand(0);
4459 } else if (VT == MVT::bf16) {
4460 SDValue B16 = matchBF16FPExtendLike(Src, IsExtractHigh);
4461 if (!B16)
4462 return false;
4463 Src = B16;
4464 } else
4465 return false;
4466
4467 if (Src.getValueType() != VT &&
4468 (VT != MVT::bf16 || Src.getValueType() != MVT::i32))
4469 return false;
4470
4471 Src = stripBitcast(Src);
4472
4473 // Be careful about folding modifiers if we already have an abs. fneg is
4474 // applied last, so we don't want to apply an earlier fneg.
4475 if ((Mods & SISrcMods::ABS) == 0) {
4476 unsigned ModsTmp;
4477 SelectVOP3ModsImpl(Src, Src, ModsTmp);
4478
4479 if ((ModsTmp & SISrcMods::NEG) != 0)
4480 Mods ^= SISrcMods::NEG;
4481
4482 if ((ModsTmp & SISrcMods::ABS) != 0)
4483 Mods |= SISrcMods::ABS;
4484 }
4485
4486 // op_sel/op_sel_hi decide the source type and source.
4487 // If the source's op_sel_hi is set, it indicates to do a conversion from
4488 // fp16. If the sources's op_sel is set, it picks the high half of the source
4489 // register.
4490
4491 Mods |= SISrcMods::OP_SEL_1;
4492 if (Src.getValueSizeInBits() == 16) {
4493 if (isExtractHiElt(Src, Src)) {
4494 Mods |= SISrcMods::OP_SEL_0;
4495
4496 // TODO: Should we try to look for neg/abs here?
4497 return true;
4498 }
4499
4500 if (Src.getOpcode() == ISD::TRUNCATE &&
4501 Src.getOperand(0).getValueType() == MVT::i32) {
4502 Src = Src.getOperand(0);
4503 return true;
4504 }
4505
4506 if (Subtarget->useRealTrue16Insts())
4507 // In true16 mode, pack src to a 32bit
4508 Src = createVOP3PSrc32FromLo16(Src, In, CurDAG, Subtarget);
4509 } else if (IsExtractHigh)
4510 Mods |= SISrcMods::OP_SEL_0;
4511
4512 return true;
4513}
4514
4515bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
4516 SDValue &SrcMods) const {
4517 unsigned Mods = 0;
4518 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::f16))
4519 return false;
4520 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4521 return true;
4522}
4523
4524bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
4525 SDValue &SrcMods) const {
4526 unsigned Mods = 0;
4527 SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::f16);
4528 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4529 return true;
4530}
4531
4532bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsExtNeg(SDValue In, SDValue &Src,
4533 SDValue &SrcMods) const {
4534 unsigned Mods = 0;
4535 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::f16))
4536 return false;
4537 SrcMods =
4538 CurDAG->getTargetConstant(Mods ^ SISrcMods::NEG, SDLoc(In), MVT::i32);
4539 return true;
4540}
4541
4542bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsNeg(SDValue In, SDValue &Src,
4543 SDValue &SrcMods) const {
4544 unsigned Mods = 0;
4545 SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::f16);
4546 SrcMods =
4547 CurDAG->getTargetConstant(Mods ^ SISrcMods::NEG, SDLoc(In), MVT::i32);
4548 return true;
4549}
4550
4551bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16ModsExt(SDValue In, SDValue &Src,
4552 SDValue &SrcMods) const {
4553 unsigned Mods = 0;
4554 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::bf16))
4555 return false;
4556 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4557 return true;
4558}
4559
4560bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16Mods(SDValue In, SDValue &Src,
4561 SDValue &SrcMods) const {
4562 unsigned Mods = 0;
4563 SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::bf16);
4564 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4565 return true;
4566}
4567
4568bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16ModsExtNeg(
4569 SDValue In, SDValue &Src, SDValue &SrcMods) const {
4570 unsigned Mods = 0;
4571 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::bf16))
4572 return false;
4573 SrcMods =
4574 CurDAG->getTargetConstant(Mods ^ SISrcMods::NEG, SDLoc(In), MVT::i32);
4575 return true;
4576}
4577
4578bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16ModsNeg(SDValue In, SDValue &Src,
4579 SDValue &SrcMods) const {
4580 unsigned Mods = 0;
4581 SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::bf16);
4582 SrcMods =
4583 CurDAG->getTargetConstant(Mods ^ SISrcMods::NEG, SDLoc(In), MVT::i32);
4584 return true;
4585}
4586
4587// Match BITOP3 operation and return a number of matched instructions plus
4588// truth table.
4589static std::pair<unsigned, uint8_t> BitOp3_Op(SDValue In,
4591 unsigned NumOpcodes = 0;
4592 uint8_t LHSBits, RHSBits;
4593
4594 auto getOperandBits = [&Src, In](SDValue Op, uint8_t &Bits) -> bool {
4595 // Define truth table given Src0, Src1, Src2 bits permutations:
4596 // 0 0 0
4597 // 0 0 1
4598 // 0 1 0
4599 // 0 1 1
4600 // 1 0 0
4601 // 1 0 1
4602 // 1 1 0
4603 // 1 1 1
4604 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4605
4606 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
4607 if (C->isAllOnes()) {
4608 Bits = 0xff;
4609 return true;
4610 }
4611 if (C->isZero()) {
4612 Bits = 0;
4613 return true;
4614 }
4615 }
4616
4617 for (unsigned I = 0; I < Src.size(); ++I) {
4618 // Try to find existing reused operand
4619 if (Src[I] == Op) {
4620 Bits = SrcBits[I];
4621 return true;
4622 }
4623 // Try to replace parent operator
4624 if (Src[I] == In) {
4625 Bits = SrcBits[I];
4626 Src[I] = Op;
4627 return true;
4628 }
4629 }
4630
4631 if (Src.size() == 3) {
4632 // No room left for operands. Try one last time, there can be a 'not' of
4633 // one of our source operands. In this case we can compute the bits
4634 // without growing Src vector.
4635 if (Op.getOpcode() == ISD::XOR) {
4636 if (auto *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
4637 if (C->isAllOnes()) {
4638 SDValue LHS = Op.getOperand(0);
4639 for (unsigned I = 0; I < Src.size(); ++I) {
4640 if (Src[I] == LHS) {
4641 Bits = ~SrcBits[I];
4642 return true;
4643 }
4644 }
4645 }
4646 }
4647 }
4648
4649 return false;
4650 }
4651
4652 Bits = SrcBits[Src.size()];
4653 Src.push_back(Op);
4654 return true;
4655 };
4656
4657 switch (In.getOpcode()) {
4658 case ISD::AND:
4659 case ISD::OR:
4660 case ISD::XOR: {
4661 SDValue LHS = In.getOperand(0);
4662 SDValue RHS = In.getOperand(1);
4663
4664 SmallVector<SDValue, 3> Backup(Src.begin(), Src.end());
4665 if (!getOperandBits(LHS, LHSBits) ||
4666 !getOperandBits(RHS, RHSBits)) {
4667 Src = std::move(Backup);
4668 return std::make_pair(0, 0);
4669 }
4670
4671 // Recursion is naturally limited by the size of the operand vector.
4672 //
4673 // When LHS and RHS share a common sub-expression, one side's recursion
4674 // may decompose that sub-expression and replace the Src slot the other
4675 // side occupies with sub-operands via the "replace parent" path in
4676 // getOperandBits. The other side's cached bit-pattern then refers to a
4677 // slot whose contents changed, producing a wrong truth table.
4678 //
4679 // We detect this in three ways:
4680 // (A) If LHS recursed, its truth table is valid against the Src state
4681 // when LHS recursion completed (SrcAfterLHS). If RHS recursion
4682 // then mutates a Src slot that LHSBits depends on, LHSBits is
4683 // stale.
4684 // (B) If RHS did not recurse, RHSBits came from getOperandBits and
4685 // refers to a specific Src slot. If that slot's contents changed
4686 // (by either recursion), RHSBits is stale.
4687 // (C) Symmetrically for LHS if it did not recurse.
4688 SmallVector<SDValue, 3> SrcBeforeRecurse(Src.begin(), Src.end());
4689 uint8_t LHSBitsOrig = LHSBits;
4690 uint8_t RHSBitsOrig = RHSBits;
4691
4692 auto LHSOp = BitOp3_Op(LHS, Src);
4693 if (LHSOp.first) {
4694 NumOpcodes += LHSOp.first;
4695 LHSBits = LHSOp.second;
4696 }
4697
4698 SmallVector<SDValue, 3> SrcAfterLHS(Src.begin(), Src.end());
4699
4700 auto RHSOp = BitOp3_Op(RHS, Src);
4701 if (RHSOp.first) {
4702 NumOpcodes += RHSOp.first;
4703 RHSBits = RHSOp.second;
4704 }
4705
4706 // dependsOnSlot: true iff the truth table TT varies with slot Slot.
4707 auto dependsOnSlot = [](uint8_t TT, int Slot) -> bool {
4708 if (Slot < 0 || Slot > 2)
4709 return false;
4710 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4711 const int Shifts[3] = {4, 2, 1};
4712 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4713 };
4714
4715 // findSlot: locate the Src slot a getOperandBits result depends on,
4716 // including negated (XOR with -1) patterns that getOperandBits
4717 // resolves via the NOT shortcut (~SrcBits[I]).
4718 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4719 auto findSlot = [&](uint8_t Bits, SDValue Op,
4720 const SmallVectorImpl<SDValue> &S) -> int {
4721 SDValue NegatedInner;
4722 bool IsNegationOp =
4723 Op.getOpcode() == ISD::XOR && isAllOnesConstant(Op.getOperand(1));
4724 if (IsNegationOp)
4725 NegatedInner = Op.getOperand(0);
4726 for (int I = 0; I < (int)S.size(); I++) {
4727 if (Bits == SrcBitsConst[I] && S[I] == Op)
4728 return I;
4729 if (IsNegationOp && Bits == (uint8_t)~SrcBitsConst[I] &&
4730 S[I] == NegatedInner)
4731 return I;
4732 }
4733 return -1;
4734 };
4735
4736 bool Stale = false;
4737
4738 // (A) LHS recursed: its truth table is against SrcAfterLHS.
4739 // Check if RHS recursion mutated a slot that LHSBits uses.
4740 if (LHSOp.first) {
4741 for (int I = 0; I < (int)SrcAfterLHS.size() && I < 3; I++) {
4742 if (I < (int)Src.size() && Src[I] != SrcAfterLHS[I] &&
4743 dependsOnSlot(LHSBits, I)) {
4744 Stale = true;
4745 break;
4746 }
4747 }
4748 }
4749
4750 // (B) RHS did not recurse: RHSBits from getOperandBits is against
4751 // SrcBeforeRecurse. Check if that slot was mutated since then.
4752 if (!Stale && !RHSOp.first) {
4753 int Slot = findSlot(RHSBitsOrig, RHS, SrcBeforeRecurse);
4754 if (Slot >= 0 &&
4755 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4756 Stale = true;
4757 }
4758
4759 // (C) LHS did not recurse: LHSBits from getOperandBits is against
4760 // SrcBeforeRecurse. Check if that slot was mutated since then.
4761 if (!Stale && !LHSOp.first) {
4762 int Slot = findSlot(LHSBitsOrig, LHS, SrcBeforeRecurse);
4763 if (Slot >= 0 &&
4764 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4765 Stale = true;
4766 }
4767
4768 if (Stale) {
4769 Src = std::move(SrcBeforeRecurse);
4770 LHSBits = LHSBitsOrig;
4771 RHSBits = RHSBitsOrig;
4772 NumOpcodes = 0;
4773 }
4774 break;
4775 }
4776 default:
4777 return std::make_pair(0, 0);
4778 }
4779
4780 uint8_t TTbl;
4781 switch (In.getOpcode()) {
4782 case ISD::AND:
4783 TTbl = LHSBits & RHSBits;
4784 break;
4785 case ISD::OR:
4786 TTbl = LHSBits | RHSBits;
4787 break;
4788 case ISD::XOR:
4789 TTbl = LHSBits ^ RHSBits;
4790 break;
4791 default:
4792 break;
4793 }
4794
4795 return std::make_pair(NumOpcodes + 1, TTbl);
4796}
4797
4798bool AMDGPUDAGToDAGISel::SelectBITOP3(SDValue In, SDValue &Src0, SDValue &Src1,
4799 SDValue &Src2, SDValue &Tbl) const {
4801 uint8_t TTbl;
4802 unsigned NumOpcodes;
4803
4804 std::tie(NumOpcodes, TTbl) = BitOp3_Op(In, Src);
4805
4806 // Src.empty() case can happen if all operands are all zero or all ones.
4807 // Normally it shall be optimized out before reaching this.
4808 if (NumOpcodes < 2 || Src.empty())
4809 return false;
4810
4811 // For a uniform case threshold should be higher to account for moves between
4812 // VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be in SGPRs
4813 // and a readtfirstlane after.
4814 if (NumOpcodes < 4 && !In->isDivergent())
4815 return false;
4816
4817 if (NumOpcodes == 2 && In.getValueType() == MVT::i32) {
4818 // Avoid using BITOP3 for OR3, XOR3, AND_OR. This is not faster but makes
4819 // asm more readable. This cannot be modeled with AddedComplexity because
4820 // selector does not know how many operations did we match.
4821 if ((In.getOpcode() == ISD::XOR || In.getOpcode() == ISD::OR) &&
4822 (In.getOperand(0).getOpcode() == In.getOpcode() ||
4823 In.getOperand(1).getOpcode() == In.getOpcode()))
4824 return false;
4825
4826 if (In.getOpcode() == ISD::OR &&
4827 (In.getOperand(0).getOpcode() == ISD::AND ||
4828 In.getOperand(1).getOpcode() == ISD::AND))
4829 return false;
4830 }
4831
4832 // Last operand can be ignored, turning a ternary operation into a binary.
4833 // For example: (~a & b & c) | (~a & b & ~c) -> (~a & b). We can replace
4834 // 'c' with 'a' here without changing the answer. In some pathological
4835 // cases it should be possible to get an operation with a single operand
4836 // too if optimizer would not catch it.
4837 while (Src.size() < 3)
4838 Src.push_back(Src[0]);
4839
4840 Src0 = Src[0];
4841 Src1 = Src[1];
4842 Src2 = Src[2];
4843
4844 Tbl = CurDAG->getTargetConstant(TTbl, SDLoc(In), MVT::i32);
4845 return true;
4846}
4847
4848SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
4849 if (In.getOpcode() == ISD::POISON)
4850 return CurDAG->getPOISON(MVT::i32);
4851
4852 if (In.getOpcode() == ISD::UNDEF)
4853 return CurDAG->getUNDEF(MVT::i32);
4854
4855 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
4856 SDLoc SL(In);
4857 return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32);
4858 }
4859
4860 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
4861 SDLoc SL(In);
4862 return CurDAG->getConstant(
4863 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
4864 }
4865
4866 SDValue Src;
4867 if (isExtractHiElt(In, Src))
4868 return Src;
4869
4870 return SDValue();
4871}
4872
4873bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
4874 assert(CurDAG->getTarget().getTargetTriple().isAMDGCN());
4875
4876 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
4877 const SIInstrInfo *SII = Subtarget->getInstrInfo();
4878
4879 unsigned Limit = 0;
4880 bool AllUsesAcceptSReg = true;
4881 for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
4882 Limit < 10 && U != E; ++U, ++Limit) {
4883 const TargetRegisterClass *RC =
4884 getOperandRegClass(U->getUser(), U->getOperandNo());
4885
4886 // If the register class is unknown, it could be an unknown
4887 // register class that needs to be an SGPR, e.g. an inline asm
4888 // constraint
4889 if (!RC || SIRI->isSGPRClass(RC))
4890 return false;
4891
4892 if (RC != &AMDGPU::VS_32RegClass && RC != &AMDGPU::VS_64RegClass &&
4893 RC != &AMDGPU::VS_64_Align2RegClass) {
4894 AllUsesAcceptSReg = false;
4895 SDNode *User = U->getUser();
4896 if (User->isMachineOpcode()) {
4897 unsigned Opc = User->getMachineOpcode();
4898 const MCInstrDesc &Desc = SII->get(Opc);
4899 if (Desc.isCommutable()) {
4900 unsigned OpIdx = Desc.getNumDefs() + U->getOperandNo();
4901 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
4902 if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) {
4903 unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
4904 const TargetRegisterClass *CommutedRC =
4905 getOperandRegClass(U->getUser(), CommutedOpNo);
4906 if (CommutedRC == &AMDGPU::VS_32RegClass ||
4907 CommutedRC == &AMDGPU::VS_64RegClass ||
4908 CommutedRC == &AMDGPU::VS_64_Align2RegClass)
4909 AllUsesAcceptSReg = true;
4910 }
4911 }
4912 }
4913 // If "AllUsesAcceptSReg == false" so far we haven't succeeded
4914 // commuting current user. This means have at least one use
4915 // that strictly require VGPR. Thus, we will not attempt to commute
4916 // other user instructions.
4917 if (!AllUsesAcceptSReg)
4918 break;
4919 }
4920 }
4921 return !AllUsesAcceptSReg && (Limit < 10);
4922}
4923
4926 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
4927 bool IsModified = false;
4928 do {
4929 IsModified = false;
4930
4931 // Go over all selected nodes and try to fold them a bit more
4932 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
4933 while (Position != CurDAG->allnodes_end()) {
4934 SDNode *Node = &*Position++;
4936 if (!MachineNode)
4937 continue;
4938
4939 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
4940 if (ResNode != Node) {
4941 if (ResNode)
4942 ReplaceUses(Node, ResNode);
4943 IsModified = true;
4944 }
4945 }
4946 CurDAG->RemoveDeadNodes();
4947 } while (IsModified);
4948}
4949
4951 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
4952 std::vector<SDValue> &OutOps) {
4953 switch (ConstraintID) {
4955 OutOps.push_back(Op);
4956 return false;
4958 // flat_load/flat_store require the address in a VGPR. Always force a copy
4959 // to the appropriate VGPR class.
4960 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
4961 MVT VT = Op.getSimpleValueType();
4962 const TargetRegisterClass *VRC =
4963 TRI->getVGPRClassForBitWidth(VT.getSizeInBits());
4964 assert(VRC && "Expected valid VGPR class for RF constraint operand");
4965 SDLoc DL(Op);
4966 SDValue RCVal = CurDAG->getTargetConstant(VRC->getID(), DL, MVT::i32);
4967 SDNode *Copy =
4968 CurDAG->getMachineNode(AMDGPU::COPY_TO_REGCLASS, DL, VT, {Op, RCVal});
4969 OutOps.push_back(SDValue(Copy, 0));
4970 return false;
4971 }
4972 default:
4973 return true;
4974 }
4975}
4976
4981
return SDValue()
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr, SDValue &N0, SDValue &N1)
static SDValue SelectSAddrFI(SelectionDAG *CurDAG, SDValue SAddr)
static SDValue matchExtFromI32orI32(SDValue Op, bool IsSigned, const SelectionDAG *DAG)
static MemSDNode * findMemSDNode(SDNode *N)
static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val)
static SDValue combineBallotPattern(SDValue VCMP, bool &Negate)
static SDValue matchBF16FPExtendLike(SDValue Op, bool &IsExtractHigh)
static void checkWMMAElementsModifiersF16(BuildVectorSDNode *BV, std::function< bool(SDValue)> ModifierCheck)
Defines an instruction selector for the AMDGPU target.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static bool isNoUnsignedWrap(MachineInstr *Addr)
static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In, Register &Out)
static std::pair< unsigned, uint8_t > BitOp3_Op(Register R, SmallVectorImpl< Register > &Src, const MachineRegisterInfo &MRI)
static unsigned gwsIntrinToOpcode(unsigned IntrID)
Base class for AMDGPU specific classes of TargetSubtarget.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
const HexagonInstrInfo * TII
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
Provides R600 specific target descriptions.
Interface definition for R600RegisterInfo.
SmallVector< MachineOperand, 4 > Cond
int64_t C1
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
MachineInstr *& Lo
SI DAG Lowering interface definition.
#define LLVM_DEBUG(...)
Definition Debug.h:119
LLVM IR instance of the generic uniformity analysis.
Value * RHS
Value * LHS
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
AMDGPUDAGToDAGISelLegacy(TargetMachine &TM, CodeGenOptLevel OptLevel)
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
StringRef getPassName() const override
getPassName - Return a nice clean name for a pass.
AMDGPU specific code to select AMDGPU machine instructions for SelectionDAG operations.
bool isSDWAOperand(const SDNode *N) const
void SelectBuildVector(SDNode *N, unsigned RegClassID)
void Select(SDNode *N) override
Main hook for targets to transform nodes into machine nodes.
bool widenRegionLoad16(SDNode *N) const
bool runOnMachineFunction(MachineFunction &MF) override
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool matchLoadD16FromBuildVector(SDNode *N) const
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
static SDValue stripBitcast(SDValue Val)
static const fltSemantics & BFloat()
Definition APFloat.h:303
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
Class for arbitrary precision integers.
Definition APInt.h:78
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition APInt.h:462
bool isMaxSignedValue() const
Determine if this is the largest signed value.
Definition APInt.h:401
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
unsigned countr_one() const
Count the number of trailing one bits.
Definition APInt.h:1676
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
LLVM Basic Block Representation.
Definition BasicBlock.h:62
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
Analysis pass which computes a DominatorTree.
Definition Dominators.h:241
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:277
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
void checkSubtargetFeatures(const Function &F) const
Diagnose inconsistent subtarget features before attempting to codegen function F.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Analysis pass that exposes the LoopInfo for a function.
Definition LoopInfo.h:598
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
The legacy pass manager's analysis pass to compute loop information.
Definition LoopInfo.h:623
unsigned getID() const
getID() - Return the register class ID number.
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:613
Machine Value Type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getIntegerVT(unsigned BitWidth)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
Wrapper class representing virtual and physical registers.
Definition Register.h:20
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isDivergent() const
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool isAnyAdd() const
Returns true if the node type is ADD or PTRADD.
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0, unsigned &SrcOpIdx1) const override
static LLVM_READONLY const TargetRegisterClass * getSGPRClassForBitWidth(unsigned BitWidth)
static unsigned getSubRegFromChannel(uint64_t Channel, uint64_t NumRegs=1)
static bool isSGPRClass(const TargetRegisterClass *RC)
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
SelectionDAGISelLegacy(char &ID, std::unique_ptr< SelectionDAGISel > S)
SelectionDAGISelPass(std::unique_ptr< SelectionDAGISel > Selector)
LLVM_ABI PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
std::unique_ptr< FunctionLoweringInfo > FuncInfo
const TargetLowering * TLI
const TargetInstrInfo * TII
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
SelectionDAGISel(TargetMachine &tm, CodeGenOptLevel OL=CodeGenOptLevel::Default)
virtual bool runOnMachineFunction(MachineFunction &mf)
const TargetLowering * getTargetLowering() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
MachineFunction & getMachineFunction() const
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
ilist< SDNode >::iterator allnodes_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
static const unsigned CommuteAnyOperandIndex
Primary interface to the complete machine description for the target machine.
Analysis pass which computes UniformityInfo.
Legacy analysis pass which computes a CycleInfo.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
std::optional< int64_t > getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, int64_t ByteOffset)
bool isGFX12Plus(const MCSubtargetInfo &STI)
constexpr int64_t getNullPointerValue(unsigned AS)
Get the null pointer value for the given address space.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST)
std::optional< int64_t > getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset, bool IsBuffer, bool HasSOffset)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ PTRADD
PTRADD represents pointer arithmetic semantics, for targets that opt in using shouldPreservePtrArith(...
@ POISON
POISON - A poison node.
Definition ISDOpcodes.h:238
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition ISDOpcodes.h:527
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ CONVERGENCECTRL_GLUE
This does not correspond to any convergence control intrinsic.
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
Definition ISDOpcodes.h:235
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:232
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition ISDOpcodes.h:226
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ TargetFrameIndex
Definition ISDOpcodes.h:189
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:331
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ BRCOND
BRCOND - Conditional branch.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
bool isExtOpcode(unsigned Opcode)
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ User
could "use" a pointer
This is an optimization pass for GlobalISel generic memory operations.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:339
@ Offset
Definition DWP.cpp:577
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition MathExtras.h:256
LLVM_ABI LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
Op::Description Desc
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
bool isBoolSGPR(SDValue V)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition MathExtras.h:151
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
static bool getConstantValue(SDValue N, uint32_t &Out)
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:280
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition MathExtras.h:156
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
FunctionPass * createAMDGPUISelDag(TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a AMDGPU-specific.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
Definition VE.h:393
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:879
#define N
Extended Value Type.
Definition ValueTypes.h:35
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition ValueTypes.h:294
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition KnownBits.h:315
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
Definition KnownBits.h:361
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition KnownBits.h:146
APInt getMinValue() const
Return the minimal unsigned value possible given these KnownBits.
Definition KnownBits.h:130
static unsigned getSubRegFromChannel(unsigned Channel)
bool hasNoUnsignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.