74 ~AMDGPUDAGToDAGISel()
override =
default;
79 void PostprocessISelDAG()
override;
83 bool isInlineImmediate(
const SDNode *
N)
const;
86 bool FoldOperands(
unsigned,
const R600InstrInfo *, std::vector<SDValue> &);
87 bool FoldDotOperands(
unsigned,
const R600InstrInfo *, std::vector<SDValue> &);
89 bool isConstantLoad(
const MemSDNode *
N,
int cbID)
const;
90 bool isUniformBr(
const SDNode *
N)
const;
95 bool SelectGlobalValueConstantOffset(
SDValue Addr,
SDValue& IntPtr);
96 bool SelectGlobalValueVariableOffset(
SDValue Addr,
SDValue &BaseReg,
101 unsigned OffsetBits)
const;
159 void SelectADD_SUB_I64(
SDNode *
N);
160 void SelectDIV_SCALE(
SDNode *
N);
161 void SelectFMA_W_CHAIN(
SDNode *
N);
162 void SelectFMUL_W_CHAIN(
SDNode *
N);
166 void SelectS_BFEFromShifts(
SDNode *
N);
168 bool isCBranchSCC(
const SDNode *
N)
const;
170 void SelectATOMIC_CMP_SWAP(
SDNode *
N);
173 #include "AMDGPUGenDAGISel.inc"
182 return new AMDGPUDAGToDAGISel(TM, OptLevel);
190 bool AMDGPUDAGToDAGISel::isInlineImmediate(
const SDNode *
N)
const {
192 =
static_cast<const SISubtarget *
>(Subtarget)->getInstrInfo();
208 unsigned OpNo)
const {
218 =
static_cast<const SISubtarget *
>(Subtarget)->getRegisterInfo();
236 return Subtarget->getRegisterInfo()->getRegClass(RegClass);
238 case AMDGPU::REG_SEQUENCE: {
239 unsigned RCID = cast<ConstantSDNode>(N->
getOperand(0))->getZExtValue();
241 Subtarget->getRegisterInfo()->getRegClass(RCID);
244 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
245 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
251 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(
SDNode *N)
const {
277 switch (NumVectorElts) {
279 return AMDGPU::SReg_32_XM0RegClassID;
281 return AMDGPU::SReg_64RegClassID;
283 return AMDGPU::SReg_128RegClassID;
285 return AMDGPU::SReg_256RegClassID;
287 return AMDGPU::SReg_512RegClassID;
300 if (isa<AtomicSDNode>(N) ||
319 SelectADD_SUB_I64(N);
323 SelectFMUL_W_CHAIN(N);
327 SelectFMA_W_CHAIN(N);
347 switch(NumVectorElts) {
348 case 2: RegClassID = AMDGPU::R600_Reg64RegClassID;
break;
351 RegClassID = AMDGPU::R600_Reg128VerticalRegClassID;
353 RegClassID = AMDGPU::R600_Reg128RegClassID;
360 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL,
MVT::i32);
362 if (NumVectorElts == 1) {
363 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->
getOperand(0),
368 assert(NumVectorElts <= 16 &&
"Vectors with more than 16 elements not "
375 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL,
MVT::i32);
376 bool IsRegSeq =
true;
378 for (
unsigned i = 0;
i < NOps;
i++) {
385 RegSeqArgs[1 + (2 *
i) + 1] =
390 if (NOps != NumVectorElts) {
394 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
396 for (
unsigned i = NOps;
i < NumVectorElts; ++
i) {
397 RegSeqArgs[1 + (2 *
i)] =
SDValue(ImpDef, 0);
398 RegSeqArgs[1 + (2 *
i) + 1] =
405 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->
getVTList(), RegSeqArgs);
415 RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, DL,
MVT::i32);
416 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL,
MVT::i32);
417 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL,
MVT::i32);
419 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL,
MVT::i32);
420 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL,
MVT::i32);
421 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL,
MVT::i32);
427 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
440 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
448 CurDAG->getConstant(Imm & 0xFFFFFFFF, DL,
451 CurDAG->getConstant(Imm >> 32, DL,
MVT::i32));
453 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL,
MVT::i32),
458 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
494 ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
523 SelectATOMIC_CMP_SWAP(N);
530 bool AMDGPUDAGToDAGISel::isConstantLoad(
const MemSDNode *N,
int CbId)
const {
539 bool AMDGPUDAGToDAGISel::isUniformBr(
const SDNode *N)
const {
540 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
546 StringRef AMDGPUDAGToDAGISel::getPassName()
const {
547 return "AMDGPU DAG->DAG Pattern Instruction Selection";
554 bool AMDGPUDAGToDAGISel::SelectGlobalValueConstantOffset(
SDValue Addr,
557 IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4,
SDLoc(Addr),
564 bool AMDGPUDAGToDAGISel::SelectGlobalValueVariableOffset(
SDValue Addr,
566 if (!isa<ConstantSDNode>(Addr)) {
568 Offset = CurDAG->getIntPtrConstant(0,
SDLoc(Addr),
true);
574 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(
SDValue Addr,
SDValue &Base,
580 &&
isInt<16>(IMMOffset->getZExtValue())) {
583 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(),
SDLoc(Addr),
587 }
else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
588 &&
isInt<16>(IMMOffset->getZExtValue())) {
589 Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
590 SDLoc(CurDAG->getEntryNode()),
592 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(),
SDLoc(Addr),
599 Offset = CurDAG->getTargetConstant(0,
SDLoc(Addr),
MVT::i32);
603 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(
SDValue Addr,
SDValue &Base,
608 if ((C = dyn_cast<ConstantSDNode>(Addr))) {
609 Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR,
MVT::i32);
612 (C = dyn_cast<ConstantSDNode>(Addr.
getOperand(0)))) {
613 Base = CurDAG->getRegister(AMDGPU::INDIRECT_BASE_ADDR,
MVT::i32);
616 (C = dyn_cast<ConstantSDNode>(Addr.
getOperand(1)))) {
621 Offset = CurDAG->getTargetConstant(0, DL,
MVT::i32);
627 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(
SDNode *N) {
642 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
644 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
647 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
649 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
654 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
655 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
660 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
663 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
670 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
673 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL,
MVT::i32),
684 CurDAG->ReplaceAllUsesOfValueWith(
SDValue(N, 1),
SDValue(AddHi, 1));
688 CurDAG->ReplaceAllUsesWith(N, RegSequence);
689 CurDAG->RemoveDeadNode(N);
692 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(
SDNode *N) {
697 SelectVOP3Mods0(N->
getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
698 SelectVOP3Mods(N->
getOperand(2), Ops[3], Ops[2]);
699 SelectVOP3Mods(N->
getOperand(3), Ops[5], Ops[4]);
703 CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->
getVTList(), Ops);
706 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(
SDNode *N) {
711 SelectVOP3Mods0(N->
getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
712 SelectVOP3Mods(N->
getOperand(2), Ops[3], Ops[2]);
716 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->
getVTList(), Ops);
721 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(
SDNode *N) {
728 = (VT ==
MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
731 CurDAG->SelectNodeTo(N, Opc, N->
getVTList(), Ops);
734 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(
const SDValue &Base,
unsigned Offset,
735 unsigned OffsetBits)
const {
736 if ((OffsetBits == 16 && !
isUInt<16>(Offset)) ||
741 Subtarget->unsafeDSOffsetFoldingEnabled())
746 return CurDAG->SignBitIsZero(Base);
749 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(
SDValue Addr,
SDValue &Base,
752 if (CurDAG->isBaseWithConstantOffset(Addr)) {
775 if (isDSOffsetLegal(Sub, ByteOffset, 16)) {
777 = CurDAG->getMachineNode(AMDGPU::V_SUB_I32_e32, DL,
MVT::i32,
781 Offset = CurDAG->getTargetConstant(ByteOffset, DL,
MVT::i16);
786 }
else if (
const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
796 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
799 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(),
DL,
MVT::i16);
806 Offset = CurDAG->getTargetConstant(0,
SDLoc(Addr),
MVT::i16);
811 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(
SDValue Addr,
SDValue &Base,
816 if (CurDAG->isBaseWithConstantOffset(Addr)) {
821 unsigned DWordOffset1 = DWordOffset0 + 1;
823 if (isDSOffsetLegal(N0, DWordOffset1, 8)) {
825 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL,
MVT::i8);
826 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL,
MVT::i8);
833 unsigned DWordOffset1 = DWordOffset0 + 1;
845 if (isDSOffsetLegal(Sub, DWordOffset1, 8)) {
847 = CurDAG->getMachineNode(AMDGPU::V_SUB_I32_e32, DL,
MVT::i32,
851 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL,
MVT::i8);
852 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL,
MVT::i8);
857 }
else if (
const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
858 unsigned DWordOffset0 = CAddr->getZExtValue() / 4;
859 unsigned DWordOffset1 = DWordOffset0 + 1;
860 assert(4 * DWordOffset0 == CAddr->getZExtValue());
865 = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
868 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL,
MVT::i8);
869 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL,
MVT::i8);
879 Offset0 = CurDAG->getTargetConstant(0, DL,
MVT::i8);
880 Offset1 = CurDAG->getTargetConstant(1, DL,
MVT::i8);
895 if (Subtarget->useFlatForGlobal())
901 GLC = CurDAG->getTargetConstant(0, DL,
MVT::i1);
903 SLC = CurDAG->getTargetConstant(0, DL,
MVT::i1);
904 TFE = CurDAG->getTargetConstant(0, DL,
MVT::i1);
906 Idxen = CurDAG->getTargetConstant(0, DL,
MVT::i1);
907 Offen = CurDAG->getTargetConstant(0, DL,
MVT::i1);
908 Addr64 = CurDAG->getTargetConstant(0, DL,
MVT::i1);
909 SOffset = CurDAG->getTargetConstant(0, DL,
MVT::i32);
911 if (CurDAG->isBaseWithConstantOffset(Addr)) {
920 Addr64 = CurDAG->getTargetConstant(1, DL,
MVT::i1);
925 VAddr = CurDAG->getTargetConstant(0, DL,
MVT::i32);
936 Offset = CurDAG->getTargetConstant(0, DL,
MVT::i16);
937 SOffset =
SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL,
MVT::i32,
948 Addr64 = CurDAG->getTargetConstant(1, DL,
MVT::i1);
951 Offset = CurDAG->getTargetConstant(0, DL,
MVT::i16);
956 VAddr = CurDAG->getTargetConstant(0, DL,
MVT::i32);
958 Offset = CurDAG->getTargetConstant(0, DL,
MVT::i16);
963 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(
SDValue Addr,
SDValue &SRsrc,
973 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
991 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(
SDValue Addr,
SDValue &SRsrc,
995 SLC = CurDAG->getTargetConstant(0,
SDLoc(Addr),
MVT::i1);
998 return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE);
1002 if (
auto FI = dyn_cast<FrameIndexSDNode>(N))
1003 return CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
1007 bool AMDGPUDAGToDAGISel::SelectMUBUFScratch(
SDValue Addr,
SDValue &Rsrc,
1019 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1026 VAddr = foldFrameIndex(N0);
1033 VAddr = foldFrameIndex(Addr);
1034 ImmOffset = CurDAG->getTargetConstant(0, DL,
MVT::i16);
1038 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(
SDValue Addr,
SDValue &SRsrc,
1044 static_cast<const SIInstrInfo *
>(Subtarget->getInstrInfo());
1046 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1050 if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1051 !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1052 !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1066 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(
SDValue Addr,
SDValue &SRsrc,
1071 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
1073 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(
SDValue Addr,
SDValue &SRsrc,
1078 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE);
1085 uint32_t Imm = cast<ConstantSDNode>(Constant)->getZExtValue();
1089 if (Imm <= 4095 + 64) {
1091 Overflow = Imm - 4095;
1102 Overflow = High - 1;
1113 ImmOffset = CurDAG->getTargetConstant(Imm, DL,
MVT::i16);
1116 SOffset = CurDAG->getTargetConstant(Overflow, DL,
MVT::i32);
1118 SOffset =
SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL,
MVT::i32,
1119 CurDAG->getTargetConstant(Overflow, DL,
MVT::i32)),
1125 bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicOffset(
SDValue Offset,
1130 if (!isa<ConstantSDNode>(Offset))
1133 return SelectMUBUFConstant(Offset, SOffset, ImmOffset);
1136 bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicVOffset(
SDValue Offset,
1143 if (isa<ConstantSDNode>(Offset)) {
1149 SelectMUBUFConstant(Offset, Tmp1, Tmp2))
1153 if (CurDAG->isBaseWithConstantOffset(Offset)) {
1156 if (cast<ConstantSDNode>(N1)->getSExtValue() >= 0 &&
1157 SelectMUBUFConstant(N1, SOffset, ImmOffset)) {
1163 SOffset = CurDAG->getTargetConstant(0, DL,
MVT::i32);
1164 ImmOffset = CurDAG->getTargetConstant(0, DL,
MVT::i16);
1170 bool AMDGPUDAGToDAGISel::SelectFlat(
SDValue Addr,
1175 TFE = SLC = CurDAG->getTargetConstant(0,
SDLoc(),
MVT::i1);
1184 int64_t EncodedOffset) {
1186 isUInt<8>(EncodedOffset) : isUInt<20>(EncodedOffset);
1189 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(
SDValue ByteOffsetNode,
1190 SDValue &Offset,
bool &Imm)
const {
1197 SDLoc SL(ByteOffsetNode);
1201 ByteOffset >> 2 : ByteOffset;
1204 Offset = CurDAG->getTargetConstant(EncodedOffset, SL,
MVT::i32);
1214 Offset = CurDAG->getTargetConstant(EncodedOffset, SL,
MVT::i32);
1217 Offset =
SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL,
MVT::i32,
1224 bool AMDGPUDAGToDAGISel::SelectSMRD(
SDValue Addr,
SDValue &SBase,
1225 SDValue &Offset,
bool &Imm)
const {
1227 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1231 if (SelectSMRDOffset(N1, Offset, Imm)) {
1237 Offset = CurDAG->getTargetConstant(0, SL,
MVT::i32);
1242 bool AMDGPUDAGToDAGISel::SelectSMRDImm(
SDValue Addr,
SDValue &SBase,
1245 return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
1248 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(
SDValue Addr,
SDValue &SBase,
1255 if (!SelectSMRD(Addr, SBase, Offset, Imm))
1258 return !Imm && isa<ConstantSDNode>(
Offset);
1261 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(
SDValue Addr,
SDValue &SBase,
1264 return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
1265 !isa<ConstantSDNode>(
Offset);
1268 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(
SDValue Addr,
1271 return SelectSMRDOffset(Addr, Offset, Imm) && Imm;
1274 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(
SDValue Addr,
1280 if (!SelectSMRDOffset(Addr, Offset, Imm))
1283 return !Imm && isa<ConstantSDNode>(
Offset);
1286 bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgpr(
SDValue Addr,
1289 return SelectSMRDOffset(Addr, Offset, Imm) && !Imm &&
1290 !isa<ConstantSDNode>(
Offset);
1293 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(
SDValue Index,
1298 if (CurDAG->isBaseWithConstantOffset(Index)) {
1309 if (isa<ConstantSDNode>(Index))
1313 Offset = CurDAG->getTargetConstant(0, DL,
MVT::i32);
1317 SDNode *AMDGPUDAGToDAGISel::getS_BFE(
unsigned Opcode,
const SDLoc &DL,
1323 uint32_t PackedVal = Offset | (Width << 16);
1324 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL,
MVT::i32);
1326 return CurDAG->getMachineNode(Opcode, DL,
MVT::i32, Val, PackedConst);
1329 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(
SDNode *N) {
1342 if (0 < BVal && BVal <= CVal && CVal < 32) {
1344 unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
1346 ReplaceNode(N, getS_BFE(Opcode,
SDLoc(N), Shl.
getOperand(0), CVal - BVal,
1354 void AMDGPUDAGToDAGISel::SelectS_BFE(
SDNode *N) {
1364 if (Shift && Mask) {
1366 uint32_t MaskVal = Mask->getZExtValue();
1371 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32,
SDLoc(N),
1386 if (Shift && Mask) {
1388 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
1393 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32,
SDLoc(N),
1399 SelectS_BFEFromShifts(N);
1405 SelectS_BFEFromShifts(N);
1420 unsigned Width = cast<VTSDNode>(N->
getOperand(1))->getVT().getSizeInBits();
1421 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32,
SDLoc(N), Src.
getOperand(0),
1430 bool AMDGPUDAGToDAGISel::isCBranchSCC(
const SDNode *N)
const {
1456 void AMDGPUDAGToDAGISel::SelectBRCOND(
SDNode *N) {
1460 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF,
MVT::Other,
1465 if (isCBranchSCC(N)) {
1474 CurDAG->SelectNodeTo(N, AMDGPU::S_CBRANCH_VCCNZ,
MVT::Other,
1481 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(
SDNode *N) {
1494 if (Subtarget->hasAddr64()) {
1497 if (SelectMUBUFAddr64(Mem->
getBasePtr(), SRsrc, VAddr, SOffset,
Offset, SLC)) {
1498 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_RTN_ADDR64 :
1499 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_RTN_ADDR64;
1508 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->
getVTList(), Ops);
1515 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_RTN_OFFSET :
1516 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_RTN_OFFSET;
1523 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->
getVTList(), Ops);
1536 unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
1538 = CurDAG->getTargetExtractSubreg(SubReg, SL, VT,
SDValue(CmpSwap, 0));
1540 ReplaceUses(
SDValue(N, 0), Extract);
1542 CurDAG->RemoveDeadNode(N);
1561 SrcMods = CurDAG->getTargetConstant(Mods,
SDLoc(In),
MVT::i32);
1566 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(
SDValue In,
SDValue &Src,
1568 bool Res = SelectVOP3Mods(In, Src, SrcMods);
1569 return Res && cast<ConstantSDNode>(SrcMods)->isNullValue();
1572 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(
SDValue In,
SDValue &Src,
1577 Clamp = CurDAG->getTargetConstant(0, DL,
MVT::i32);
1578 Omod = CurDAG->getTargetConstant(0, DL,
MVT::i32);
1580 return SelectVOP3Mods(In, Src, SrcMods);
1583 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods0(
SDValue In,
SDValue &Src,
1586 bool Res = SelectVOP3Mods0(In, Src, SrcMods, Clamp, Omod);
1588 return Res && cast<ConstantSDNode>(SrcMods)->isNullValue() &&
1589 cast<ConstantSDNode>(Clamp)->isNullValue() &&
1590 cast<ConstantSDNode>(Omod)->isNullValue();
1593 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp(
SDValue In,
SDValue &Src,
1599 return SelectVOP3Mods(In, Src, SrcMods);
1602 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(
SDValue In,
SDValue &Src,
1606 Clamp = Omod = CurDAG->getTargetConstant(0,
SDLoc(In),
MVT::i32);
1607 return SelectVOP3Mods(In, Src, SrcMods);
1610 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
1613 bool IsModified =
false;
1617 for (
SDNode &Node : CurDAG->allnodes()) {
1623 if (ResNode != &Node) {
1624 ReplaceUses(&Node, ResNode);
1628 CurDAG->RemoveDeadNodes();
1629 }
while (IsModified);
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
constexpr bool isUInt< 32 >(uint64_t x)
void push_back(const T &Elt)
MachineSDNode * wrapAddr64Rsrc(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr) const
Interface definition for SIRegisterInfo.
SDValue getValue(unsigned R) const
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
AMDGPU specific subclass of TargetSubtarget.
uint64_t getZExtValue() const
Get zero extended value.
void legalizeTargetIndependentNode(SDNode *Node, SelectionDAG &DAG) const
Legalize target independent instructions (e.g.
void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd)
Assign this MachineSDNodes's memory reference descriptor list.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDVTList getVTList() const
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Carry-setting nodes for multiple precision addition and subtraction.
Describe properties that are true of each instruction in the target description file.
MachineInstrBuilder MachineInstrBuilder &DefMI const MCInstrDesc & Desc
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getScratchWaveOffsetReg() const
unsigned getNumOperands() const
Return the number of values used by this operation.
virtual SDNode * PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const =0
constexpr bool isInt< 16 >(int64_t x)
const SDValue & getOperand(unsigned Num) const
Address space for local memory.
void setNodeId(int Id)
Set unique node id.
FunctionPass * createAMDGPUISelDag(TargetMachine &TM, CodeGenOpt::Level OptLevel)
This pass converts a legalized DAG into a AMDGPU-specific.
constexpr bool isMask_32(uint32_t Value)
isMask_32 - This function returns true if the argument is a non-empty sequence of ones starting at th...
unsigned getScratchRSrcReg() const
Returns the physical register reserved for use as the resource descriptor for scratch accesses...
static bool isLegalSMRDImmOffset(const AMDGPUSubtarget *ST, int64_t EncodedOffset)
A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const HexagonInstrInfo * TII
Shift and rotation operations.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
CopyToReg - This node has three operands: a chain, a register number to set to this value...
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
Reg
All possible values of the reg field in the ModR/M byte.
This file implements a class to represent arbitrary precision integral constant values and operations...
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
TargetRegisterInfo interface that is implemented by all hw codegen targets.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Simple integer binary arithmetic operators.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Generation getGeneration() const
This node is for VLIW targets and it is used to represent a vector that is stored in consecutive regi...
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
static bool isLegalMUBUFImmOffset(const ConstantSDNode *Imm)
SDNode * getNode() const
get the SDNode which holds the desired result
const SDValue & getBasePtr() const
constexpr bool isUInt< 8 >(uint64_t x)
unsigned const MachineRegisterInfo * MRI
MVT - Machine Value Type.
LLVM Basic Block Representation.
const SDValue & getOperand(unsigned i) const
Address space for constant memory (VTX2)
This is an important base class in LLVM.
unsigned getSubRegFromChannel(unsigned Channel) const
Carry-using nodes for multiple precision addition and subtraction.
SI DAG Lowering interface definition.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
unsigned getOpcode() const
FunctionPass class - This class is used to implement most global optimizations.
uint64_t getDefaultRsrcDataFormat() const
Address space for flat memory.
EVT - Extended Value Type.
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts)
bool isInlineConstant(const APInt &Imm) const
unsigned countPopulation(T Value)
Count the number of set bits in a value.
shadow stack gc Shadow Stack GC Lowering
Interface definition of the TargetLowering class that is common to all AMD GPUs.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
BRCOND - Conditional branch.
An SDNode that represents everything that will be needed to construct a MachineInstr.
const SDValue & getChain() const
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
This is an abstract virtual class for memory operations.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
int64_t getSExtValue() const
SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL, SDValue V) const
const TargetRegisterClass * getPhysRegClass(unsigned Reg) const
Return the 'base' register class for this register.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Interface definition for SIInstrInfo.
Bitwise operators - logical and, logical or, logical xor.
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
constexpr bool isUInt< 16 >(uint64_t x)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
const MCOperandInfo * OpInfo
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Primary interface to the complete machine description for the target machine.
StringRef - Represent a constant reference to a string, i.e.
SetCC operator - This evaluates to a true value iff the condition is true.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
MachineSDNode * buildRSRC(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr, uint32_t RsrcDword1, uint64_t RsrcDword2And3) const
Return a resource descriptor with the 'Add TID' bit enabled The TID (Thread ID) is multiplied by the ...
MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num)
allocateMemRefsArray - Allocate an array to hold MachineMemOperand pointers.
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
uint64_t getZExtValue() const
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.