27#include "llvm/IR/IntrinsicsAMDGPU.h"
31#ifdef EXPENSIVE_CHECKS
36#define DEBUG_TYPE "amdgpu-isel"
51 In = stripBitcast(In);
57 Out = In.getOperand(0);
68 if (ShiftAmt->getZExtValue() == 16) {
74 if (Result.getValueSizeInBits() != 32)
85static bool isLoEltExtractHi(
SDValue BV) {
88 Lo = stripBitcast(
Lo.getOperand(0));
103 if (
Lo->isDivergent()) {
105 SL,
Lo.getValueType()),
113 Src.getValueType(),
Ops),
131 SDValue Idx = In.getOperand(1);
133 return In.getOperand(0);
137 SDValue Src = In.getOperand(0);
138 if (Src.getValueType().getSizeInBits() == 32)
139 return stripBitcast(Src);
149 assert(Elts.
size() == SubRegClass.
size() &&
"array size mismatch");
150 unsigned NumElts = Elts.
size();
153 for (
unsigned i = 0; i < NumElts; ++i) {
154 Ops[2 * i + 1] = Elts[i];
164 "AMDGPU DAG->DAG Pattern Instruction Selection",
false,
168#ifdef EXPENSIVE_CHECKS
173 "AMDGPU DAG->DAG Pattern Instruction Selection",
false,
194bool AMDGPUDAGToDAGISel::fp16SrcZerosHighBits(
unsigned Opc)
const {
230 case AMDGPUISD::FRACT:
231 case AMDGPUISD::CLAMP:
232 case AMDGPUISD::COS_HW:
233 case AMDGPUISD::SIN_HW:
234 case AMDGPUISD::FMIN3:
235 case AMDGPUISD::FMAX3:
236 case AMDGPUISD::FMED3:
237 case AMDGPUISD::FMAD_FTZ:
240 case AMDGPUISD::RCP_IFLAG:
250 case AMDGPUISD::DIV_FIXUP:
260#ifdef EXPENSIVE_CHECKS
264 assert(L->isLCSSAForm(DT));
272#ifdef EXPENSIVE_CHECKS
280 assert(Subtarget->d16PreservesUnusedBits());
281 MVT VT =
N->getValueType(0).getSimpleVT();
282 if (VT != MVT::v2i16 && VT != MVT::v2f16)
304 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
307 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
313 CurDAG->getMemIntrinsicNode(LoadOp,
SDLoc(LdHi), VTList,
326 if (LdLo &&
Lo.hasOneUse()) {
332 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
335 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
347 CurDAG->getMemIntrinsicNode(LoadOp,
SDLoc(LdLo), VTList,
361 EVT VT =
N->getValueType(0);
375 Ld ?
CurDAG->getExtLoad(ExtType, SL, MVT::i32, Mem->getChain(),
376 Mem->getBasePtr(), MemVT, Mem->getMemOperand())
377 :
CurDAG->getAtomicLoad(ExtType, SL, MemVT, MVT::i32, Mem->getChain(),
378 Mem->getBasePtr(), Mem->getMemOperand());
389 bool MadeChange =
false;
390 while (Position !=
CurDAG->allnodes_begin()) {
395 switch (
N->getOpcode()) {
398 if (Subtarget->d16PreservesUnusedBits())
403 if (Subtarget->useRealTrue16Insts())
412 CurDAG->RemoveDeadNodes();
418bool AMDGPUDAGToDAGISel::isInlineImmediate(
const SDNode *
N)
const {
424 return TII->isInlineConstant(
C->getAPIntValue());
427 return TII->isInlineConstant(
C->getValueAPF());
437 unsigned OpNo)
const {
438 if (!
N->isMachineOpcode()) {
441 if (
Reg.isVirtual()) {
443 return MRI.getRegClass(
Reg);
446 const SIRegisterInfo *
TRI = Subtarget->getRegisterInfo();
447 return TRI->getPhysRegBaseClass(
Reg);
453 switch (
N->getMachineOpcode()) {
455 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
456 const MCInstrDesc &
Desc =
TII->get(
N->getMachineOpcode());
457 unsigned OpIdx =
Desc.getNumDefs() + OpNo;
458 if (OpIdx >=
Desc.getNumOperands())
461 int16_t RegClass =
TII->getOpRegClassID(
Desc.operands()[OpIdx]);
465 return Subtarget->getRegisterInfo()->getRegClass(RegClass);
467 case AMDGPU::REG_SEQUENCE: {
468 unsigned RCID =
N->getConstantOperandVal(0);
470 Subtarget->getRegisterInfo()->getRegClass(RCID);
472 SDValue SubRegOp =
N->getOperand(OpNo + 1);
474 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
483 Ops.push_back(NewChain);
484 for (
unsigned i = 1, e =
N->getNumOperands(); i != e; ++i)
485 Ops.push_back(
N->getOperand(i));
488 return CurDAG->MorphNodeTo(
N,
N->getOpcode(),
N->getVTList(),
Ops);
495 assert(
N->getOperand(0).getValueType() == MVT::Other &&
"Expected chain");
498 return glueCopyToOp(
N,
M0,
M0.getValue(1));
501SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(
SDNode *
N)
const {
504 if (Subtarget->ldsRequiresM0Init())
506 N,
CurDAG->getSignedTargetConstant(-1, SDLoc(
N), MVT::i32));
509 unsigned Value =
MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
511 glueCopyToM0(
N,
CurDAG->getTargetConstant(
Value, SDLoc(
N), MVT::i32));
518 SDNode *
Lo =
CurDAG->getMachineNode(
519 AMDGPU::S_MOV_B32,
DL, MVT::i32,
521 SDNode *
Hi =
CurDAG->getMachineNode(
522 AMDGPU::S_MOV_B32,
DL, MVT::i32,
525 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID,
DL, MVT::i32),
529 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, VT,
Ops);
532SDNode *AMDGPUDAGToDAGISel::packConstantV2I16(
const SDNode *
N,
537 uint32_t LHSVal, RHSVal;
541 uint32_t
K = (LHSVal & 0xffff) | (RHSVal << 16);
543 isVGPRImm(
N) ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32, SL,
551 EVT VT =
N->getValueType(0);
555 SDValue RegClass =
CurDAG->getTargetConstant(RegClassID,
DL, MVT::i32);
557 if (NumVectorElts == 1) {
558 CurDAG->SelectNodeTo(
N, AMDGPU::COPY_TO_REGCLASS, EltVT,
N->getOperand(0),
563 bool IsGCN =
CurDAG->getSubtarget().getTargetTriple().isAMDGCN();
564 if (IsGCN && Subtarget->has64BitLiterals() && VT.
getSizeInBits() == 64 &&
567 bool AllConst =
true;
569 for (
unsigned I = 0;
I < NumVectorElts; ++
I) {
577 Val = CF->getValueAPF().bitcastToAPInt().getZExtValue();
580 C |= Val << (EltSize *
I);
585 CurDAG->getMachineNode(AMDGPU::S_MOV_B64_IMM_PSEUDO,
DL, VT, CV);
586 CurDAG->SelectNodeTo(
N, AMDGPU::COPY_TO_REGCLASS, VT,
SDValue(Copy, 0),
592 assert(NumVectorElts <= 32 &&
"Vectors with more than 32 elements not "
599 RegSeqArgs[0] =
CurDAG->getTargetConstant(RegClassID,
DL, MVT::i32);
600 bool IsRegSeq =
true;
601 unsigned NOps =
N->getNumOperands();
603 assert(IsGCN || EltSizeInRegs == 1);
604 for (
unsigned i = 0; i < NOps; i++) {
611 i * EltSizeInRegs, EltSizeInRegs)
613 RegSeqArgs[1 + (2 * i)] =
N->getOperand(i);
614 RegSeqArgs[1 + (2 * i) + 1] =
CurDAG->getTargetConstant(
Sub,
DL, MVT::i32);
616 if (NOps != NumVectorElts) {
621 for (
unsigned i = NOps; i < NumVectorElts; ++i) {
623 i * EltSizeInRegs, EltSizeInRegs)
625 RegSeqArgs[1 + (2 * i)] =
SDValue(ImpDef, 0);
626 RegSeqArgs[1 + (2 * i) + 1] =
633 CurDAG->SelectNodeTo(
N, AMDGPU::REG_SEQUENCE,
N->getVTList(), RegSeqArgs);
637 EVT VT =
N->getValueType(0);
641 if (!Subtarget->hasPkMovB32() || !EltVT.
bitsEq(MVT::i32) ||
649 SDValue Src0 = SVN->getOperand(0);
650 SDValue Src1 = SVN->getOperand(1);
654 assert(Src0.getValueType().getVectorNumElements() == 2 && Mask.size() == 2 &&
655 Mask[0] < 4 && Mask[1] < 4);
657 SDValue VSrc0 = Mask[0] < 2 ? Src0 : Src1;
658 SDValue VSrc1 = Mask[1] < 2 ? Src0 : Src1;
659 unsigned Src0SubReg = Mask[0] & 1 ? AMDGPU::sub1 : AMDGPU::sub0;
660 unsigned Src1SubReg = Mask[1] & 1 ? AMDGPU::sub1 : AMDGPU::sub0;
663 Src0SubReg = Src1SubReg;
665 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, VT);
670 Src1SubReg = Src0SubReg;
672 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, VT);
682 if (
N->isDivergent() && Src0SubReg == AMDGPU::sub1 &&
683 Src1SubReg == AMDGPU::sub0) {
699 SDValue Src0OpSelVal =
CurDAG->getTargetConstant(Src0OpSel,
DL, MVT::i32);
700 SDValue Src1OpSelVal =
CurDAG->getTargetConstant(Src1OpSel,
DL, MVT::i32);
703 CurDAG->SelectNodeTo(
N, AMDGPU::V_PK_MOV_B32,
N->getVTList(),
704 {Src0OpSelVal, VSrc0, Src1OpSelVal, VSrc1,
714 CurDAG->getTargetExtractSubreg(Src0SubReg,
DL, EltVT, VSrc0);
716 CurDAG->getTargetExtractSubreg(Src1SubReg,
DL, EltVT, VSrc1);
719 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID,
DL, MVT::i32),
720 ResultElt0,
CurDAG->getTargetConstant(AMDGPU::sub0,
DL, MVT::i32),
721 ResultElt1,
CurDAG->getTargetConstant(AMDGPU::sub1,
DL, MVT::i32)};
722 CurDAG->SelectNodeTo(
N, TargetOpcode::REG_SEQUENCE, VT,
Ops);
726 unsigned int Opc =
N->getOpcode();
727 if (
N->isMachineOpcode()) {
735 N = glueCopyToM0LDSInit(
N);
745 if (
N->getValueType(0) == MVT::i64) {
746 SelectAddcSubbI64(
N);
750 if (
N->getValueType(0) != MVT::i32)
757 if (
N->getValueType(0) == MVT::i64) {
758 SelectAddcSubbI64(
N);
762 SelectUADDO_USUBO(
N);
765 case AMDGPUISD::FMUL_W_CHAIN: {
766 SelectFMUL_W_CHAIN(
N);
769 case AMDGPUISD::FMA_W_CHAIN: {
770 SelectFMA_W_CHAIN(
N);
776 EVT VT =
N->getValueType(0);
794 N->isDivergent() ?
TRI->getDefaultVectorSuperClassForBitWidth(VecInBits)
806 if (
N->getValueType(0) == MVT::i128) {
807 RC =
CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID,
DL, MVT::i32);
808 SubReg0 =
CurDAG->getTargetConstant(AMDGPU::sub0_sub1,
DL, MVT::i32);
809 SubReg1 =
CurDAG->getTargetConstant(AMDGPU::sub2_sub3,
DL, MVT::i32);
810 }
else if (
N->getValueType(0) == MVT::i64) {
811 RC =
CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID,
DL, MVT::i32);
812 SubReg0 =
CurDAG->getTargetConstant(AMDGPU::sub0,
DL, MVT::i32);
813 SubReg1 =
CurDAG->getTargetConstant(AMDGPU::sub1,
DL, MVT::i32);
817 const SDValue Ops[] = { RC,
N->getOperand(0), SubReg0,
818 N->getOperand(1), SubReg1 };
820 N->getValueType(0),
Ops));
826 if (
N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(
N) ||
827 Subtarget->has64BitLiterals())
832 Imm =
FP->getValueAPF().bitcastToAPInt().getZExtValue();
837 Imm =
C->getZExtValue();
846 case AMDGPUISD::BFE_I32:
847 case AMDGPUISD::BFE_U32: {
873 case AMDGPUISD::DIV_SCALE: {
884 return SelectMUL_LOHI(
N);
895 if (
N->getValueType(0) != MVT::i32)
906 case AMDGPUISD::CVT_PKRTZ_F16_F32:
907 case AMDGPUISD::CVT_PKNORM_I16_F32:
908 case AMDGPUISD::CVT_PKNORM_U16_F32:
909 case AMDGPUISD::CVT_PK_U16_U32:
910 case AMDGPUISD::CVT_PK_I16_I32: {
912 if (
N->getValueType(0) == MVT::i32) {
913 MVT NewVT =
Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
915 { N->getOperand(0), N->getOperand(1) });
923 SelectINTRINSIC_W_CHAIN(
N);
927 SelectINTRINSIC_WO_CHAIN(
N);
931 SelectINTRINSIC_VOID(
N);
935 SelectWAVE_ADDRESS(
N);
939 SelectSTACKRESTORE(
N);
943 SelectWRITE_REGISTER(
N);
952 if (!Subtarget->hasSDWA())
962 return RHS->getZExtValue() == 0xFF || RHS->getZExtValue() == 0xFFFF;
966 return (RHS->getZExtValue() % 8) == 0;
971bool AMDGPUDAGToDAGISel::isUniformBr(
const SDNode *
N)
const {
974 return Term->getMetadata(
"amdgpu.uniform") ||
975 Term->getMetadata(
"structurizecfg.uniform");
978bool AMDGPUDAGToDAGISel::isUnneededShiftMask(
const SDNode *
N,
979 unsigned ShAmtBits)
const {
982 const APInt &
RHS =
N->getConstantOperandAPInt(1);
983 if (
RHS.countr_one() >= ShAmtBits)
1013 N1 =
Lo.getOperand(1);
1023 if (
CurDAG->isBaseWithConstantOffset(Addr)) {
1038 return "AMDGPU DAG->DAG Pattern Instruction Selection";
1054#ifdef EXPENSIVE_CHECKS
1057 for (
auto &L : LI.getLoopsInPreorder())
1058 assert(L->isLCSSAForm(DT) &&
"Loop is not in LCSSA form!");
1080 }
else if ((Addr.
getOpcode() == AMDGPUISD::DWORDADDR) &&
1082 Base =
CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
1096SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
1098 SDNode *Mov =
CurDAG->getMachineNode(
1099 AMDGPU::S_MOV_B32,
DL, MVT::i32,
1100 CurDAG->getTargetConstant(Val,
DL, MVT::i32));
1104void AMDGPUDAGToDAGISel::SelectAddcSubb(
SDNode *
N) {
1109 if (
N->isDivergent()) {
1111 : AMDGPU::V_SUBB_U32_e64;
1113 N,
Opc,
N->getVTList(),
1115 CurDAG->getTargetConstant(0, {}, MVT::i1) });
1118 : AMDGPU::S_SUB_CO_PSEUDO;
1119 CurDAG->SelectNodeTo(
N,
Opc,
N->getVTList(), {LHS, RHS, CI});
1123void AMDGPUDAGToDAGISel::SelectAddcSubbI64(
SDNode *
N) {
1128 unsigned Opcode =
N->getOpcode();
1135 SDNode *Lo0 =
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
DL,
1136 MVT::i32,
LHS, Sub0);
1137 SDNode *Hi0 =
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
DL,
1138 MVT::i32,
LHS, Sub1);
1140 SDNode *Lo1 =
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
DL,
1141 MVT::i32,
RHS, Sub0);
1142 SDNode *Hi1 =
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
DL,
1143 MVT::i32,
RHS, Sub1);
1145 SDVTList VTList =
CurDAG->getVTList(MVT::i32,
N->getValueType(1));
1147 static const unsigned NoCarryOpcMap[2][2] = {
1148 {AMDGPU::S_USUBO_PSEUDO, AMDGPU::S_UADDO_PSEUDO},
1149 {AMDGPU::V_SUB_CO_U32_e64, AMDGPU::V_ADD_CO_U32_e64}};
1150 static const unsigned CarryOpcMap[2][2] = {
1151 {AMDGPU::S_SUB_CO_PSEUDO, AMDGPU::S_ADD_CO_PSEUDO},
1152 {AMDGPU::V_SUBB_U32_e64, AMDGPU::V_ADDC_U32_e64}};
1154 bool IsVALU =
N->isDivergent();
1156 unsigned NoCarryOpc = NoCarryOpcMap[IsVALU][IsAdd];
1157 unsigned CarryOpc = CarryOpcMap[IsVALU][IsAdd];
1161 if (!ConsumeCarry) {
1164 AddLo =
CurDAG->getMachineNode(NoCarryOpc,
DL, VTList, Args);
1167 AddLo =
CurDAG->getMachineNode(NoCarryOpc,
DL, VTList, Args);
1173 AddLo =
CurDAG->getMachineNode(CarryOpc,
DL, VTList, Args);
1176 AddLo =
CurDAG->getMachineNode(CarryOpc,
DL, VTList, Args);
1184 AddHi =
CurDAG->getMachineNode(CarryOpc,
DL, VTList, Args);
1187 AddHi =
CurDAG->getMachineNode(CarryOpc,
DL, VTList, Args);
1190 unsigned RC = IsVALU ? AMDGPU::VReg_64RegClassID : AMDGPU::SReg_64RegClassID;
1191 SDValue RegSequenceArgs[] = {
CurDAG->getTargetConstant(RC,
DL, MVT::i32),
1195 MVT::i64, RegSequenceArgs);
1201void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(
SDNode *
N) {
1206 bool IsVALU =
N->isDivergent();
1208 for (SDNode::user_iterator UI =
N->user_begin(),
E =
N->user_end(); UI !=
E;
1210 if (UI.getUse().getResNo() == 1) {
1211 if (UI->isMachineOpcode()) {
1212 if (UI->getMachineOpcode() !=
1213 (IsAdd ? AMDGPU::S_ADD_CO_PSEUDO : AMDGPU::S_SUB_CO_PSEUDO)) {
1226 unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
1229 N,
Opc,
N->getVTList(),
1230 {N->getOperand(0), N->getOperand(1),
1231 CurDAG->getTargetConstant(0, {}, MVT::i1) });
1233 unsigned Opc = IsAdd ? AMDGPU::S_UADDO_PSEUDO : AMDGPU::S_USUBO_PSEUDO;
1235 CurDAG->SelectNodeTo(
N,
Opc,
N->getVTList(),
1236 {N->getOperand(0), N->getOperand(1)});
1240void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(
SDNode *
N) {
1244 SelectVOP3Mods0(
N->getOperand(1),
Ops[1],
Ops[0],
Ops[6],
Ops[7]);
1245 SelectVOP3Mods(
N->getOperand(2),
Ops[3],
Ops[2]);
1246 SelectVOP3Mods(
N->getOperand(3),
Ops[5],
Ops[4]);
1247 Ops[8] =
N->getOperand(0);
1248 Ops[9] =
N->getOperand(4);
1252 bool UseFMAC = Subtarget->hasDLInsts() &&
1256 unsigned Opcode = UseFMAC ? AMDGPU::V_FMAC_F32_e64 : AMDGPU::V_FMA_F32_e64;
1257 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(),
Ops);
1260void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(
SDNode *
N) {
1264 SelectVOP3Mods0(
N->getOperand(1),
Ops[1],
Ops[0],
Ops[4],
Ops[5]);
1265 SelectVOP3Mods(
N->getOperand(2),
Ops[3],
Ops[2]);
1266 Ops[6] =
N->getOperand(0);
1267 Ops[7] =
N->getOperand(3);
1269 CurDAG->SelectNodeTo(
N, AMDGPU::V_MUL_F32_e64,
N->getVTList(),
Ops);
1274void AMDGPUDAGToDAGISel::SelectDIV_SCALE(
SDNode *
N) {
1275 EVT VT =
N->getValueType(0);
1277 assert(VT == MVT::f32 || VT == MVT::f64);
1280 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64_e64 : AMDGPU::V_DIV_SCALE_F32_e64;
1285 SelectVOP3BMods0(
N->getOperand(0),
Ops[1],
Ops[0],
Ops[6],
Ops[7]);
1286 SelectVOP3BMods(
N->getOperand(1),
Ops[3],
Ops[2]);
1287 SelectVOP3BMods(
N->getOperand(2),
Ops[5],
Ops[4]);
1293void AMDGPUDAGToDAGISel::SelectMAD_64_32(
SDNode *
N) {
1297 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() && !
N->hasAnyUseOfValue(1);
1298 if (Subtarget->hasMADIntraFwdBug())
1299 Opc =
Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1300 : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1301 else if (UseNoCarry)
1302 Opc =
Signed ? AMDGPU::V_MAD_NC_I64_I32_e64 : AMDGPU::V_MAD_NC_U64_U32_e64;
1304 Opc =
Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1307 SDValue Ops[] = {
N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
1311 MachineSDNode *Mad =
CurDAG->getMachineNode(
Opc, SL, MVT::i64,
Ops);
1322void AMDGPUDAGToDAGISel::SelectMUL_LOHI(
SDNode *
N) {
1327 if (Subtarget->hasMadNC64_32Insts()) {
1328 VTList =
CurDAG->getVTList(MVT::i64);
1329 Opc =
Signed ? AMDGPU::V_MAD_NC_I64_I32_e64 : AMDGPU::V_MAD_NC_U64_U32_e64;
1331 VTList =
CurDAG->getVTList(MVT::i64, MVT::i1);
1332 if (Subtarget->hasMADIntraFwdBug()) {
1333 Opc =
Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1334 : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1336 Opc =
Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1343 SDNode *Mad =
CurDAG->getMachineNode(
Opc, SL, VTList,
Ops);
1345 SDValue Sub0 =
CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32);
1346 SDNode *
Lo =
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1347 MVT::i32,
SDValue(Mad, 0), Sub0);
1351 SDValue Sub1 =
CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32);
1352 SDNode *
Hi =
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1353 MVT::i32,
SDValue(Mad, 0), Sub1);
1363 if (!
Base || Subtarget->hasUsableDSOffset() ||
1364 Subtarget->unsafeDSOffsetFoldingEnabled())
1375 if (
CurDAG->isBaseWithConstantOffset(Addr)) {
1379 if (isDSOffsetLegal(N0,
C1->getSExtValue())) {
1388 int64_t ByteOffset =
C->getSExtValue();
1389 if (isDSOffsetLegal(
SDValue(), ByteOffset)) {
1398 if (isDSOffsetLegal(
Sub, ByteOffset)) {
1404 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1405 if (Subtarget->hasAddNoCarryInsts()) {
1406 SubOp = AMDGPU::V_SUB_U32_e64;
1408 CurDAG->getTargetConstant(0, {}, MVT::i1));
1411 MachineSDNode *MachineSub =
1412 CurDAG->getMachineNode(SubOp,
DL, MVT::i32, Opnds);
1428 if (isDSOffsetLegal(
SDValue(), CAddr->getZExtValue())) {
1430 MachineSDNode *MovZero =
CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1433 Offset =
CurDAG->getTargetConstant(CAddr->getZExtValue(),
DL, MVT::i16);
1440 Offset =
CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1444bool AMDGPUDAGToDAGISel::isDSOffset2Legal(
SDValue Base,
unsigned Offset0,
1446 unsigned Size)
const {
1447 if (Offset0 %
Size != 0 || Offset1 %
Size != 0)
1452 if (!
Base || Subtarget->hasUsableDSOffset() ||
1453 Subtarget->unsafeDSOffsetFoldingEnabled())
1471bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegal(
SDValue Addr)
const {
1477 if (Subtarget->hasSignedScratchOffsets())
1487 ConstantSDNode *ImmOp =
nullptr;
1498bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSV(
SDValue Addr)
const {
1504 if (Subtarget->hasSignedScratchOffsets())
1514bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSVImm(
SDValue Addr)
const {
1528 (RHSImm->getSExtValue() < 0 && RHSImm->getSExtValue() > -0x40000000)))
1531 auto LHS =
Base.getOperand(0);
1532 auto RHS =
Base.getOperand(1);
1540 return SelectDSReadWrite2(Addr,
Base, Offset0, Offset1, 4);
1546 return SelectDSReadWrite2(Addr,
Base, Offset0, Offset1, 8);
1551 unsigned Size)
const {
1554 if (
CurDAG->isBaseWithConstantOffset(Addr)) {
1558 unsigned OffsetValue0 =
C1->getZExtValue();
1559 unsigned OffsetValue1 = OffsetValue0 +
Size;
1562 if (isDSOffset2Legal(N0, OffsetValue0, OffsetValue1,
Size)) {
1564 Offset0 =
CurDAG->getTargetConstant(OffsetValue0 /
Size,
DL, MVT::i32);
1565 Offset1 =
CurDAG->getTargetConstant(OffsetValue1 /
Size,
DL, MVT::i32);
1570 if (
const ConstantSDNode *
C =
1572 unsigned OffsetValue0 =
C->getZExtValue();
1573 unsigned OffsetValue1 = OffsetValue0 +
Size;
1575 if (isDSOffset2Legal(
SDValue(), OffsetValue0, OffsetValue1,
Size)) {
1585 if (isDSOffset2Legal(
Sub, OffsetValue0, OffsetValue1,
Size)) {
1589 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1590 if (Subtarget->hasAddNoCarryInsts()) {
1591 SubOp = AMDGPU::V_SUB_U32_e64;
1593 CurDAG->getTargetConstant(0, {}, MVT::i1));
1596 MachineSDNode *MachineSub =
CurDAG->getMachineNode(
1601 CurDAG->getTargetConstant(OffsetValue0 /
Size,
DL, MVT::i32);
1603 CurDAG->getTargetConstant(OffsetValue1 /
Size,
DL, MVT::i32);
1609 unsigned OffsetValue0 = CAddr->getZExtValue();
1610 unsigned OffsetValue1 = OffsetValue0 +
Size;
1612 if (isDSOffset2Legal(
SDValue(), OffsetValue0, OffsetValue1,
Size)) {
1614 MachineSDNode *MovZero =
1615 CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
DL, MVT::i32,
Zero);
1617 Offset0 =
CurDAG->getTargetConstant(OffsetValue0 /
Size,
DL, MVT::i32);
1618 Offset1 =
CurDAG->getTargetConstant(OffsetValue1 /
Size,
DL, MVT::i32);
1626 Offset0 =
CurDAG->getTargetConstant(0,
DL, MVT::i32);
1627 Offset1 =
CurDAG->getTargetConstant(1,
DL, MVT::i32);
1637 if (Subtarget->useFlatForGlobal())
1640 Addr = Expand32BitAddress(Addr);
1643 Idxen =
CurDAG->getTargetConstant(0,
DL, MVT::i1);
1644 Offen =
CurDAG->getTargetConstant(0,
DL, MVT::i1);
1645 Addr64 =
CurDAG->getTargetConstant(0,
DL, MVT::i1);
1646 SOffset = Subtarget->hasRestrictedSOffset()
1647 ?
CurDAG->getRegister(AMDGPU::SGPR_NULL, MVT::i32)
1648 :
CurDAG->getTargetConstant(0,
DL, MVT::i32);
1650 ConstantSDNode *
C1 =
nullptr;
1652 if (
CurDAG->isBaseWithConstantOffset(Addr)) {
1665 Addr64 =
CurDAG->getTargetConstant(1,
DL, MVT::i1);
1671 Ptr =
SDValue(buildSMovImm64(
DL, 0, MVT::v2i32), 0);
1687 Ptr =
SDValue(buildSMovImm64(
DL, 0, MVT::v2i32), 0);
1689 Addr64 =
CurDAG->getTargetConstant(1,
DL, MVT::i1);
1693 VAddr =
CurDAG->getTargetConstant(0,
DL, MVT::i32);
1703 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
1704 if (
TII->isLegalMUBUFImmOffset(
C1->getZExtValue())) {
1714 AMDGPU::S_MOV_B32,
DL, MVT::i32,
1715 CurDAG->getTargetConstant(
C1->getZExtValue(),
DL, MVT::i32)),
1720bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(
SDValue Addr,
SDValue &SRsrc,
1723 SDValue Ptr, Offen, Idxen, Addr64;
1727 if (!Subtarget->hasAddr64())
1730 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset,
Offset, Offen, Idxen, Addr64))
1734 if (
C->getSExtValue()) {
1747std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(
SDValue N)
const {
1752 FI ?
CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) :
N;
1758 return std::pair(TFI,
CurDAG->getTargetConstant(0,
DL, MVT::i32));
1761bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(
SDNode *Parent,
1768 const SIMachineFunctionInfo *
Info =
MF.getInfo<SIMachineFunctionInfo>();
1770 Rsrc =
CurDAG->getRegister(
Info->getScratchRSrcReg(), MVT::v4i32);
1773 int64_t
Imm = CAddr->getSExtValue();
1774 const int64_t NullPtr =
1777 if (
Imm != NullPtr) {
1780 CurDAG->getTargetConstant(
Imm & ~MaxOffset,
DL, MVT::i32);
1781 MachineSDNode *MovHighBits =
CurDAG->getMachineNode(
1782 AMDGPU::V_MOV_B32_e32,
DL, MVT::i32, HighBits);
1783 VAddr =
SDValue(MovHighBits, 0);
1785 SOffset =
CurDAG->getTargetConstant(0,
DL, MVT::i32);
1786 ImmOffset =
CurDAG->getTargetConstant(
Imm & MaxOffset,
DL, MVT::i32);
1791 if (
CurDAG->isBaseWithConstantOffset(Addr)) {
1812 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
1813 if (
TII->isLegalMUBUFImmOffset(
C1) &&
1814 (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1815 CurDAG->SignBitIsZero(N0))) {
1816 std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1817 ImmOffset =
CurDAG->getTargetConstant(
C1,
DL, MVT::i32);
1823 std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1824 ImmOffset =
CurDAG->getTargetConstant(0,
DL, MVT::i32);
1832 if (!
Reg.isPhysical())
1834 const auto *RC =
TRI.getPhysRegBaseClass(
Reg);
1835 return RC &&
TRI.isSGPRClass(RC);
1838bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(
SDNode *Parent,
1843 const SIRegisterInfo *
TRI = Subtarget->getRegisterInfo();
1844 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
1846 const SIMachineFunctionInfo *
Info =
MF.getInfo<SIMachineFunctionInfo>();
1851 SRsrc =
CurDAG->getRegister(
Info->getScratchRSrcReg(), MVT::v4i32);
1857 ConstantSDNode *CAddr;
1870 SOffset =
CurDAG->getTargetConstant(0,
DL, MVT::i32);
1875 SRsrc =
CurDAG->getRegister(
Info->getScratchRSrcReg(), MVT::v4i32);
1881bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(
SDValue Addr,
SDValue &SRsrc,
1884 SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1885 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
1887 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset,
Offset, Offen, Idxen, Addr64))
1906bool AMDGPUDAGToDAGISel::SelectBUFSOffset(
SDValue ByteOffsetNode,
1908 if (Subtarget->hasRestrictedSOffset() &&
isNullConstant(ByteOffsetNode)) {
1909 SOffset =
CurDAG->getRegister(AMDGPU::SGPR_NULL, MVT::i32);
1913 SOffset = ByteOffsetNode;
1931bool AMDGPUDAGToDAGISel::SelectFlatOffsetImpl(
1935 int64_t OffsetVal = 0;
1939 Addr = Expand32BitAddress(Addr);
1941 bool CanHaveFlatSegmentOffsetBug =
1942 Subtarget->hasFlatSegmentOffsetBug() &&
1943 FlatVariant == FlatAddrSpace::FLAT &&
1946 if (Subtarget->hasFlatInstOffsets() && !CanHaveFlatSegmentOffsetBug) {
1948 if (isBaseWithConstantOffset64(Addr, N0, N1) &&
1949 (FlatVariant != FlatAddrSpace::FlatScratch ||
1950 isFlatScratchBaseLegal(Addr))) {
1958 if (COffsetVal == 0 || FlatVariant != FlatAddrSpace::FLAT || IsInBounds) {
1959 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
1960 if (
TII->isLegalFLATOffset(COffsetVal, AS, FlatVariant)) {
1962 OffsetVal = COffsetVal;
1977 std::tie(OffsetVal, RemainderOffset) =
1978 TII->splitFlatOffset(COffsetVal, AS, FlatVariant);
1981 getMaterializedScalarImm32(
Lo_32(RemainderOffset),
DL);
1988 unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1989 if (Subtarget->hasAddNoCarryInsts()) {
1990 AddOp = AMDGPU::V_ADD_U32_e64;
1999 CurDAG->getTargetConstant(AMDGPU::sub0,
DL, MVT::i32);
2001 CurDAG->getTargetConstant(AMDGPU::sub1,
DL, MVT::i32);
2003 SDNode *N0Lo =
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
2004 DL, MVT::i32, N0, Sub0);
2005 SDNode *N0Hi =
CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
2006 DL, MVT::i32, N0, Sub1);
2009 getMaterializedScalarImm32(
Hi_32(RemainderOffset),
DL);
2011 SDVTList VTs =
CurDAG->getVTList(MVT::i32, MVT::i1);
2014 CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64,
DL, VTs,
2015 {AddOffsetLo,
SDValue(N0Lo, 0), Clamp});
2017 SDNode *Addc =
CurDAG->getMachineNode(
2018 AMDGPU::V_ADDC_U32_e64,
DL, VTs,
2022 CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID,
DL,
2027 MVT::i64, RegSequenceArgs),
2036 Offset =
CurDAG->getSignedTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2040bool AMDGPUDAGToDAGISel::SelectFlatOffset(
SDNode *
N,
SDValue Addr,
2043 return SelectFlatOffsetImpl(
N, Addr, VAddr,
Offset,
2047bool AMDGPUDAGToDAGISel::SelectGlobalOffset(
SDNode *
N,
SDValue Addr,
2050 return SelectFlatOffsetImpl(
N, Addr, VAddr,
Offset,
2054bool AMDGPUDAGToDAGISel::SelectScratchOffset(
SDNode *
N,
SDValue Addr,
2057 return SelectFlatOffsetImpl(
N, Addr, VAddr,
Offset,
2065 if (
Op.getValueType() == MVT::i32)
2080bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(
SDNode *
N,
SDValue Addr,
2083 bool NeedIOffset)
const {
2085 int64_t ImmOffset = 0;
2086 ScaleOffset =
false;
2088 Addr = Expand32BitAddress(Addr);
2094 if (isBaseWithConstantOffset64(Addr,
LHS,
RHS)) {
2096 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
2100 FlatAddrSpace::FlatGlobal)) {
2102 ImmOffset = COffsetVal;
2103 }
else if (!
LHS->isDivergent()) {
2104 if (COffsetVal != 0) {
2109 int64_t SplitImmOffset = 0, RemainderOffset = COffsetVal;
2111 std::tie(SplitImmOffset, RemainderOffset) =
TII->splitFlatOffset(
2115 if (Subtarget->hasSignedGVSOffset() ?
isInt<32>(RemainderOffset)
2117 SDNode *VMov =
CurDAG->getMachineNode(
2118 AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
2119 CurDAG->getTargetConstant(
Lo_32(RemainderOffset), SDLoc(),
2123 Offset =
CurDAG->getSignedTargetConstant(SplitImmOffset, SDLoc(),
2134 unsigned NumLiterals =
2135 !
TII->isInlineConstant(APInt(32,
Lo_32(COffsetVal))) +
2136 !
TII->isInlineConstant(APInt(32,
Hi_32(COffsetVal)));
2137 if (Subtarget->getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals)
2146 if (!
LHS->isDivergent()) {
2149 ScaleOffset = SelectScaleOffset(
N,
RHS, Subtarget->hasSignedGVSOffset());
2151 RHS, Subtarget->hasSignedGVSOffset(),
CurDAG)) {
2158 if (!SAddr && !
RHS->isDivergent()) {
2160 ScaleOffset = SelectScaleOffset(
N,
LHS, Subtarget->hasSignedGVSOffset());
2162 LHS, Subtarget->hasSignedGVSOffset(),
CurDAG)) {
2169 Offset =
CurDAG->getSignedTargetConstant(ImmOffset, SDLoc(), MVT::i32);
2174 if (Subtarget->hasScaleOffset() &&
2175 (Addr.
getOpcode() == (Subtarget->hasSignedGVSOffset()
2190 Offset =
CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i32);
2202 CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, SDLoc(Addr), MVT::i32,
2203 CurDAG->getTargetConstant(0, SDLoc(), MVT::i32));
2205 Offset =
CurDAG->getSignedTargetConstant(ImmOffset, SDLoc(), MVT::i32);
2209bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(
SDNode *
N,
SDValue Addr,
2214 if (!SelectGlobalSAddr(
N, Addr, SAddr, VOffset,
Offset, ScaleOffset))
2222bool AMDGPUDAGToDAGISel::SelectGlobalSAddrCPol(
SDNode *
N,
SDValue Addr,
2227 if (!SelectGlobalSAddr(
N, Addr, SAddr, VOffset,
Offset, ScaleOffset))
2232 N->getConstantOperandVal(
N->getNumOperands() - 1) & ~AMDGPU::CPol::SCAL;
2238bool AMDGPUDAGToDAGISel::SelectGlobalSAddrCPolM0(
SDNode *
N,
SDValue Addr,
2244 if (!SelectGlobalSAddr(
N, Addr, SAddr, VOffset,
Offset, ScaleOffset))
2249 N->getConstantOperandVal(
N->getNumOperands() - 2) & ~AMDGPU::CPol::SCAL;
2255bool AMDGPUDAGToDAGISel::SelectGlobalSAddrGLC(
SDNode *
N,
SDValue Addr,
2260 if (!SelectGlobalSAddr(
N, Addr, SAddr, VOffset,
Offset, ScaleOffset))
2264 CPol =
CurDAG->getTargetConstant(CPolVal, SDLoc(), MVT::i32);
2268bool AMDGPUDAGToDAGISel::SelectGlobalSAddrNoIOffset(
SDNode *
N,
SDValue Addr,
2274 if (!SelectGlobalSAddr(
N, Addr, SAddr, VOffset, DummyOffset, ScaleOffset,
2280 N->getConstantOperandVal(
N->getNumOperands() - 1) & ~AMDGPU::CPol::SCAL;
2286bool AMDGPUDAGToDAGISel::SelectGlobalSAddrNoIOffsetM0(
SDNode *
N,
SDValue Addr,
2292 if (!SelectGlobalSAddr(
N, Addr, SAddr, VOffset, DummyOffset, ScaleOffset,
2313 FI->getValueType(0));
2323bool AMDGPUDAGToDAGISel::SelectScratchSAddr(
SDNode *Parent,
SDValue Addr,
2332 int64_t COffsetVal = 0;
2334 if (
CurDAG->isBaseWithConstantOffset(Addr) && isFlatScratchBaseLegal(Addr)) {
2343 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
2346 FlatAddrSpace::FlatScratch)) {
2347 int64_t SplitImmOffset, RemainderOffset;
2348 std::tie(SplitImmOffset, RemainderOffset) =
TII->splitFlatOffset(
2351 COffsetVal = SplitImmOffset;
2355 ? getMaterializedScalarImm32(
Lo_32(RemainderOffset),
DL)
2356 :
CurDAG->getSignedTargetConstant(RemainderOffset,
DL, MVT::i32);
2357 SAddr =
SDValue(
CurDAG->getMachineNode(AMDGPU::S_ADD_I32,
DL, MVT::i32,
2362 Offset =
CurDAG->getSignedTargetConstant(COffsetVal,
DL, MVT::i32);
2368bool AMDGPUDAGToDAGISel::checkFlatScratchSVSSwizzleBug(
2370 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
2376 KnownBits VKnown =
CurDAG->computeKnownBits(VAddr);
2383 return (VMax & 3) + (
SMax & 3) >= 4;
2386bool AMDGPUDAGToDAGISel::SelectScratchSVAddr(
SDNode *
N,
SDValue Addr,
2390 int64_t ImmOffset = 0;
2394 if (isBaseWithConstantOffset64(Addr,
LHS,
RHS)) {
2396 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
2401 ImmOffset = COffsetVal;
2402 }
else if (!
LHS->isDivergent() && COffsetVal > 0) {
2406 int64_t SplitImmOffset, RemainderOffset;
2407 std::tie(SplitImmOffset, RemainderOffset) =
2412 SDNode *VMov =
CurDAG->getMachineNode(
2413 AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
2414 CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
2417 if (!isFlatScratchBaseLegal(Addr))
2419 if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, SplitImmOffset))
2421 Offset =
CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i32);
2422 CPol =
CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2434 if (!
LHS->isDivergent() &&
RHS->isDivergent()) {
2437 }
else if (!
RHS->isDivergent() &&
LHS->isDivergent()) {
2444 if (OrigAddr != Addr) {
2445 if (!isFlatScratchBaseLegalSVImm(OrigAddr))
2448 if (!isFlatScratchBaseLegalSV(OrigAddr))
2452 if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, ImmOffset))
2455 Offset =
CurDAG->getSignedTargetConstant(ImmOffset, SDLoc(), MVT::i32);
2457 bool ScaleOffset = SelectScaleOffset(
N, VAddr,
true );
2466bool AMDGPUDAGToDAGISel::isSOffsetLegalWithImmOffset(
SDValue *SOffset,
2469 int64_t ImmOffset)
const {
2470 if (!IsBuffer && !Imm32Only && ImmOffset < 0 &&
2472 KnownBits SKnown =
CurDAG->computeKnownBits(*SOffset);
2484 bool IsSigned,
bool IsSMEM)
const {
2485 bool ScaleOffset =
false;
2486 if (!Subtarget->hasScaleOffset() || !
Offset)
2500 (IsSigned &&
Offset.getOpcode() == AMDGPUISD::MUL_I24) ||
2501 Offset.getOpcode() == AMDGPUISD::MUL_U24 ||
2502 (
Offset.isMachineOpcode() &&
2503 Offset.getMachineOpcode() ==
2504 (IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO
2505 : AMDGPU::S_MUL_U64_U32_PSEUDO))) {
2507 ScaleOffset =
C->getZExtValue() ==
Size;
2516 if (
Off.getValueType() == MVT::i64) {
2518 if (
Known.countMaxActiveBits() + Bits > 32)
2522 SDNodeFlags
Flags =
Off->getFlags();
2524 if (IsSigned && !
Flags.hasNoSignedWrap() &&
2525 CurDAG->ComputeNumSignBits(Idx) <= Bits)
2527 if (!IsSigned && !
Flags.hasNoUnsignedWrap() &&
2528 CurDAG->computeKnownBits(Idx).countMinLeadingZeros() < Bits)
2539bool AMDGPUDAGToDAGISel::SelectSMRDOffset(
SDNode *
N,
SDValue ByteOffsetNode,
2541 bool Imm32Only,
bool IsBuffer,
2542 bool HasSOffset, int64_t ImmOffset,
2543 bool *ScaleOffset)
const {
2545 "Cannot match both soffset and offset at the same time!");
2550 *ScaleOffset = SelectScaleOffset(
N, ByteOffsetNode,
false ,
2561 *SOffset = ByteOffsetNode;
2562 return isSOffsetLegalWithImmOffset(SOffset, Imm32Only, IsBuffer,
2568 return isSOffsetLegalWithImmOffset(SOffset, Imm32Only, IsBuffer,
2575 SDLoc SL(ByteOffsetNode);
2579 int64_t ByteOffset = IsBuffer ?
C->getZExtValue() :
C->getSExtValue();
2581 *Subtarget, ByteOffset, IsBuffer, HasSOffset);
2582 if (EncodedOffset &&
Offset && !Imm32Only) {
2583 *
Offset =
CurDAG->getSignedTargetConstant(*EncodedOffset, SL, MVT::i32);
2592 if (EncodedOffset &&
Offset && Imm32Only) {
2593 *
Offset =
CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
2601 SDValue C32Bit =
CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
2603 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0);
2610SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(
SDValue Addr)
const {
2618 const SIMachineFunctionInfo *
Info =
MF.getInfo<SIMachineFunctionInfo>();
2619 unsigned AddrHiVal =
Info->get32BitAddressHighBits();
2620 SDValue AddrHi =
CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
2623 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
2625 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
2626 SDValue(
CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
2628 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
2631 return SDValue(
CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
2638bool AMDGPUDAGToDAGISel::SelectSMRDBaseOffset(
SDNode *
N,
SDValue Addr,
2641 bool IsBuffer,
bool HasSOffset,
2643 bool *ScaleOffset)
const {
2645 assert(!Imm32Only && !IsBuffer);
2648 if (!SelectSMRDBaseOffset(
N, Addr,
B,
nullptr,
Offset,
false,
false,
true))
2653 ImmOff =
C->getSExtValue();
2655 return SelectSMRDBaseOffset(
N,
B, SBase, SOffset,
nullptr,
false,
false,
2656 true, ImmOff, ScaleOffset);
2676 if (SelectSMRDOffset(
N, N1, SOffset,
Offset, Imm32Only, IsBuffer, HasSOffset,
2677 ImmOffset, ScaleOffset)) {
2681 if (SelectSMRDOffset(
N, N0, SOffset,
Offset, Imm32Only, IsBuffer, HasSOffset,
2682 ImmOffset, ScaleOffset)) {
2691 bool Imm32Only,
bool *ScaleOffset)
const {
2692 if (SelectSMRDBaseOffset(
N, Addr, SBase, SOffset,
Offset, Imm32Only,
2695 SBase = Expand32BitAddress(SBase);
2700 SBase = Expand32BitAddress(Addr);
2701 *
Offset =
CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
2708bool AMDGPUDAGToDAGISel::SelectSMRDImm(
SDValue Addr,
SDValue &SBase,
2710 return SelectSMRD(
nullptr, Addr, SBase,
nullptr,
2714bool AMDGPUDAGToDAGISel::SelectSMRDImm32(
SDValue Addr,
SDValue &SBase,
2717 return SelectSMRD(
nullptr, Addr, SBase,
nullptr,
2724 if (!SelectSMRD(
N, Addr, SBase, &SOffset,
nullptr,
2725 false, &ScaleOffset))
2729 SDLoc(
N), MVT::i32);
2733bool AMDGPUDAGToDAGISel::SelectSMRDSgprImm(
SDNode *
N,
SDValue Addr,
2738 if (!SelectSMRD(
N, Addr, SBase, &SOffset, &
Offset,
false, &ScaleOffset))
2742 SDLoc(
N), MVT::i32);
2747 return SelectSMRDOffset(
nullptr,
N,
nullptr, &
Offset,
2751bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(
SDValue N,
2754 return SelectSMRDOffset(
nullptr,
N,
nullptr, &
Offset,
2758bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgprImm(
SDValue N,
SDValue &SOffset,
2762 return N.getValueType() == MVT::i32 &&
2763 SelectSMRDBaseOffset(
nullptr,
N, SOffset,
2768bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(
SDValue Index,
2773 if (
CurDAG->isBaseWithConstantOffset(Index)) {
2782 if (
C1->getSExtValue() <= 0 ||
CurDAG->SignBitIsZero(N0) ||
2798SDNode *AMDGPUDAGToDAGISel::getBFE32(
bool IsSigned,
const SDLoc &
DL,
2802 unsigned Opcode = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2806 return CurDAG->getMachineNode(Opcode,
DL, MVT::i32, Val,
Off, W);
2808 unsigned Opcode = IsSigned ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2812 uint32_t PackedVal =
Offset | (Width << 16);
2813 SDValue PackedConst =
CurDAG->getTargetConstant(PackedVal,
DL, MVT::i32);
2815 return CurDAG->getMachineNode(Opcode,
DL, MVT::i32, Val, PackedConst);
2818void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(
SDNode *
N) {
2823 const SDValue &Shl =
N->getOperand(0);
2828 uint32_t BVal =
B->getZExtValue();
2829 uint32_t CVal =
C->getZExtValue();
2831 if (0 < BVal && BVal <= CVal && CVal < 32) {
2841void AMDGPUDAGToDAGISel::SelectS_BFE(
SDNode *
N) {
2842 switch (
N->getOpcode()) {
2844 if (
N->getOperand(0).getOpcode() ==
ISD::SRL) {
2847 const SDValue &Srl =
N->getOperand(0);
2851 if (Shift && Mask) {
2853 uint32_t MaskVal =
Mask->getZExtValue();
2865 if (
N->getOperand(0).getOpcode() ==
ISD::AND) {
2872 if (Shift && Mask) {
2874 uint32_t MaskVal =
Mask->getZExtValue() >> ShiftVal;
2883 }
else if (
N->getOperand(0).getOpcode() ==
ISD::SHL) {
2884 SelectS_BFEFromShifts(
N);
2889 if (
N->getOperand(0).getOpcode() ==
ISD::SHL) {
2890 SelectS_BFEFromShifts(
N);
2905 unsigned Width =
cast<VTSDNode>(
N->getOperand(1))->getVT().getSizeInBits();
2915bool AMDGPUDAGToDAGISel::isCBranchSCC(
const SDNode *
N)
const {
2917 if (!
N->hasOneUse())
2927 MVT VT =
Cond.getOperand(0).getSimpleValueType();
2931 if (VT == MVT::i64) {
2934 Subtarget->hasScalarCompareEq64();
2937 if ((VT == MVT::f16 || VT == MVT::f32) && Subtarget->hasSALUFloatInsts())
2970void AMDGPUDAGToDAGISel::SelectBRCOND(
SDNode *
N) {
2973 if (
Cond.isUndef()) {
2974 CurDAG->SelectNodeTo(
N, AMDGPU::SI_BR_UNDEF, MVT::Other,
2975 N->getOperand(2),
N->getOperand(0));
2979 const SIRegisterInfo *
TRI = Subtarget->getRegisterInfo();
2981 bool UseSCCBr = isCBranchSCC(
N) && isUniformBr(
N);
2982 bool AndExec = !UseSCCBr;
2983 bool Negate =
false;
2986 Cond->getOperand(0)->getOpcode() == AMDGPUISD::SETCC) {
2988 auto CC =
Cond->getOperandCondCode(2);
3001 bool NegatedBallot =
false;
3004 UseSCCBr = !BallotCond->isDivergent();
3005 Negate = Negate ^ NegatedBallot;
3020 UseSCCBr ? (Negate ? AMDGPU::S_CBRANCH_SCC0 : AMDGPU::S_CBRANCH_SCC1)
3021 : (Negate ? AMDGPU::S_CBRANCH_VCCZ : AMDGPU::S_CBRANCH_VCCNZ);
3022 Register CondReg = UseSCCBr ? AMDGPU::SCC :
TRI->getVCC();
3041 Subtarget->isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64, SL,
3043 CurDAG->getRegister(Subtarget->isWave32() ? AMDGPU::EXEC_LO
3051 CurDAG->SelectNodeTo(
N, BrOp, MVT::Other,
3056void AMDGPUDAGToDAGISel::SelectFP_EXTEND(
SDNode *
N) {
3057 if (Subtarget->hasSALUFloatInsts() &&
N->getValueType(0) == MVT::f32 &&
3058 !
N->isDivergent()) {
3060 if (Src.getValueType() == MVT::f16) {
3062 CurDAG->SelectNodeTo(
N, AMDGPU::S_CVT_HI_F32_F16,
N->getVTList(),
3072void AMDGPUDAGToDAGISel::SelectDSAppendConsume(
SDNode *
N,
unsigned IntrID) {
3075 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
3076 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
3081 MachineMemOperand *MMO =
M->getMemOperand();
3085 if (
CurDAG->isBaseWithConstantOffset(Ptr)) {
3090 if (isDSOffsetLegal(PtrBase, OffsetVal.
getZExtValue())) {
3091 N = glueCopyToM0(
N, PtrBase);
3092 Offset =
CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
3097 N = glueCopyToM0(
N, Ptr);
3098 Offset =
CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
3103 CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
3108 SDNode *Selected =
CurDAG->SelectNodeTo(
N,
Opc,
N->getVTList(),
Ops);
3114void AMDGPUDAGToDAGISel::SelectDSBvhStackIntrinsic(
SDNode *
N,
unsigned IntrID) {
3117 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
3118 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
3119 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
3121 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
3122 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
3124 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
3125 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
3128 SDValue Ops[] = {
N->getOperand(2),
N->getOperand(3),
N->getOperand(4),
3129 N->getOperand(5),
N->getOperand(0)};
3132 MachineMemOperand *MMO =
M->getMemOperand();
3133 SDNode *Selected =
CurDAG->SelectNodeTo(
N,
Opc,
N->getVTList(),
Ops);
3137void AMDGPUDAGToDAGISel::SelectTensorLoadStore(
SDNode *
N,
unsigned IntrID) {
3138 bool IsLoad = IntrID == Intrinsic::amdgcn_tensor_load_to_lds;
3140 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3152 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3153 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3165 (void)
CurDAG->SelectNodeTo(
N,
Opc, MVT::Other, TensorOps);
3170 case Intrinsic::amdgcn_ds_gws_init:
3171 return AMDGPU::DS_GWS_INIT;
3172 case Intrinsic::amdgcn_ds_gws_barrier:
3173 return AMDGPU::DS_GWS_BARRIER;
3174 case Intrinsic::amdgcn_ds_gws_sema_v:
3175 return AMDGPU::DS_GWS_SEMA_V;
3176 case Intrinsic::amdgcn_ds_gws_sema_br:
3177 return AMDGPU::DS_GWS_SEMA_BR;
3178 case Intrinsic::amdgcn_ds_gws_sema_p:
3179 return AMDGPU::DS_GWS_SEMA_P;
3180 case Intrinsic::amdgcn_ds_gws_sema_release_all:
3181 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
3187void AMDGPUDAGToDAGISel::SelectDS_GWS(
SDNode *
N,
unsigned IntrID) {
3188 if (!Subtarget->hasGWS() ||
3189 (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
3190 !Subtarget->hasGWSSemaReleaseAll())) {
3197 const bool HasVSrc =
N->getNumOperands() == 4;
3198 assert(HasVSrc ||
N->getNumOperands() == 3);
3201 SDValue BaseOffset =
N->getOperand(HasVSrc ? 3 : 2);
3204 MachineMemOperand *MMO =
M->getMemOperand();
3217 glueCopyToM0(
N,
CurDAG->getTargetConstant(0, SL, MVT::i32));
3218 ImmOffset = ConstOffset->getZExtValue();
3220 if (
CurDAG->isBaseWithConstantOffset(BaseOffset)) {
3229 =
CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
3233 =
CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
3235 CurDAG->getTargetConstant(16, SL, MVT::i32));
3236 glueCopyToM0(
N,
SDValue(M0Base, 0));
3240 SDValue OffsetField =
CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
3244 const MCInstrDesc &InstrDesc =
TII->get(
Opc);
3245 int Data0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::data0);
3251 const SIRegisterInfo *
TRI = Subtarget->getRegisterInfo();
3254 MVT DataVT =
Data.getValueType().getSimpleVT();
3255 if (
TRI->isTypeLegalForClass(*DataRC, DataVT)) {
3257 Ops.push_back(
N->getOperand(2));
3263 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
3265 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL, MVT::i32),
3267 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32)};
3270 SL, MVT::v2i32, RegSeqOps),
3275 Ops.push_back(OffsetField);
3276 Ops.push_back(Chain);
3278 SDNode *Selected =
CurDAG->SelectNodeTo(
N,
Opc,
N->getVTList(),
Ops);
3282void AMDGPUDAGToDAGISel::SelectInterpP1F16(
SDNode *
N) {
3283 if (Subtarget->getLDSBankCount() != 16) {
3313 SDVTList VTs =
CurDAG->getVTList(MVT::f32, MVT::Other);
3316 CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32,
DL, VTs, {
3317 CurDAG->getTargetConstant(2,
DL, MVT::i32),
3323 SDNode *InterpP1LV =
3324 CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16,
DL, MVT::f32, {
3325 CurDAG->getTargetConstant(0,
DL, MVT::i32),
3329 CurDAG->getTargetConstant(0,
DL, MVT::i32),
3332 CurDAG->getTargetConstant(0,
DL, MVT::i1),
3333 CurDAG->getTargetConstant(0,
DL, MVT::i32),
3340void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(
SDNode *
N) {
3341 unsigned IntrID =
N->getConstantOperandVal(1);
3343 case Intrinsic::amdgcn_ds_append:
3344 case Intrinsic::amdgcn_ds_consume: {
3345 if (
N->getValueType(0) != MVT::i32)
3347 SelectDSAppendConsume(
N, IntrID);
3350 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
3351 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
3352 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
3353 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
3354 SelectDSBvhStackIntrinsic(
N, IntrID);
3356 case Intrinsic::amdgcn_init_whole_wave:
3357 CurDAG->getMachineFunction()
3358 .getInfo<SIMachineFunctionInfo>()
3359 ->setInitWholeWave();
3366void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(
SDNode *
N) {
3367 unsigned IntrID =
N->getConstantOperandVal(0);
3368 unsigned Opcode = AMDGPU::INSTRUCTION_LIST_END;
3369 SDNode *ConvGlueNode =
N->getGluedNode();
3375 CurDAG->getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE, {},
3376 MVT::Glue,
SDValue(ConvGlueNode, 0));
3378 ConvGlueNode =
nullptr;
3381 case Intrinsic::amdgcn_wqm:
3382 Opcode = AMDGPU::WQM;
3384 case Intrinsic::amdgcn_softwqm:
3385 Opcode = AMDGPU::SOFT_WQM;
3387 case Intrinsic::amdgcn_wwm:
3388 case Intrinsic::amdgcn_strict_wwm:
3389 Opcode = AMDGPU::STRICT_WWM;
3391 case Intrinsic::amdgcn_strict_wqm:
3392 Opcode = AMDGPU::STRICT_WQM;
3394 case Intrinsic::amdgcn_interp_p1_f16:
3395 SelectInterpP1F16(
N);
3397 case Intrinsic::amdgcn_permlane16_swap:
3398 case Intrinsic::amdgcn_permlane32_swap: {
3399 if ((IntrID == Intrinsic::amdgcn_permlane16_swap &&
3400 !Subtarget->hasPermlane16Swap()) ||
3401 (IntrID == Intrinsic::amdgcn_permlane32_swap &&
3402 !Subtarget->hasPermlane32Swap())) {
3407 Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
3408 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
3409 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
3413 NewOps.push_back(
SDValue(ConvGlueNode, 0));
3415 bool FI =
N->getConstantOperandVal(3);
3416 NewOps[2] =
CurDAG->getTargetConstant(
3419 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(), NewOps);
3427 if (Opcode != AMDGPU::INSTRUCTION_LIST_END) {
3429 CurDAG->SelectNodeTo(
N, Opcode,
N->getVTList(), {Src});
3434 NewOps.push_back(
SDValue(ConvGlueNode, 0));
3435 CurDAG->MorphNodeTo(
N,
N->getOpcode(),
N->getVTList(), NewOps);
3439void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(
SDNode *
N) {
3440 unsigned IntrID =
N->getConstantOperandVal(1);
3442 case Intrinsic::amdgcn_ds_gws_init:
3443 case Intrinsic::amdgcn_ds_gws_barrier:
3444 case Intrinsic::amdgcn_ds_gws_sema_v:
3445 case Intrinsic::amdgcn_ds_gws_sema_br:
3446 case Intrinsic::amdgcn_ds_gws_sema_p:
3447 case Intrinsic::amdgcn_ds_gws_sema_release_all:
3448 SelectDS_GWS(
N, IntrID);
3450 case Intrinsic::amdgcn_tensor_load_to_lds:
3451 case Intrinsic::amdgcn_tensor_store_from_lds:
3452 SelectTensorLoadStore(
N, IntrID);
3461void AMDGPUDAGToDAGISel::SelectWAVE_ADDRESS(
SDNode *
N) {
3463 CurDAG->getTargetConstant(Subtarget->getWavefrontSizeLog2(), SDLoc(
N), MVT::i32);
3464 CurDAG->SelectNodeTo(
N, AMDGPU::S_LSHR_B32,
N->getVTList(),
3465 {N->getOperand(0), Log2WaveSize});
3468void AMDGPUDAGToDAGISel::SelectSTACKRESTORE(
SDNode *
N) {
3483 Subtarget->getWavefrontSizeLog2(), SL, MVT::i32);
3485 if (
N->isDivergent()) {
3486 SrcVal =
SDValue(
CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL,
3491 CopyVal =
SDValue(
CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
3492 {SrcVal, Log2WaveSize}),
3496 SDValue CopyToSP =
CurDAG->getCopyToReg(
N->getOperand(0), SL,
SP, CopyVal);
3500void AMDGPUDAGToDAGISel::SelectWRITE_REGISTER(
SDNode *
N) {
3507 CurDAG->getMachineFunction());
3511 "invalid register \"" + Twine(RegStr->
getString()) +
3512 "\" for llvm.write_register",
3513 Fn,
N->getDebugLoc()));
3525 }
else if (VT == MVT::i32) {
3526 CopyVal =
SDValue(
CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL,
3532 CurDAG->getTargetExtractSubreg(AMDGPU::sub0, SL, MVT::i32, SrcVal);
3534 CurDAG->getTargetExtractSubreg(AMDGPU::sub1, SL, MVT::i32, SrcVal);
3536 CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
Lo),
3539 CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
Hi),
3541 CopyVal = emitRegSequence(*
CurDAG, AMDGPU::SReg_64RegClassID, VT, {
Lo,
Hi},
3542 {AMDGPU::sub0, AMDGPU::sub1}, SL);
3549bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(
SDValue In,
SDValue &Src,
3551 bool IsCanonicalizing,
3552 bool AllowAbs)
const {
3558 Src = Src.getOperand(0);
3559 }
else if (Src.getOpcode() ==
ISD::FSUB && IsCanonicalizing) {
3563 if (
LHS &&
LHS->isZero()) {
3565 Src = Src.getOperand(1);
3569 if (AllowAbs && Src.getOpcode() ==
ISD::FABS) {
3571 Src = Src.getOperand(0);
3584 if (IsCanonicalizing)
3599 EVT VT = Src.getValueType();
3601 (VT != MVT::i32 && VT != MVT::v2i32 && VT != MVT::i64))
3608 auto ReplaceSrc = [&]() ->
SDValue {
3610 return Src.getOperand(0);
3615 Src.getValueType(),
LHS, Index);
3641 if (SelectVOP3ModsImpl(In, Src, Mods,
true,
3643 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3650bool AMDGPUDAGToDAGISel::SelectVOP3ModsNonCanonicalizing(
3653 if (SelectVOP3ModsImpl(In, Src, Mods,
false,
3655 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3662bool AMDGPUDAGToDAGISel::SelectVOP3BMods(
SDValue In,
SDValue &Src,
3665 if (SelectVOP3ModsImpl(In, Src, Mods,
3668 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3675bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(
SDValue In,
SDValue &Src)
const {
3683bool AMDGPUDAGToDAGISel::SelectVOP3NoOpSelMods(
SDValue In,
SDValue &Src)
const {
3688 return SelectVOP3NoMods(In, Src);
3691bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(
SDValue In,
SDValue &Src,
3695 if (SelectVOP3ModsImpl(In, Src, Mods,
3700 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3707bool AMDGPUDAGToDAGISel::SelectVINTERPMods(
SDValue In,
SDValue &Src,
3709 return SelectVINTERPModsImpl(In, Src, SrcMods,
false);
3712bool AMDGPUDAGToDAGISel::SelectVINTERPModsHi(
SDValue In,
SDValue &Src,
3714 return SelectVINTERPModsImpl(In, Src, SrcMods,
true);
3717bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(
SDValue In,
SDValue &Src,
3721 Clamp =
CurDAG->getTargetConstant(0,
DL, MVT::i1);
3722 Omod =
CurDAG->getTargetConstant(0,
DL, MVT::i1);
3724 return SelectVOP3Mods(In, Src, SrcMods);
3727bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(
SDValue In,
SDValue &Src,
3731 Clamp =
CurDAG->getTargetConstant(0,
DL, MVT::i1);
3732 Omod =
CurDAG->getTargetConstant(0,
DL, MVT::i1);
3734 return SelectVOP3BMods(In, Src, SrcMods);
3737bool AMDGPUDAGToDAGISel::SelectVOP3OMods(
SDValue In,
SDValue &Src,
3742 Clamp =
CurDAG->getTargetConstant(0,
DL, MVT::i1);
3743 Omod =
CurDAG->getTargetConstant(0,
DL, MVT::i1);
3748bool AMDGPUDAGToDAGISel::SelectVOP3PModsPkFmaOpSel0(
SDValue In,
SDValue &Src,
3750 return SelectVOP3PMods(In, Src, SrcMods,
false,
3751 Subtarget->hasPkF32InvalidOpSel() &&
3752 In.getValueType() == MVT::v2f32);
3755bool AMDGPUDAGToDAGISel::SelectVOP3PModsPkAddMulOpSel0(
SDValue In,
SDValue &Src,
3757 if (!SelectVOP3PMods(In, Src, SrcMods))
3760 return !Subtarget->hasPkF32InvalidOpSel() ||
3761 In.getValueType() != MVT::v2f32 ||
3766bool AMDGPUDAGToDAGISel::SelectVOP3PMods(
SDValue In,
SDValue &Src,
3768 bool NoOpSel0)
const {
3775 Src = Src.getOperand(0);
3779 bool HasOpSel = Src.getValueSizeInBits() != 128;
3782 (!IsDOT || !Subtarget->hasDOTOpSelHazard()) &&
3783 (!NoOpSel0 || !isLoEltExtractHi(Src))) {
3784 unsigned VecMods = Mods;
3786 SDValue Lo = stripBitcast(Src.getOperand(0));
3787 SDValue Hi = stripBitcast(Src.getOperand(1));
3790 Lo = stripBitcast(
Lo.getOperand(0));
3795 Hi = stripBitcast(
Hi.getOperand(0));
3807 unsigned VecSize = Src.getValueSizeInBits();
3808 Lo = stripExtractLoElt(
Lo);
3809 Hi = stripExtractLoElt(
Hi);
3811 if (
Lo.getValueSizeInBits() > VecSize) {
3812 Lo =
CurDAG->getTargetExtractSubreg(
3813 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
3817 if (
Hi.getValueSizeInBits() > VecSize) {
3818 Hi =
CurDAG->getTargetExtractSubreg(
3819 (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
3823 assert(
Lo.getValueSizeInBits() <= VecSize &&
3824 Hi.getValueSizeInBits() <= VecSize);
3826 if (
Lo ==
Hi && !isInlineImmediate(
Lo.getNode())) {
3830 if (VecSize ==
Lo.getValueSizeInBits()) {
3832 }
else if (VecSize == 32) {
3833 Src = createVOP3PSrc32FromLo16(
Lo, Src,
CurDAG, Subtarget);
3835 assert((
Lo.getValueSizeInBits() == 32 && VecSize == 64) ||
3836 (
Lo.getValueSizeInBits() == 64 && VecSize == 128));
3840 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL,
3841 Lo.getValueType()), 0);
3842 const SIRegisterInfo *
TRI = Subtarget->getRegisterInfo();
3847 auto RC =
Lo->isDivergent() ?
TRI->getVGPRClassForBitWidth(VecSize)
3848 :
TRI->getSGPRClassForBitWidth(VecSize);
3849 unsigned NumRegs =
Lo.getValueSizeInBits() == 32 ? 1 : 2;
3851 CurDAG->getTargetConstant(RC->getID(), SL, MVT::i32),
Lo,
3852 CurDAG->getTargetConstant(
TRI->getSubRegFromChannel(0, NumRegs), SL,
3857 CurDAG->getTargetConstant(
3858 TRI->getSubRegFromChannel(NumRegs, NumRegs), SL, MVT::i32)};
3860 Src =
SDValue(
CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
3861 Src.getValueType(),
Ops), 0);
3865 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3871 .bitcastToAPInt().getZExtValue();
3873 Src =
CurDAG->getTargetConstant(
Lit, SDLoc(In), MVT::i64);
3874 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3881 Src.getNumOperands() == 2) {
3886 assert(Src.getValueSizeInBits() != 128 &&
3887 "<2 x 64> VECTOR_SHUFFLE should not be legal.");
3890 ArrayRef<int>
Mask = SVN->getMask();
3892 if (Mask[0] < 2 && Mask[1] < 2 && (!NoOpSel0 || Mask[0] != 1)) {
3894 SDValue ShuffleSrc = SVN->getOperand(0);
3907 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3915 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3919bool AMDGPUDAGToDAGISel::SelectVOP3PModsDOT(
SDValue In,
SDValue &Src,
3921 return SelectVOP3PMods(In, Src, SrcMods,
true);
3924bool AMDGPUDAGToDAGISel::SelectVOP3PNoModsDOT(
SDValue In,
SDValue &Src)
const {
3926 SelectVOP3PMods(In, SrcTmp, SrcModsTmp,
true);
3935bool AMDGPUDAGToDAGISel::SelectVOP3PModsF32(
SDValue In,
SDValue &Src,
3937 SelectVOP3Mods(In, Src, SrcMods);
3940 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3944bool AMDGPUDAGToDAGISel::SelectVOP3PNoModsF32(
SDValue In,
SDValue &Src)
const {
3946 SelectVOP3PModsF32(In, SrcTmp, SrcModsTmp);
3955bool AMDGPUDAGToDAGISel::SelectWMMAOpSelVOP3PMods(
SDValue In,
3958 assert(
C->getAPIntValue().getBitWidth() == 1 &&
"expected i1 value");
3961 unsigned SrcVal =
C->getZExtValue();
3965 Src =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
3972 unsigned DstRegClass;
3974 switch (Elts.
size()) {
3976 DstRegClass = AMDGPU::VReg_256RegClassID;
3980 DstRegClass = AMDGPU::VReg_128RegClassID;
3984 DstRegClass = AMDGPU::VReg_64RegClassID;
3992 Ops.push_back(
CurDAG->getTargetConstant(DstRegClass,
DL, MVT::i32));
3993 for (
unsigned i = 0; i < Elts.
size(); ++i) {
3994 Ops.push_back(Elts[i]);
3995 Ops.push_back(
CurDAG->getTargetConstant(
3998 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
DL, DstTy,
Ops);
4005 assert(
"unhandled Reg sequence size" &&
4006 (Elts.
size() == 8 || Elts.
size() == 16));
4010 for (
unsigned i = 0; i < Elts.
size(); i += 2) {
4011 SDValue LoSrc = stripExtractLoElt(stripBitcast(Elts[i]));
4016 if (Subtarget->useRealTrue16Insts()) {
4021 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
DL, MVT::i16),
4024 emitRegSequence(*
CurDAG, AMDGPU::VGPR_32RegClassID, MVT::i32,
4025 {Elts[i],
Undef}, {AMDGPU::lo16, AMDGPU::hi16},
DL);
4026 Elts[i + 1] = emitRegSequence(*
CurDAG, AMDGPU::VGPR_32RegClassID,
4027 MVT::i32, {Elts[i + 1],
Undef},
4028 {AMDGPU::lo16, AMDGPU::hi16},
DL);
4030 SDValue PackLoLo =
CurDAG->getTargetConstant(0x05040100,
DL, MVT::i32);
4032 CurDAG->getMachineNode(AMDGPU::V_PERM_B32_e64,
DL, MVT::i32,
4033 {Elts[i + 1], Elts[i], PackLoLo});
4037 return buildRegSequence32(PackedElts,
DL);
4043 unsigned ElementSize)
const {
4044 if (ElementSize == 16)
4045 return buildRegSequence16(Elts,
DL);
4046 if (ElementSize == 32)
4047 return buildRegSequence32(Elts,
DL);
4051void AMDGPUDAGToDAGISel::selectWMMAModsNegAbs(
unsigned ModOpcode,
4055 unsigned ElementSize)
const {
4060 for (
auto El : Elts) {
4063 NegAbsElts.
push_back(El->getOperand(0));
4065 if (Elts.size() != NegAbsElts.
size()) {
4067 Src =
SDValue(buildRegSequence(Elts,
DL, ElementSize), 0);
4071 Src =
SDValue(buildRegSequence(NegAbsElts,
DL, ElementSize), 0);
4077 Src =
SDValue(buildRegSequence(Elts,
DL, ElementSize), 0);
4085 std::function<
bool(
SDValue)> ModifierCheck) {
4089 for (
unsigned i = 0; i < F16Pair->getNumOperands(); ++i) {
4090 SDValue ElF16 = stripBitcast(F16Pair->getOperand(i));
4091 if (!ModifierCheck(ElF16))
4098bool AMDGPUDAGToDAGISel::SelectWMMAModsF16Neg(
SDValue In,
SDValue &Src,
4116 Src =
SDValue(buildRegSequence16(EltsF16, SDLoc(In)), 0);
4135 Src =
SDValue(buildRegSequence32(EltsV2F16, SDLoc(In)), 0);
4141 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4145bool AMDGPUDAGToDAGISel::SelectWMMAModsF16NegAbs(
SDValue In,
SDValue &Src,
4156 if (EltsF16.
empty())
4166 selectWMMAModsNegAbs(ModOpcode, Mods, EltsF16, Src, SDLoc(In), 16);
4176 if (EltsV2F16.
empty())
4185 selectWMMAModsNegAbs(ModOpcode, Mods, EltsV2F16, Src, SDLoc(In), 32);
4188 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4192bool AMDGPUDAGToDAGISel::SelectWMMAModsF32NegAbs(
SDValue In,
SDValue &Src,
4202 unsigned ModOpcode =
4213 selectWMMAModsNegAbs(ModOpcode, Mods, EltsF32, Src, SDLoc(In), 32);
4216 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4220bool AMDGPUDAGToDAGISel::SelectWMMAVISrc(
SDValue In,
SDValue &Src)
const {
4222 BitVector UndefElements;
4224 if (isInlineImmediate(
Splat.getNode())) {
4226 unsigned Imm =
C->getAPIntValue().getSExtValue();
4227 Src =
CurDAG->getTargetConstant(
Imm, SDLoc(In), MVT::i32);
4231 unsigned Imm =
C->getValueAPF().bitcastToAPInt().getSExtValue();
4232 Src =
CurDAG->getTargetConstant(
Imm, SDLoc(In), MVT::i32);
4240 SDValue SplatSrc32 = stripBitcast(In);
4242 if (
SDValue Splat32 = SplatSrc32BV->getSplatValue()) {
4243 SDValue SplatSrc16 = stripBitcast(Splat32);
4246 const SIInstrInfo *
TII = Subtarget->getInstrInfo();
4247 std::optional<APInt> RawValue;
4249 RawValue =
C->getValueAPF().bitcastToAPInt();
4251 RawValue =
C->getAPIntValue();
4253 if (RawValue.has_value()) {
4254 EVT VT =
In.getValueType().getScalarType();
4260 if (
TII->isInlineConstant(FloatVal)) {
4261 Src =
CurDAG->getTargetConstant(RawValue.value(), SDLoc(In),
4266 if (
TII->isInlineConstant(RawValue.value())) {
4267 Src =
CurDAG->getTargetConstant(RawValue.value(), SDLoc(In),
4280 if (
CurDAG->isConstantIntBuildVectorOrConstantInt(SplatSrc32)) {
4285 int64_t LoImm = Lo32->getAPIntValue().getSExtValue();
4286 int64_t HiImm = Hi32->getAPIntValue().getSExtValue();
4287 int64_t Imm64I = (HiImm << 32) + LoImm;
4289 if (!isInlineImmediate(APInt(64, Imm64I)))
4292 }
else if (Imm64I != Imm64)
4296 Src =
CurDAG->getTargetConstant(Imm64, SDLoc(In), MVT::i64);
4303bool AMDGPUDAGToDAGISel::SelectSWMMACIndex8(
SDValue In,
SDValue &Src,
4309 const llvm::SDValue &ShiftSrc =
In.getOperand(0);
4318 IndexKey =
CurDAG->getTargetConstant(
Key, SDLoc(In), MVT::i32);
4322bool AMDGPUDAGToDAGISel::SelectSWMMACIndex16(
SDValue In,
SDValue &Src,
4328 const llvm::SDValue &ShiftSrc =
In.getOperand(0);
4337 IndexKey =
CurDAG->getTargetConstant(
Key, SDLoc(In), MVT::i32);
4341bool AMDGPUDAGToDAGISel::SelectSWMMACIndex32(
SDValue In,
SDValue &Src,
4349 const SDValue &ExtendSrc =
In.getOperand(0);
4353 const SDValue &CastSrc =
In.getOperand(0);
4357 if (
Zero &&
Zero->getZExtValue() == 0)
4368 Src = ExtractVecEltSrc;
4372 IndexKey =
CurDAG->getTargetConstant(
Key, SDLoc(In), MVT::i32);
4376bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(
SDValue In,
SDValue &Src,
4380 if (!Subtarget->useRealTrue16Insts() &&
In.getValueSizeInBits() == 16 &&
4383 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4387bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(
SDValue In,
SDValue &Src,
4390 if (!SelectVOP3ModsImpl(In, Src, Mods,
true,
4394 if (!Subtarget->useRealTrue16Insts() &&
In.getValueSizeInBits() == 16 &&
4398 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4411 Op =
Op.getOperand(0);
4413 IsExtractHigh =
false;
4416 if (!Low16 || !Low16->isZero())
4418 Op = stripBitcast(
Op.getOperand(1));
4419 if (
Op.getValueType() != MVT::bf16)
4424 if (
Op.getValueType() != MVT::i32)
4429 if (Mask->getZExtValue() == 0xffff0000) {
4430 IsExtractHigh =
true;
4431 return Op.getOperand(0);
4440 return Op.getOperand(0);
4449bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(
SDValue In,
SDValue &Src,
4453 SelectVOP3ModsImpl(In, Src, Mods);
4455 bool IsExtractHigh =
false;
4457 Src.getOperand(0).getValueType() == VT) {
4458 Src = Src.getOperand(0);
4459 }
else if (VT == MVT::bf16) {
4467 if (Src.getValueType() != VT &&
4468 (VT != MVT::bf16 || Src.getValueType() != MVT::i32))
4471 Src = stripBitcast(Src);
4477 SelectVOP3ModsImpl(Src, Src, ModsTmp);
4492 if (Src.getValueSizeInBits() == 16) {
4501 Src.getOperand(0).getValueType() == MVT::i32) {
4502 Src = Src.getOperand(0);
4506 if (Subtarget->useRealTrue16Insts())
4508 Src = createVOP3PSrc32FromLo16(Src, In,
CurDAG, Subtarget);
4509 }
else if (IsExtractHigh)
4515bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsExt(
SDValue In,
SDValue &Src,
4518 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::f16))
4520 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4524bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(
SDValue In,
SDValue &Src,
4527 SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::f16);
4528 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4532bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsExtNeg(
SDValue In,
SDValue &Src,
4535 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::f16))
4542bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsNeg(
SDValue In,
SDValue &Src,
4545 SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::f16);
4551bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16ModsExt(
SDValue In,
SDValue &Src,
4554 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::bf16))
4556 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4560bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16Mods(
SDValue In,
SDValue &Src,
4563 SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::bf16);
4564 SrcMods =
CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
4568bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16ModsExtNeg(
4571 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::bf16))
4578bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16ModsNeg(
SDValue In,
SDValue &Src,
4581 SelectVOP3PMadMixModsImpl(In, Src, Mods, MVT::bf16);
4591 unsigned NumOpcodes = 0;
4604 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4607 if (
C->isAllOnes()) {
4617 for (
unsigned I = 0;
I < Src.size(); ++
I) {
4631 if (Src.size() == 3) {
4637 if (
C->isAllOnes()) {
4639 for (
unsigned I = 0;
I < Src.size(); ++
I) {
4640 if (Src[
I] ==
LHS) {
4652 Bits = SrcBits[Src.size()];
4657 switch (In.getOpcode()) {
4665 if (!getOperandBits(
LHS, LHSBits) ||
4666 !getOperandBits(
RHS, RHSBits)) {
4667 Src = std::move(Backup);
4668 return std::make_pair(0, 0);
4689 uint8_t LHSBitsOrig = LHSBits;
4690 uint8_t RHSBitsOrig = RHSBits;
4694 NumOpcodes += LHSOp.first;
4695 LHSBits = LHSOp.second;
4702 NumOpcodes += RHSOp.first;
4703 RHSBits = RHSOp.second;
4707 auto dependsOnSlot = [](
uint8_t TT,
int Slot) ->
bool {
4708 if (Slot < 0 || Slot > 2)
4710 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4711 const int Shifts[3] = {4, 2, 1};
4712 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4718 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4725 NegatedInner =
Op.getOperand(0);
4726 for (
int I = 0;
I < (int)S.size();
I++) {
4727 if (Bits == SrcBitsConst[
I] && S[
I] ==
Op)
4729 if (IsNegationOp && Bits == (
uint8_t)~SrcBitsConst[
I] &&
4730 S[
I] == NegatedInner)
4741 for (
int I = 0;
I < (int)SrcAfterLHS.
size() &&
I < 3;
I++) {
4742 if (
I < (
int)Src.size() && Src[
I] != SrcAfterLHS[
I] &&
4743 dependsOnSlot(LHSBits,
I)) {
4752 if (!Stale && !RHSOp.first) {
4753 int Slot = findSlot(RHSBitsOrig,
RHS, SrcBeforeRecurse);
4755 (Slot >= (
int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4761 if (!Stale && !LHSOp.first) {
4762 int Slot = findSlot(LHSBitsOrig,
LHS, SrcBeforeRecurse);
4764 (Slot >= (
int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4769 Src = std::move(SrcBeforeRecurse);
4770 LHSBits = LHSBitsOrig;
4771 RHSBits = RHSBitsOrig;
4777 return std::make_pair(0, 0);
4781 switch (In.getOpcode()) {
4783 TTbl = LHSBits & RHSBits;
4786 TTbl = LHSBits | RHSBits;
4789 TTbl = LHSBits ^ RHSBits;
4795 return std::make_pair(NumOpcodes + 1, TTbl);
4802 unsigned NumOpcodes;
4804 std::tie(NumOpcodes, TTbl) =
BitOp3_Op(In, Src);
4808 if (NumOpcodes < 2 || Src.empty())
4814 if (NumOpcodes < 4 && !In->isDivergent())
4817 if (NumOpcodes == 2 &&
In.getValueType() == MVT::i32) {
4822 (
In.getOperand(0).getOpcode() ==
In.getOpcode() ||
4823 In.getOperand(1).getOpcode() ==
In.getOpcode()))
4837 while (Src.size() < 3)
4838 Src.push_back(Src[0]);
4844 Tbl =
CurDAG->getTargetConstant(TTbl, SDLoc(In), MVT::i32);
4850 return CurDAG->getPOISON(MVT::i32);
4853 return CurDAG->getUNDEF(MVT::i32);
4857 return CurDAG->getConstant(
C->getZExtValue() << 16, SL, MVT::i32);
4862 return CurDAG->getConstant(
4863 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
4873bool AMDGPUDAGToDAGISel::isVGPRImm(
const SDNode *
N)
const {
4874 assert(
CurDAG->getTarget().getTargetTriple().isAMDGCN());
4876 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
4877 const SIInstrInfo *SII = Subtarget->getInstrInfo();
4880 bool AllUsesAcceptSReg =
true;
4882 Limit < 10 && U !=
E; ++U, ++Limit) {
4884 getOperandRegClass(
U->getUser(),
U->getOperandNo());
4892 if (RC != &AMDGPU::VS_32RegClass && RC != &AMDGPU::VS_64RegClass &&
4893 RC != &AMDGPU::VS_64_Align2RegClass) {
4894 AllUsesAcceptSReg =
false;
4895 SDNode *
User =
U->getUser();
4896 if (
User->isMachineOpcode()) {
4897 unsigned Opc =
User->getMachineOpcode();
4898 const MCInstrDesc &
Desc = SII->get(
Opc);
4899 if (
Desc.isCommutable()) {
4900 unsigned OpIdx =
Desc.getNumDefs() +
U->getOperandNo();
4903 unsigned CommutedOpNo = CommuteIdx1 -
Desc.getNumDefs();
4905 getOperandRegClass(
U->getUser(), CommutedOpNo);
4906 if (CommutedRC == &AMDGPU::VS_32RegClass ||
4907 CommutedRC == &AMDGPU::VS_64RegClass ||
4908 CommutedRC == &AMDGPU::VS_64_Align2RegClass)
4909 AllUsesAcceptSReg =
true;
4917 if (!AllUsesAcceptSReg)
4921 return !AllUsesAcceptSReg && (Limit < 10);
4927 bool IsModified =
false;
4933 while (Position !=
CurDAG->allnodes_end()) {
4940 if (ResNode !=
Node) {
4946 CurDAG->RemoveDeadNodes();
4947 }
while (IsModified);
4952 std::vector<SDValue> &OutOps) {
4953 switch (ConstraintID) {
4955 OutOps.push_back(
Op);
4961 MVT VT =
Op.getSimpleValueType();
4964 assert(VRC &&
"Expected valid VGPR class for RF constraint operand");
4968 CurDAG->getMachineNode(AMDGPU::COPY_TO_REGCLASS,
DL, VT, {
Op, RCVal});
4969 OutOps.push_back(
SDValue(Copy, 0));
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr, SDValue &N0, SDValue &N1)
static SDValue SelectSAddrFI(SelectionDAG *CurDAG, SDValue SAddr)
static SDValue matchExtFromI32orI32(SDValue Op, bool IsSigned, const SelectionDAG *DAG)
static MemSDNode * findMemSDNode(SDNode *N)
static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val)
static SDValue combineBallotPattern(SDValue VCMP, bool &Negate)
static SDValue matchBF16FPExtendLike(SDValue Op, bool &IsExtractHigh)
static void checkWMMAElementsModifiersF16(BuildVectorSDNode *BV, std::function< bool(SDValue)> ModifierCheck)
Defines an instruction selector for the AMDGPU target.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static bool isNoUnsignedWrap(MachineInstr *Addr)
static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In, Register &Out)
static std::pair< unsigned, uint8_t > BitOp3_Op(Register R, SmallVectorImpl< Register > &Src, const MachineRegisterInfo &MRI)
static unsigned gwsIntrinToOpcode(unsigned IntrID)
Base class for AMDGPU specific classes of TargetSubtarget.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
const HexagonInstrInfo * TII
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Register const TargetRegisterInfo * TRI
Promote Memory to Register
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Provides R600 specific target descriptions.
Interface definition for R600RegisterInfo.
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
SI DAG Lowering interface definition.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
AMDGPUDAGToDAGISelLegacy(TargetMachine &TM, CodeGenOptLevel OptLevel)
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
StringRef getPassName() const override
getPassName - Return a nice clean name for a pass.
AMDGPU specific code to select AMDGPU machine instructions for SelectionDAG operations.
bool isSDWAOperand(const SDNode *N) const
void SelectBuildVector(SDNode *N, unsigned RegClassID)
void Select(SDNode *N) override
Main hook for targets to transform nodes into machine nodes.
bool widenRegionLoad16(SDNode *N) const
bool runOnMachineFunction(MachineFunction &MF) override
void SelectVectorShuffle(SDNode *N)
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
AMDGPUDAGToDAGISel()=delete
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool matchLoadD16FromBuildVector(SDNode *N) const
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
AMDGPUISelDAGToDAGPass(TargetMachine &TM)
static SDValue stripBitcast(SDValue Val)
static const fltSemantics & BFloat()
static const fltSemantics & IEEEhalf()
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
bool isMaxSignedValue() const
Determine if this is the largest signed value.
int64_t getSExtValue() const
Get sign extended value.
unsigned countr_one() const
Count the number of trailing one bits.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
LLVM Basic Block Representation.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
int64_t getSExtValue() const
Analysis pass which computes a DominatorTree.
Legacy analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
FunctionPass class - This class is used to implement most global optimizations.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void checkSubtargetFeatures(const Function &F) const
Diagnose inconsistent subtarget features before attempting to codegen function F.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Analysis pass that exposes the LoopInfo for a function.
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
The legacy pass manager's analysis pass to compute loop information.
unsigned getID() const
getID() - Return the register class ID number.
LLVM_ABI StringRef getString() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getIntegerVT(unsigned BitWidth)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
A set of analyses that are preserved following a run of a transformation pass.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool isAnyAdd() const
Returns true if the node type is ADD or PTRADD.
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0, unsigned &SrcOpIdx1) const override
static LLVM_READONLY const TargetRegisterClass * getSGPRClassForBitWidth(unsigned BitWidth)
static unsigned getSubRegFromChannel(uint64_t Channel, uint64_t NumRegs=1)
static bool isSGPRClass(const TargetRegisterClass *RC)
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
SelectionDAGISelLegacy(char &ID, std::unique_ptr< SelectionDAGISel > S)
SelectionDAGISelPass(std::unique_ptr< SelectionDAGISel > Selector)
LLVM_ABI PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
std::unique_ptr< FunctionLoweringInfo > FuncInfo
const TargetLowering * TLI
const TargetInstrInfo * TII
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
SelectionDAGISel(TargetMachine &tm, CodeGenOptLevel OL=CodeGenOptLevel::Default)
virtual bool runOnMachineFunction(MachineFunction &mf)
const TargetLowering * getTargetLowering() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
MachineFunction & getMachineFunction() const
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
ilist< SDNode >::iterator allnodes_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
static const unsigned CommuteAnyOperandIndex
Primary interface to the complete machine description for the target machine.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
std::optional< int64_t > getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, int64_t ByteOffset)
bool isGFX12Plus(const MCSubtargetInfo &STI)
constexpr int64_t getNullPointerValue(unsigned AS)
Get the null pointer value for the given address space.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST)
std::optional< int64_t > getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset, bool IsBuffer, bool HasSOffset)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ PTRADD
PTRADD represents pointer arithmetic semantics, for targets that opt in using shouldPreservePtrArith(...
@ POISON
POISON - A poison node.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ FADD
Simple binary floating point operators.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ CONVERGENCECTRL_GLUE
This does not correspond to any convergence control intrinsic.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isExtOpcode(unsigned Opcode)
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
@ User
could "use" a pointer
This is an optimization pass for GlobalISel generic memory operations.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
LLVM_ABI LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isBoolSGPR(SDValue V)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static bool getConstantValue(SDValue N, uint32_t &Out)
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
FunctionPass * createAMDGPUISelDag(TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a AMDGPU-specific.
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
MCRegisterClass TargetRegisterClass
Implement std::hash so that hash_code can be used in STL containers.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isVector() const
Return true if this is a vector value type.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
APInt getMinValue() const
Return the minimal unsigned value possible given these KnownBits.
static unsigned getSubRegFromChannel(unsigned Channel)
bool hasNoUnsignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.