2151 B.setInstrAndDebugLoc(
MI);
2152 unsigned Opc =
MI.getOpcode();
2155 case AMDGPU::G_CONSTANT:
2156 case AMDGPU::G_IMPLICIT_DEF: {
2158 LLT DstTy =
MRI.getType(DstReg);
2164 if (DstBank == &AMDGPU::VCCRegBank)
2167 if (DefRegs.
empty())
2170 B.setInsertPt(*
MI.getParent(), ++
MI.getIterator());
2173 LLVMContext &Ctx =
B.getMF().getFunction().getContext();
2175 MI.getOperand(0).setReg(NewDstReg);
2176 if (
Opc != AMDGPU::G_IMPLICIT_DEF) {
2177 uint64_t ConstVal =
MI.getOperand(1).getCImm()->getZExtValue();
2178 MI.getOperand(1).setCImm(
2182 MRI.setRegBank(NewDstReg, *DstBank);
2183 B.buildTrunc(DefRegs[0], NewDstReg);
2186 case AMDGPU::G_PHI: {
2188 LLT DstTy =
MRI.getType(DstReg);
2195 if (DstBank == &AMDGPU::VCCRegBank) {
2202 for (
unsigned I = 1, E =
MI.getNumOperands();
I != E;
I += 2) {
2206 if (SrcBank != &AMDGPU::VCCRegBank) {
2211 MRI.setRegBank(Copy.getReg(0), AMDGPU::VCCRegBank);
2212 MI.getOperand(
I).setReg(Copy.getReg(0));
2223 ApplyRegBankMapping ApplyBank(
B, *
this,
MRI, DstBank);
2224 B.setInsertPt(
B.getMBB(),
MI);
2232 case AMDGPU::G_FCMP:
2236 case AMDGPU::G_ICMP:
2237 case AMDGPU::G_UADDO:
2238 case AMDGPU::G_USUBO:
2239 case AMDGPU::G_UADDE:
2240 case AMDGPU::G_SADDE:
2241 case AMDGPU::G_USUBE:
2242 case AMDGPU::G_SSUBE: {
2243 unsigned BoolDstOp =
2244 (
Opc == AMDGPU::G_ICMP ||
Opc == AMDGPU::G_FCMP) ? 0 : 1;
2245 Register DstReg =
MI.getOperand(BoolDstOp).getReg();
2249 if (DstBank != &AMDGPU::SGPRRegBank)
2252 const bool HasCarryIn =
MI.getNumOperands() == 5;
2258 MRI.setRegBank(NewDstReg, AMDGPU::SGPRRegBank);
2259 MI.getOperand(BoolDstOp).setReg(NewDstReg);
2263 MRI.setRegBank(NewSrcReg, AMDGPU::SGPRRegBank);
2264 B.buildZExt(NewSrcReg,
MI.getOperand(4).getReg());
2265 MI.getOperand(4).setReg(NewSrcReg);
2269 B.setInsertPt(*
MBB, std::next(
MI.getIterator()));
2274 if (DefRegs.
empty())
2276 B.buildTrunc(DefRegs[0], NewDstReg);
2279 case AMDGPU::G_SELECT: {
2281 LLT DstTy =
MRI.getType(DstReg);
2284 if (CondRegs.
empty())
2291 if (CondBank == &AMDGPU::SGPRRegBank) {
2294 MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank);
2296 MI.getOperand(1).setReg(NewCondReg);
2297 B.buildZExt(NewCondReg, CondRegs[0]);
2310 if (DefRegs.
empty()) {
2315 if (Src1Regs.
empty())
2321 if (Src2Regs.
empty())
2328 auto Flags =
MI.getFlags();
2329 B.buildSelect(DefRegs[0], CondRegs[0], Src1Regs[0], Src2Regs[0], Flags);
2330 B.buildSelect(DefRegs[1], CondRegs[0], Src1Regs[1], Src2Regs[1], Flags);
2332 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
2333 MI.eraseFromParent();
2336 case AMDGPU::G_BRCOND: {
2337 Register CondReg =
MI.getOperand(0).getReg();
2342 if (CondBank == &AMDGPU::SGPRRegBank) {
2345 MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank);
2347 MI.getOperand(0).setReg(NewCondReg);
2348 B.buildZExt(NewCondReg, CondReg);
2356 case AMDGPU::G_XOR: {
2360 LLT DstTy =
MRI.getType(DstReg);
2366 if (DstBank == &AMDGPU::VCCRegBank)
2370 ApplyRegBankMapping ApplyBank(
B, *
this,
MRI, DstBank);
2379 if (DstTy.
getSizeInBits() == 16 && DstBank == &AMDGPU::SGPRRegBank) {
2383 ApplyRegBankMapping ApplySALU(
B, *
this,
MRI, &AMDGPU::SGPRRegBank);
2388 if (
MI.getOpcode() == AMDGPU::G_XOR &&
2409 if (DefRegs.
empty()) {
2416 (Src0Regs.
empty() || Src0Regs.
size() == 2));
2422 if (Src0Regs.
empty())
2427 if (Src1Regs.
empty())
2434 auto Flags =
MI.getFlags();
2435 B.buildInstr(
Opc, {DefRegs[0]}, {Src0Regs[0], Src1Regs[0]}, Flags);
2436 B.buildInstr(
Opc, {DefRegs[1]}, {Src0Regs[1], Src1Regs[1]}, Flags);
2438 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
2439 MI.eraseFromParent();
2442 case AMDGPU::G_ABS: {
2448 if (SrcBank && SrcBank == &AMDGPU::VGPRRegBank) {
2450 ApplyRegBankMapping Apply(
B, *
this,
MRI, &AMDGPU::VGPRRegBank);
2463 case AMDGPU::G_LSHR:
2464 case AMDGPU::G_ASHR:
2465 case AMDGPU::G_SMIN:
2466 case AMDGPU::G_SMAX:
2467 case AMDGPU::G_UMIN:
2468 case AMDGPU::G_UMAX: {
2470 LLT DstTy =
MRI.getType(DstReg);
2475 if (!
Subtarget.useVMulU64Inst() &&
Opc == AMDGPU::G_MUL &&
2488 if (DstBank == &AMDGPU::VGPRRegBank)
2494 ApplyRegBankMapping ApplySALU(
B, *
this,
MRI, &AMDGPU::SGPRRegBank);
2499 std::tie(WideSrcLo, WideSrcHi) =
2501 auto Lo =
B.buildInstr(AMDGPU::G_ABS, {
S32}, {WideSrcLo});
2502 auto Hi =
B.buildInstr(AMDGPU::G_ABS, {
S32}, {WideSrcHi});
2503 B.buildBuildVectorTrunc(DstReg, {
Lo.getReg(0),
Hi.getReg(0)});
2504 MI.eraseFromParent();
2513 std::tie(WideSrc0Lo, WideSrc0Hi)
2515 std::tie(WideSrc1Lo, WideSrc1Hi)
2517 auto Lo =
B.buildInstr(
MI.getOpcode(), {S32}, {WideSrc0Lo, WideSrc1Lo});
2518 auto Hi =
B.buildInstr(
MI.getOpcode(), {S32}, {WideSrc0Hi, WideSrc1Hi});
2519 B.buildBuildVectorTrunc(DstReg, {
Lo.getReg(0),
Hi.getReg(0)});
2520 MI.eraseFromParent();
2528 if (
Opc == AMDGPU::G_SHL ||
Opc == AMDGPU::G_LSHR ||
2529 Opc == AMDGPU::G_ASHR) {
2530 B.setInsertPt(*
MBB,
MI.getIterator());
2538 case AMDGPU::G_AMDGPU_S_MUL_I64_I32:
2539 case AMDGPU::G_AMDGPU_S_MUL_U64_U32: {
2553 Register SrcReg0 =
MI.getOperand(1).getReg();
2554 Register SrcReg1 =
MI.getOperand(2).getReg();
2557 assert(
MRI.getType(DstReg) ==
S64 &&
"This is a special case for s_mul_u64 "
2558 "that handles only 64-bit operands.");
2564 if (DstBank == &AMDGPU::SGPRRegBank) {
2565 MI.setDesc(
TII->get(AMDGPU::S_MUL_U64));
2566 MRI.setRegClass(DstReg, &AMDGPU::SGPR_64RegClass);
2567 MRI.setRegClass(SrcReg0, &AMDGPU::SGPR_64RegClass);
2568 MRI.setRegClass(SrcReg1, &AMDGPU::SGPR_64RegClass);
2574 assert(
MRI.getRegBankOrNull(DstReg) == &AMDGPU::VGPRRegBank &&
2575 "The destination operand should be in vector registers.");
2578 Register Op0L =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2579 MRI.setRegClass(Op0L, &AMDGPU::VGPR_32RegClass);
2581 B.buildTrunc(Op0L, SrcReg0);
2584 Register Op1L =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2585 MRI.setRegClass(Op1L, &AMDGPU::VGPR_32RegClass);
2587 B.buildTrunc(Op1L, SrcReg1);
2589 unsigned NewOpc =
Opc == AMDGPU::G_AMDGPU_S_MUL_U64_U32
2590 ? AMDGPU::G_AMDGPU_MAD_U64_U32
2591 : AMDGPU::G_AMDGPU_MAD_I64_I32;
2595 MRI.setRegClass(Zero64, &AMDGPU::VReg_64RegClass);
2596 Register CarryOut =
MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
2597 MRI.setRegClass(CarryOut, &AMDGPU::VReg_64RegClass);
2598 B.buildInstr(NewOpc, {DstReg, CarryOut}, {Op0L, Op1L, Zero64});
2599 MI.eraseFromParent();
2602 case AMDGPU::G_SEXT_INREG: {
2604 if (SrcRegs.
empty())
2608 ApplyRegBankMapping O(
B, *
this,
MRI, &AMDGPU::VGPRRegBank);
2615 int Amt =
MI.getOperand(2).getImm();
2621 B.buildFreeze(DstRegs[0], SrcRegs[0]);
2623 auto Freeze =
B.buildFreeze(
S32, SrcRegs[0]);
2625 B.buildSExtInReg(DstRegs[0], Freeze, Amt);
2628 B.buildAShr(DstRegs[1], DstRegs[0],
B.buildConstant(
S32, 31));
2632 B.buildCopy(DstRegs[0], SrcRegs[0]);
2633 B.buildSExtInReg(DstRegs[1], DstRegs[0], Amt - 32);
2637 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank);
2638 MI.eraseFromParent();
2641 case AMDGPU::G_CTPOP:
2642 case AMDGPU::G_BITREVERSE: {
2645 if (DstBank == &AMDGPU::SGPRRegBank)
2650 LLT Ty =
MRI.getType(SrcReg);
2654 ApplyRegBankMapping ApplyVALU(
B, *
this,
MRI, &AMDGPU::VGPRRegBank);
2663 case AMDGPU::G_AMDGPU_FFBH_U32:
2664 case AMDGPU::G_AMDGPU_FFBL_B32:
2665 case AMDGPU::G_CTLZ_ZERO_POISON:
2666 case AMDGPU::G_CTTZ_ZERO_POISON: {
2669 if (DstBank == &AMDGPU::SGPRRegBank)
2674 LLT Ty =
MRI.getType(SrcReg);
2684 ApplyRegBankMapping ApplyVALU(
B, *
this,
MRI, &AMDGPU::VGPRRegBank);
2686 unsigned NewOpc =
Opc == AMDGPU::G_CTLZ_ZERO_POISON
2687 ? (
unsigned)AMDGPU::G_AMDGPU_FFBH_U32
2688 :
Opc == AMDGPU::G_CTTZ_ZERO_POISON
2689 ? (
unsigned)AMDGPU::G_AMDGPU_FFBL_B32
2691 unsigned Idx = NewOpc == AMDGPU::G_AMDGPU_FFBH_U32;
2692 auto X =
B.buildInstr(NewOpc, {
S32}, {SrcRegs[Idx]});
2693 auto Y =
B.buildInstr(NewOpc, {
S32}, {SrcRegs[Idx ^ 1]});
2695 Opc == AMDGPU::G_CTLZ_ZERO_POISON ||
Opc == AMDGPU::G_CTTZ_ZERO_POISON
2697 : AMDGPU::G_UADDSAT;
2698 Y =
B.buildInstr(AddOpc, {
S32}, {
Y,
B.buildConstant(
S32, 32)});
2700 B.buildUMin(DstReg,
X,
Y);
2701 MI.eraseFromParent();
2704 case AMDGPU::G_SEXT:
2705 case AMDGPU::G_ZEXT:
2706 case AMDGPU::G_ANYEXT: {
2708 LLT SrcTy =
MRI.getType(SrcReg);
2709 const bool Signed =
Opc == AMDGPU::G_SEXT;
2717 LLT DstTy =
MRI.getType(DstReg);
2719 SrcBank != &AMDGPU::SGPRRegBank &&
2720 SrcBank != &AMDGPU::VCCRegBank &&
2724 SrcTy.getSizeInBits() <= 32) {
2730 B.buildSExtOrTrunc(DefRegs[0], SrcReg);
2731 }
else if (
Opc == AMDGPU::G_ZEXT) {
2732 B.buildZExtOrTrunc(DefRegs[0], SrcReg);
2734 B.buildAnyExtOrTrunc(DefRegs[0], SrcReg);
2738 MRI.setRegBank(DstReg, *SrcBank);
2739 MI.eraseFromParent();
2749 if (SrcBank == &AMDGPU::VCCRegBank) {
2756 const bool UseSel64 = DstSize > 32 &&
2757 SrcBank->
getID() == AMDGPU::SGPRRegBankID;
2761 auto True =
B.buildConstant(SelType,
Signed ? -1 : 1);
2762 auto False =
B.buildConstant(SelType, 0);
2764 MRI.setRegBank(
True.getReg(0), *DstBank);
2765 MRI.setRegBank(
False.getReg(0), *DstBank);
2766 MRI.setRegBank(DstReg, *DstBank);
2769 B.buildSelect(DefRegs[0], SrcReg,
True,
False);
2771 }
else if (DstSize < 32) {
2772 auto Sel =
B.buildSelect(SelType, SrcReg,
True,
False);
2773 MRI.setRegBank(Sel.getReg(0), *DstBank);
2774 B.buildTrunc(DstReg, Sel);
2779 MI.eraseFromParent();
2785 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
2794 LLT DstTy =
MRI.getType(DstReg);
2795 LLT SrcTy =
MRI.getType(SrcReg);
2797 if (foldExtractEltToCmpSelect(
B,
MI, OpdMapper))
2809 unsigned ConstOffset;
2810 std::tie(BaseIdxReg, ConstOffset) =
2817 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2819 ConstOffset < SrcTy.getNumElements();
2822 if (ShouldMoveIndexIntoLoop)
2823 MI.getOperand(2).setReg(BaseIdxReg);
2829 const bool NeedCopyToVGPR = DstBank == &AMDGPU::VGPRRegBank &&
2830 SrcBank == &AMDGPU::SGPRRegBank;
2831 if (DstRegs.
empty()) {
2836 if (NeedCopyToVGPR) {
2838 Register TmpReg =
MRI.createGenericVirtualRegister(DstTy);
2839 MRI.setRegBank(TmpReg, AMDGPU::SGPRRegBank);
2840 MI.getOperand(0).setReg(TmpReg);
2841 B.setInsertPt(*
MI.getParent(), ++
MI.getIterator());
2848 if (ShouldMoveIndexIntoLoop)
2858 auto CastSrc =
B.buildBitcast(Vec32, SrcReg);
2859 auto One =
B.buildConstant(
S32, 1);
2870 auto IdxLo =
B.buildShl(
S32, BaseIdxReg, One);
2871 auto IdxHi =
B.buildAdd(
S32, IdxLo, One);
2873 auto Extract0 =
B.buildExtractVectorElement(DstRegs[0], CastSrc, IdxLo);
2874 auto Extract1 =
B.buildExtractVectorElement(DstRegs[1], CastSrc, IdxHi);
2876 MRI.setRegBank(DstReg, *DstBank);
2877 MRI.setRegBank(CastSrc.getReg(0), *SrcBank);
2878 MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank);
2879 MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank);
2880 MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank);
2884 MI.eraseFromParent();
2890 B.setInstr(*Span.
begin());
2891 MI.eraseFromParent();
2895 if (NeedCopyToVGPR) {
2899 MRI.setRegBank(TmpReg0, AMDGPU::SGPRRegBank);
2900 MRI.setRegBank(TmpReg1, AMDGPU::SGPRRegBank);
2902 Extract0->getOperand(0).setReg(TmpReg0);
2903 Extract1->getOperand(0).setReg(TmpReg1);
2911 if (ShouldMoveIndexIntoLoop)
2916 case AMDGPU::G_INSERT_VECTOR_ELT: {
2920 LLT VecTy =
MRI.getType(DstReg);
2926 MRI.setType(
MI.getOperand(1).getReg(), VecTy);
2928 if (foldInsertEltToCmpSelect(
B,
MI, OpdMapper))
2936 LLT InsTy =
MRI.getType(InsReg);
2940 unsigned ConstOffset;
2941 std::tie(BaseIdxReg, ConstOffset) =
2948 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2953 if (ShouldMoveIndexIntoLoop)
2954 MI.getOperand(3).setReg(BaseIdxReg);
2957 if (InsRegs.
empty()) {
2961 if (ShouldMoveIndexIntoLoop) {
2973 auto CastSrc =
B.buildBitcast(Vec32, SrcReg);
2974 auto One =
B.buildConstant(
S32, 1);
2983 auto IdxLo =
B.buildShl(
S32, BaseIdxReg, One);
2984 auto IdxHi =
B.buildAdd(
S32, IdxLo, One);
2986 auto InsLo =
B.buildInsertVectorElement(Vec32, CastSrc, InsRegs[0], IdxLo);
2987 auto InsHi =
B.buildInsertVectorElement(Vec32, InsLo, InsRegs[1], IdxHi);
2996 MRI.setRegBank(InsReg, *InsSrcBank);
2997 MRI.setRegBank(CastSrc.getReg(0), *SrcBank);
2998 MRI.setRegBank(InsLo.getReg(0), *DstBank);
2999 MRI.setRegBank(InsHi.getReg(0), *DstBank);
3000 MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank);
3001 MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank);
3002 MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank);
3007 B.setInsertPt(
B.getMBB(),
MI);
3008 B.buildBitcast(DstReg, InsHi);
3009 MI.eraseFromParent();
3013 B.setInstr(*Span.
begin());
3014 MI.eraseFromParent();
3025 B.buildBitcast(DstReg, InsHi);
3028 if (ShouldMoveIndexIntoLoop)
3033 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
3034 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
3035 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
3036 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
3037 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
3038 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE:
3039 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE:
3040 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE:
3041 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE:
3042 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE:
3043 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
3044 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
3045 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
3046 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
3047 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
3048 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
3049 case AMDGPU::G_AMDGPU_BUFFER_STORE:
3050 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
3051 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
3052 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
3053 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16:
3054 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
3055 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: {
3060 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
3061 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
3062 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
3063 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
3064 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
3065 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
3066 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
3067 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
3068 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
3069 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
3070 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
3071 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC:
3072 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32:
3073 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32:
3074 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD:
3075 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN:
3076 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: {
3081 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
3086 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
3087 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
3088 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
3089 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
3090 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: {
3094 case AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH:
3098 case AMDGPU::G_INTRINSIC:
3099 case AMDGPU::G_INTRINSIC_CONVERGENT: {
3101 case Intrinsic::amdgcn_readlane: {
3112 case Intrinsic::amdgcn_writelane: {
3122 case Intrinsic::amdgcn_interp_p1:
3123 case Intrinsic::amdgcn_interp_p2:
3124 case Intrinsic::amdgcn_interp_mov:
3125 case Intrinsic::amdgcn_interp_p1_f16:
3126 case Intrinsic::amdgcn_interp_p2_f16:
3127 case Intrinsic::amdgcn_lds_param_load: {
3135 case Intrinsic::amdgcn_interp_inreg_p10:
3136 case Intrinsic::amdgcn_interp_inreg_p2:
3137 case Intrinsic::amdgcn_interp_inreg_p10_f16:
3138 case Intrinsic::amdgcn_interp_inreg_p2_f16:
3139 case Intrinsic::amdgcn_interp_p10_rtz_f16:
3140 case Intrinsic::amdgcn_interp_p2_rtz_f16:
3141 case Intrinsic::amdgcn_permlane16_swap:
3142 case Intrinsic::amdgcn_permlane32_swap:
3145 case Intrinsic::amdgcn_permlane16:
3146 case Intrinsic::amdgcn_permlanex16: {
3154 case Intrinsic::amdgcn_permlane_bcast:
3155 case Intrinsic::amdgcn_permlane_up:
3156 case Intrinsic::amdgcn_permlane_down:
3157 case Intrinsic::amdgcn_permlane_xor:
3162 case Intrinsic::amdgcn_permlane_idx_gen: {
3166 case Intrinsic::amdgcn_sbfe:
3169 case Intrinsic::amdgcn_ubfe:
3172 case Intrinsic::amdgcn_inverse_ballot:
3173 case Intrinsic::amdgcn_s_bitreplicate:
3174 case Intrinsic::amdgcn_s_quadmask:
3175 case Intrinsic::amdgcn_s_wqm:
3179 case Intrinsic::amdgcn_ballot:
3185 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
3186 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
3187 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
3188 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
3189 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
3199 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
3200 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
3201 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY: {
3203 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY ||
3204 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY;
3205 unsigned NumMods = IsDualOrBVH8 ? 0 : 1;
3206 unsigned LastRegOpIdx =
MI.getNumExplicitOperands() - 1 - NumMods;
3211 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
3212 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
3215 case Intrinsic::amdgcn_ds_ordered_add:
3216 case Intrinsic::amdgcn_ds_ordered_swap: {
3223 case Intrinsic::amdgcn_ds_gws_init:
3224 case Intrinsic::amdgcn_ds_gws_barrier:
3225 case Intrinsic::amdgcn_ds_gws_sema_br: {
3231 case Intrinsic::amdgcn_ds_gws_sema_v:
3232 case Intrinsic::amdgcn_ds_gws_sema_p:
3233 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
3238 case Intrinsic::amdgcn_ds_append:
3239 case Intrinsic::amdgcn_ds_consume: {
3243 case Intrinsic::amdgcn_s_alloc_vgpr:
3246 case Intrinsic::amdgcn_s_sendmsg:
3247 case Intrinsic::amdgcn_s_sendmsghalt: {
3252 case Intrinsic::amdgcn_s_setreg: {
3256 case Intrinsic::amdgcn_s_ttracedata:
3259 case Intrinsic::amdgcn_raw_buffer_load_lds:
3260 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
3261 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
3262 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds: {
3269 case Intrinsic::amdgcn_struct_buffer_load_lds:
3270 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
3271 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
3272 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
3279 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
3280 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
3281 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
3282 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
3287 case Intrinsic::amdgcn_load_to_lds:
3288 case Intrinsic::amdgcn_load_async_to_lds:
3289 case Intrinsic::amdgcn_global_load_lds:
3290 case Intrinsic::amdgcn_global_load_async_lds: {
3295 case Intrinsic::amdgcn_lds_direct_load: {
3301 case Intrinsic::amdgcn_exp_row:
3305 case Intrinsic::amdgcn_cluster_load_b32:
3306 case Intrinsic::amdgcn_cluster_load_b64:
3307 case Intrinsic::amdgcn_cluster_load_b128: {
3312 case Intrinsic::amdgcn_s_sleep_var:
3316 case Intrinsic::amdgcn_s_barrier_join:
3317 case Intrinsic::amdgcn_s_wakeup_barrier:
3320 case Intrinsic::amdgcn_s_barrier_init:
3321 case Intrinsic::amdgcn_s_barrier_signal_var:
3325 case Intrinsic::amdgcn_s_get_barrier_state:
3326 case Intrinsic::amdgcn_s_get_named_barrier_state: {
3330 case Intrinsic::amdgcn_s_prefetch_data:
3331 case Intrinsic::amdgcn_s_prefetch_inst: {
3333 unsigned AS =
MRI.getType(PtrReg).getAddressSpace();
3338 MI.eraseFromParent();
3341 case Intrinsic::amdgcn_tensor_load_to_lds:
3342 case Intrinsic::amdgcn_tensor_store_from_lds: {
3356 if (RSrcIntrin->IsImage) {
3367 case AMDGPU::G_SI_CALL: {
3378 unsigned FrameSetupOpcode = AMDGPU::ADJCALLSTACKUP;
3379 unsigned FrameDestroyOpcode = AMDGPU::ADJCALLSTACKDOWN;
3385 unsigned NonCopyInstrsLen = 0;
3391 while (Start->getOpcode() != FrameSetupOpcode) {
3393 bool IsCopy =
false;
3394 if (Start->getOpcode() == AMDGPU::COPY) {
3395 auto &Dst = Start->getOperand(0);
3398 if (Reg.isPhysical() &&
MI.readsRegister(Reg,
TRI)) {
3403 auto &Src = Start->getOperand(1);
3406 IsCopy = Info->getScratchRSrcReg() == Reg;
3414 NonCopyInstrsLen = NonCopyInstrs.
size();
3419 NonCopyInstrs.
resize(NonCopyInstrsLen);
3421 for (
auto *NonCopy :
reverse(NonCopyInstrs)) {
3422 MBB->splice(LastCopy,
MBB, NonCopy->getIterator());
3427 NonCopyInstrs.
clear();
3428 NonCopyInstrsLen = 0;
3431 while (
End->getOpcode() != FrameDestroyOpcode) {
3433 bool IsCopy =
false;
3434 if (
End->getOpcode() == AMDGPU::COPY) {
3435 auto &Src =
End->getOperand(1);
3438 IsCopy = Reg.isPhysical() &&
MI.modifiesRegister(Reg,
TRI);
3444 NonCopyInstrsLen = NonCopyInstrs.
size();
3449 NonCopyInstrs.
resize(NonCopyInstrsLen);
3453 for (
auto *NonCopy :
reverse(NonCopyInstrs)) {
3454 MBB->splice(LastCopy,
MBB, NonCopy->getIterator());
3458 B.setInsertPt(
B.getMBB(), Start);
3462 case AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR:
3463 case AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR:
3464 case AMDGPU::G_LOAD:
3465 case AMDGPU::G_ZEXTLOAD:
3466 case AMDGPU::G_SEXTLOAD: {
3471 case AMDGPU::G_DYN_STACKALLOC:
3474 case AMDGPU::G_STACKRESTORE: {
3479 case AMDGPU::G_SBFX:
3482 case AMDGPU::G_UBFX:
3485 case AMDGPU::G_AMDGPU_MAD_U64_U32:
3486 case AMDGPU::G_AMDGPU_MAD_I64_I32:
3489 case AMDGPU::G_PREFETCH: {
3491 MI.eraseFromParent();
3496 if (PtrBank == AMDGPU::VGPRRegBankID &&
3497 (!
Subtarget.hasVmemPrefInsts() || !
MI.getOperand(3).getImm())) {
3499 MI.eraseFromParent();
3502 unsigned AS =
MRI.getType(PtrReg).getAddressSpace();
3507 !
MI.getOperand(3).getImm() ))) {
3508 MI.eraseFromParent();
3805 if (
MI.isCopy() ||
MI.getOpcode() == AMDGPU::G_FREEZE) {
3820 DstBank = &AMDGPU::VCCRegBank;
3823 DstBank = &AMDGPU::VCCRegBank;
3834 if (
MI.getOpcode() != AMDGPU::G_FREEZE &&
3839 unsigned OpdsMappingSize =
MI.isCopy() ? 1 : 2;
3841 OpdsMapping[0] = &ValMap;
3842 if (
MI.getOpcode() == AMDGPU::G_FREEZE)
3843 OpdsMapping[1] = &ValMap;
3850 if (
MI.isRegSequence()) {
3853 unsigned BankID = AMDGPU::SGPRRegBankID;
3855 for (
unsigned I = 1, E =
MI.getNumOperands();
I != E;
I += 2) {
3859 if (OpBank != AMDGPU::SGPRRegBankID) {
3860 BankID = AMDGPU::VGPRRegBankID;
3864 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
3877 unsigned ResultBank = AMDGPU::InvalidRegBankID;
3882 ResultBank = DstBank->
getID();
3884 for (
unsigned I = 0;
I <
PHI->getNumIncomingValues(); ++
I) {
3889 if (!Bank || Bank->
getID() == AMDGPU::VGPRRegBankID) {
3890 ResultBank = AMDGPU::VGPRRegBankID;
3895 unsigned OpBank = Bank->
getID();
3899 assert(ResultBank != AMDGPU::InvalidRegBankID);
3901 unsigned Size =
MRI.getType(DstReg).getSizeInBits();
3916 switch (
MI.getOpcode()) {
3923 case AMDGPU::G_MUL: {
3924 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
3929 unsigned TargetBankID = AMDGPU::InvalidRegBankID;
3930 unsigned BankLHS = AMDGPU::InvalidRegBankID;
3931 unsigned BankRHS = AMDGPU::InvalidRegBankID;
3933 TargetBankID = DstBank->
getID();
3934 if (DstBank == &AMDGPU::VCCRegBank) {
3935 TargetBankID = AMDGPU::VCCRegBankID;
3936 BankLHS = AMDGPU::VCCRegBankID;
3937 BankRHS = AMDGPU::VCCRegBankID;
3940 AMDGPU::SGPRRegBankID);
3942 AMDGPU::SGPRRegBankID);
3946 AMDGPU::VCCRegBankID);
3948 AMDGPU::VCCRegBankID);
3951 if (BankLHS == AMDGPU::VGPRRegBankID || BankRHS == AMDGPU::VGPRRegBankID) {
3952 TargetBankID = AMDGPU::VGPRRegBankID;
3953 }
else if (BankLHS == AMDGPU::VCCRegBankID || BankRHS == AMDGPU::VCCRegBankID) {
3954 TargetBankID = AMDGPU::VCCRegBankID;
3955 BankLHS = AMDGPU::VCCRegBankID;
3956 BankRHS = AMDGPU::VCCRegBankID;
3957 }
else if (BankLHS == AMDGPU::SGPRRegBankID && BankRHS == AMDGPU::SGPRRegBankID) {
3958 TargetBankID = AMDGPU::SGPRRegBankID;
3962 OpdsMapping[0] = AMDGPU::getValueMapping(TargetBankID,
Size);
3963 OpdsMapping[1] = AMDGPU::getValueMapping(BankLHS,
Size);
3964 OpdsMapping[2] = AMDGPU::getValueMapping(BankRHS,
Size);
3971 OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID,
Size);
3972 OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
3974 if (
MI.getOpcode() == AMDGPU::G_MUL &&
Subtarget.useVMulU64Inst())
3975 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
3978 getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID,
Size);
3980 OpdsMapping[1] = AMDGPU::getValueMapping(Bank1,
Size);
3983 OpdsMapping[2] = AMDGPU::getValueMapping(Bank2,
Size);
3991 case AMDGPU::G_PTR_ADD:
3992 case AMDGPU::G_PTRMASK:
3996 case AMDGPU::G_LSHR:
3997 case AMDGPU::G_ASHR:
3998 case AMDGPU::G_UADDO:
3999 case AMDGPU::G_USUBO:
4000 case AMDGPU::G_UADDE:
4001 case AMDGPU::G_SADDE:
4002 case AMDGPU::G_USUBE:
4003 case AMDGPU::G_SSUBE:
4005 case AMDGPU::G_SHUFFLE_VECTOR:
4006 case AMDGPU::G_SBFX:
4007 case AMDGPU::G_UBFX:
4008 case AMDGPU::G_AMDGPU_S_MUL_I64_I32:
4009 case AMDGPU::G_AMDGPU_S_MUL_U64_U32:
4011 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
4012 unsigned Size = Ty.getSizeInBits();
4014 if (
Subtarget.hasAnyPackedU64Ops() && Ty.isVector() &&
Size == 128)
4019 case AMDGPU::G_SMIN:
4020 case AMDGPU::G_SMAX:
4021 case AMDGPU::G_UMIN:
4022 case AMDGPU::G_UMAX:
4025 if (
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits() == 64 &&
4031 case AMDGPU::G_FADD:
4032 case AMDGPU::G_FSUB:
4033 case AMDGPU::G_FMUL:
4035 case AMDGPU::G_FFLOOR:
4036 case AMDGPU::G_FCEIL:
4037 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
4038 case AMDGPU::G_INTRINSIC_TRUNC:
4039 case AMDGPU::G_STRICT_FADD:
4040 case AMDGPU::G_STRICT_FSUB:
4041 case AMDGPU::G_STRICT_FMUL:
4042 case AMDGPU::G_STRICT_FMA: {
4043 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
4044 unsigned Size = Ty.getSizeInBits();
4045 if (
Subtarget.hasSALUFloatInsts() && Ty.isScalar() &&
4050 case AMDGPU::G_FMINNUM:
4051 case AMDGPU::G_FMAXNUM:
4052 case AMDGPU::G_FMINIMUMNUM:
4053 case AMDGPU::G_FMAXIMUMNUM: {
4054 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
4055 unsigned Size = Ty.getSizeInBits();
4066 bool ScalarNumIsSafe =
4068 if (
Subtarget.hasSALUFloatInsts() && Ty.isScalar() &&
4073 case AMDGPU::G_FMINIMUM:
4074 case AMDGPU::G_FMAXIMUM: {
4075 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
4076 unsigned Size = Ty.getSizeInBits();
4077 if (
Subtarget.hasSALUMinimumMaximumInsts() && Ty.isScalar() &&
4082 case AMDGPU::G_FPTOSI:
4083 case AMDGPU::G_FPTOUI:
4084 case AMDGPU::G_FPTOSI_SAT:
4085 case AMDGPU::G_FPTOUI_SAT:
4086 case AMDGPU::G_SITOFP:
4087 case AMDGPU::G_UITOFP: {
4088 unsigned SizeDst =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4089 unsigned SizeSrc =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
4090 if (
Subtarget.hasSALUFloatInsts() && SizeDst == 32 && SizeSrc == 32 &&
4095 case AMDGPU::G_FPTRUNC:
4096 case AMDGPU::G_FPEXT: {
4097 unsigned SizeDst =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4098 unsigned SizeSrc =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
4099 if (
Subtarget.hasSALUFloatInsts() && SizeDst != 64 && SizeSrc != 64 &&
4104 case AMDGPU::G_FSQRT:
4105 case AMDGPU::G_FEXP2:
4106 case AMDGPU::G_FLOG2: {
4107 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4113 case AMDGPU::G_SADDSAT:
4114 case AMDGPU::G_SSUBSAT:
4115 case AMDGPU::G_UADDSAT:
4116 case AMDGPU::G_USUBSAT:
4117 case AMDGPU::G_FMAD:
4118 case AMDGPU::G_FLDEXP:
4119 case AMDGPU::G_FMINNUM_IEEE:
4120 case AMDGPU::G_FMAXNUM_IEEE:
4121 case AMDGPU::G_FCANONICALIZE:
4122 case AMDGPU::G_STRICT_FLDEXP:
4123 case AMDGPU::G_BSWAP:
4124 case AMDGPU::G_FSHR:
4125 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
4126 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
4127 case AMDGPU::G_AMDGPU_RCP_IFLAG:
4128 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
4129 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
4130 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
4131 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
4132 case AMDGPU::G_AMDGPU_CVT_PK_I16_I32:
4133 case AMDGPU::G_AMDGPU_SMED3:
4134 case AMDGPU::G_AMDGPU_FMED3:
4136 case AMDGPU::G_UMULH:
4137 case AMDGPU::G_SMULH: {
4142 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4143 case AMDGPU::G_AMDGPU_MAD_I64_I32: {
4152 bool AllSalu =
true;
4153 bool MulSalu =
true;
4154 for (
unsigned i = 0; i < 5; ++i) {
4157 if (Bank->getID() != AMDGPU::SGPRRegBankID) {
4159 if (i == 2 || i == 3) {
4173 if (!MulSalu ||
Subtarget.hasFullRate64Ops())
4177 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 64);
4178 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4179 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
4180 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
4181 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 64);
4184 case AMDGPU::G_IMPLICIT_DEF: {
4185 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4186 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID,
Size);
4189 case AMDGPU::G_FCONSTANT:
4190 case AMDGPU::G_CONSTANT:
4191 case AMDGPU::G_GLOBAL_VALUE:
4192 case AMDGPU::G_FRAME_INDEX:
4193 case AMDGPU::G_BLOCK_ADDR:
4194 case AMDGPU::G_READSTEADYCOUNTER:
4195 case AMDGPU::G_READCYCLECOUNTER: {
4196 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4197 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID,
Size);
4200 case AMDGPU::G_DYN_STACKALLOC: {
4202 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
4204 OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, 32);
4207 case AMDGPU::G_AMDGPU_WAVE_ADDRESS: {
4212 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
4213 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
4216 case AMDGPU::G_INSERT: {
4218 unsigned DstSize =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
4219 unsigned SrcSize =
TRI->getRegSizeInBits(
MI.getOperand(1).getReg(),
MRI);
4220 unsigned EltSize =
TRI->getRegSizeInBits(
MI.getOperand(2).getReg(),
MRI);
4221 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize);
4222 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize);
4223 OpdsMapping[2] = AMDGPU::getValueMapping(BankID, EltSize);
4224 OpdsMapping[3] =
nullptr;
4227 case AMDGPU::G_EXTRACT: {
4229 unsigned DstSize =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
4230 unsigned SrcSize =
TRI->getRegSizeInBits(
MI.getOperand(1).getReg(),
MRI);
4231 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize);
4232 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize);
4233 OpdsMapping[2] =
nullptr;
4236 case AMDGPU::G_BUILD_VECTOR:
4237 case AMDGPU::G_BUILD_VECTOR_TRUNC: {
4238 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
4241 unsigned SrcSize =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
4244 unsigned DstBankID =
regBankUnion(Src0BankID, Src1BankID);
4246 OpdsMapping[0] = AMDGPU::getValueMapping(DstBankID, DstSize);
4247 OpdsMapping[1] = AMDGPU::getValueMapping(Src0BankID, SrcSize);
4248 OpdsMapping[2] = AMDGPU::getValueMapping(Src1BankID, SrcSize);
4254 case AMDGPU::G_MERGE_VALUES:
4255 case AMDGPU::G_CONCAT_VECTORS: {
4257 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4258 unsigned SrcSize =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
4260 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize);
4262 for (
unsigned i = 1, e =
MI.getNumOperands(); i != e; ++i)
4263 OpdsMapping[i] = AMDGPU::getValueMapping(Bank, SrcSize);
4266 case AMDGPU::G_BITREVERSE:
4267 case AMDGPU::G_BITCAST:
4268 case AMDGPU::G_INTTOPTR:
4269 case AMDGPU::G_PTRTOINT:
4270 case AMDGPU::G_FABS:
4271 case AMDGPU::G_FNEG: {
4272 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4274 OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID,
Size);
4277 case AMDGPU::G_AMDGPU_FFBH_U32:
4278 case AMDGPU::G_AMDGPU_FFBL_B32:
4279 case AMDGPU::G_CTLZ_ZERO_POISON:
4280 case AMDGPU::G_CTTZ_ZERO_POISON: {
4281 unsigned Size =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
4283 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, 32);
4284 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(BankID,
Size);
4287 case AMDGPU::G_CTPOP: {
4288 unsigned Size =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
4290 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, 32);
4295 OpdsMapping[1] = AMDGPU::getValueMapping(BankID,
Size);
4298 case AMDGPU::G_TRUNC: {
4302 unsigned DstSize =
TRI->getRegSizeInBits(Dst,
MRI);
4303 unsigned SrcSize =
TRI->getRegSizeInBits(Src,
MRI);
4304 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize);
4305 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, SrcSize);
4308 case AMDGPU::G_ZEXT:
4309 case AMDGPU::G_SEXT:
4310 case AMDGPU::G_ANYEXT:
4311 case AMDGPU::G_SEXT_INREG: {
4314 unsigned DstSize =
TRI->getRegSizeInBits(Dst,
MRI);
4315 unsigned SrcSize =
TRI->getRegSizeInBits(Src,
MRI);
4320 switch (SrcBank->
getID()) {
4321 case AMDGPU::SGPRRegBankID:
4322 DstBank = AMDGPU::SGPRRegBankID;
4325 DstBank = AMDGPU::VGPRRegBankID;
4331 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(DstBank, DstSize);
4332 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(SrcBank->
getID(),
4336 case AMDGPU::G_IS_FPCLASS: {
4338 unsigned SrcSize =
MRI.getType(SrcReg).getSizeInBits();
4339 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4340 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize);
4341 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4344 case AMDGPU::G_STORE: {
4346 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4351 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
4352 OpdsMapping[0] = ValMapping;
4356 case AMDGPU::G_ICMP:
4357 case AMDGPU::G_FCMP: {
4358 unsigned Size =
MRI.getType(
MI.getOperand(2).getReg()).getSizeInBits();
4363 AMDGPU::SGPRRegBankID);
4367 auto canUseSCCICMP = [&]() {
4370 return Size == 32 ||
4375 auto canUseSCCFCMP = [&]() {
4379 bool isICMP =
MI.getOpcode() == AMDGPU::G_ICMP;
4380 bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID &&
4381 Op2Bank == AMDGPU::SGPRRegBankID &&
4382 Op3Bank == AMDGPU::SGPRRegBankID &&
4383 (isICMP ? canUseSCCICMP() : canUseSCCFCMP());
4385 DstBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
4386 unsigned SrcBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4390 const unsigned ResultSize = 1;
4392 OpdsMapping[0] = AMDGPU::getValueMapping(DstBank, ResultSize);
4393 OpdsMapping[1] =
nullptr;
4394 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank,
Size);
4395 OpdsMapping[3] = AMDGPU::getValueMapping(SrcBank,
Size);
4398 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
4401 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4402 unsigned SrcSize =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
4403 unsigned IdxSize =
MRI.getType(
MI.getOperand(2).getReg()).getSizeInBits();
4405 unsigned OutputBankID =
regBankUnion(SrcBankID, IdxBank);
4407 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(OutputBankID, DstSize);
4408 OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, SrcSize);
4411 OpdsMapping[2] = AMDGPU::getValueMapping(IdxBank, IdxSize);
4414 case AMDGPU::G_INSERT_VECTOR_ELT: {
4416 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4418 unsigned VecSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4419 unsigned InsertSize =
MRI.getType(
MI.getOperand(2).getReg()).getSizeInBits();
4420 unsigned IdxSize =
MRI.getType(
MI.getOperand(3).getReg()).getSizeInBits();
4424 OpdsMapping[0] = AMDGPU::getValueMapping(OutputBankID, VecSize);
4425 OpdsMapping[1] = AMDGPU::getValueMapping(OutputBankID, VecSize);
4429 if (InsertSize == 64 && OutputBankID == AMDGPU::VGPRRegBankID) {
4430 OpdsMapping[2] = AMDGPU::getValueMappingSplit64(InsertEltBankID,
4433 assert(InsertSize == 32 || InsertSize == 64);
4434 OpdsMapping[2] = AMDGPU::getValueMapping(InsertEltBankID, InsertSize);
4438 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBankID, IdxSize);
4441 case AMDGPU::G_UNMERGE_VALUES: {
4446 for (
unsigned i = 0, e =
MI.getNumOperands(); i != e; ++i) {
4447 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(i).getReg(),
MRI);
4448 OpdsMapping[i] = AMDGPU::getValueMapping(Bank,
Size);
4452 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
4453 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
4454 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
4455 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
4456 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
4457 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE:
4458 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE:
4459 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE:
4460 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE:
4461 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE:
4462 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
4463 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
4464 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
4465 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
4466 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
4467 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
4468 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
4469 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16:
4470 case AMDGPU::G_AMDGPU_BUFFER_STORE:
4471 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
4472 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
4473 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
4474 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: {
4493 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
4494 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
4495 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
4496 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
4497 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
4498 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
4499 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
4500 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
4501 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
4502 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
4503 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
4504 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC:
4505 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32:
4506 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32:
4507 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD:
4508 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN:
4509 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: {
4532 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
4558 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
4559 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
4560 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
4561 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
4562 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: {
4570 unsigned RSrcBank = OpdsMapping[1]->BreakDown[0].RegBank->getID();
4571 unsigned OffsetBank = OpdsMapping[2]->BreakDown[0].RegBank->getID();
4572 unsigned ResultBank =
regBankUnion(RSrcBank, OffsetBank);
4574 unsigned Size0 =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4575 OpdsMapping[0] = AMDGPU::getValueMapping(ResultBank, Size0);
4578 case AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH:
4582 case AMDGPU::G_AMDGPU_SPONENTRY: {
4583 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4584 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID,
Size);
4587 case AMDGPU::G_INTRINSIC:
4588 case AMDGPU::G_INTRINSIC_CONVERGENT: {
4592 case Intrinsic::amdgcn_div_fmas: {
4600 case Intrinsic::amdgcn_div_fixup:
4601 case Intrinsic::amdgcn_trig_preop:
4602 case Intrinsic::amdgcn_sin:
4603 case Intrinsic::amdgcn_cos:
4604 case Intrinsic::amdgcn_log_clamp:
4605 case Intrinsic::amdgcn_rcp_legacy:
4606 case Intrinsic::amdgcn_rsq_legacy:
4607 case Intrinsic::amdgcn_rsq_clamp:
4608 case Intrinsic::amdgcn_tanh:
4609 case Intrinsic::amdgcn_fmul_legacy:
4610 case Intrinsic::amdgcn_fma_legacy:
4611 case Intrinsic::amdgcn_frexp_mant:
4612 case Intrinsic::amdgcn_frexp_exp:
4613 case Intrinsic::amdgcn_fract:
4614 case Intrinsic::amdgcn_cvt_pknorm_i16:
4615 case Intrinsic::amdgcn_cvt_pknorm_u16:
4616 case Intrinsic::amdgcn_cvt_pk_i16:
4617 case Intrinsic::amdgcn_cvt_pk_u16:
4618 case Intrinsic::amdgcn_cvt_sr_pk_f16_f32:
4619 case Intrinsic::amdgcn_cvt_sr_pk_bf16_f32:
4620 case Intrinsic::amdgcn_cvt_pk_f16_fp8:
4621 case Intrinsic::amdgcn_cvt_pk_f16_bf8:
4622 case Intrinsic::amdgcn_cvt_pk_fp8_f16:
4623 case Intrinsic::amdgcn_cvt_pk_bf8_f16:
4624 case Intrinsic::amdgcn_cvt_sr_fp8_f16:
4625 case Intrinsic::amdgcn_cvt_sr_bf8_f16:
4626 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
4627 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
4628 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
4629 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
4630 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
4631 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
4632 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
4633 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
4634 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
4635 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
4636 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
4637 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
4638 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
4639 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
4640 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
4641 case Intrinsic::amdgcn_cvt_scale_pk32_f16_fp6:
4642 case Intrinsic::amdgcn_cvt_scale_pk32_bf16_fp6:
4643 case Intrinsic::amdgcn_cvt_scale_pk32_f16_bf6:
4644 case Intrinsic::amdgcn_cvt_scale_pk32_bf16_bf6:
4645 case Intrinsic::amdgcn_cvt_scale_pk32_f32_fp6:
4646 case Intrinsic::amdgcn_cvt_scale_pk32_f32_bf6:
4647 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_bf16:
4648 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_bf16:
4649 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_f16:
4650 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_f16:
4651 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_f32:
4652 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_f32:
4653 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_f32:
4654 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_f16:
4655 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_bf16:
4656 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_f32:
4657 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_f32:
4658 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_f16:
4659 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_f16:
4660 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_bf16:
4661 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_bf16:
4662 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_bf16:
4663 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_bf16:
4664 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_f16:
4665 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_f16:
4666 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_f32:
4667 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_f32:
4668 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_f32:
4669 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_f16:
4670 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_bf16:
4671 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_f32:
4672 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_f32:
4673 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_f16:
4674 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_f16:
4675 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_bf16:
4676 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_bf16:
4677 case Intrinsic::amdgcn_sat_pk4_i4_i8:
4678 case Intrinsic::amdgcn_sat_pk4_u4_u8:
4679 case Intrinsic::amdgcn_fmed3:
4680 case Intrinsic::amdgcn_cubeid:
4681 case Intrinsic::amdgcn_cubema:
4682 case Intrinsic::amdgcn_cubesc:
4683 case Intrinsic::amdgcn_cubetc:
4684 case Intrinsic::amdgcn_sffbh:
4685 case Intrinsic::amdgcn_fmad_ftz:
4686 case Intrinsic::amdgcn_mbcnt_lo:
4687 case Intrinsic::amdgcn_mbcnt_hi:
4688 case Intrinsic::amdgcn_mul_u24:
4689 case Intrinsic::amdgcn_mul_i24:
4690 case Intrinsic::amdgcn_mulhi_u24:
4691 case Intrinsic::amdgcn_mulhi_i24:
4692 case Intrinsic::amdgcn_lerp:
4693 case Intrinsic::amdgcn_sad_u8:
4694 case Intrinsic::amdgcn_msad_u8:
4695 case Intrinsic::amdgcn_sad_hi_u8:
4696 case Intrinsic::amdgcn_sad_u16:
4697 case Intrinsic::amdgcn_qsad_pk_u16_u8:
4698 case Intrinsic::amdgcn_mqsad_pk_u16_u8:
4699 case Intrinsic::amdgcn_mqsad_u32_u8:
4700 case Intrinsic::amdgcn_cvt_pk_u8_f32:
4701 case Intrinsic::amdgcn_alignbyte:
4702 case Intrinsic::amdgcn_perm:
4703 case Intrinsic::amdgcn_prng_b32:
4704 case Intrinsic::amdgcn_exclusive_scan_sum_i32:
4705 case Intrinsic::amdgcn_exclusive_scan_sum_u32:
4706 case Intrinsic::amdgcn_exclusive_scan_xor_b32:
4707 case Intrinsic::amdgcn_exclusive_scan_or_b32:
4708 case Intrinsic::amdgcn_exclusive_scan_and_b32:
4709 case Intrinsic::amdgcn_exclusive_scan_min_i16:
4710 case Intrinsic::amdgcn_exclusive_scan_min_u16:
4711 case Intrinsic::amdgcn_exclusive_scan_min_i32:
4712 case Intrinsic::amdgcn_exclusive_scan_min_u32:
4713 case Intrinsic::amdgcn_exclusive_scan_max_i16:
4714 case Intrinsic::amdgcn_exclusive_scan_max_u16:
4715 case Intrinsic::amdgcn_exclusive_scan_max_i32:
4716 case Intrinsic::amdgcn_exclusive_scan_max_u32:
4717 case Intrinsic::amdgcn_wave_match_b32:
4718 case Intrinsic::amdgcn_fdot2:
4719 case Intrinsic::amdgcn_sdot2:
4720 case Intrinsic::amdgcn_udot2:
4721 case Intrinsic::amdgcn_sdot4:
4722 case Intrinsic::amdgcn_udot4:
4723 case Intrinsic::amdgcn_sdot8:
4724 case Intrinsic::amdgcn_udot8:
4725 case Intrinsic::amdgcn_fdot2_bf16_bf16:
4726 case Intrinsic::amdgcn_fdot2_f16_f16:
4727 case Intrinsic::amdgcn_fdot2_f32_bf16:
4728 case Intrinsic::amdgcn_fdot2c_f32_bf16:
4729 case Intrinsic::amdgcn_sudot4:
4730 case Intrinsic::amdgcn_sudot8:
4731 case Intrinsic::amdgcn_dot4_f32_fp8_bf8:
4732 case Intrinsic::amdgcn_dot4_f32_bf8_fp8:
4733 case Intrinsic::amdgcn_dot4_f32_fp8_fp8:
4734 case Intrinsic::amdgcn_dot4_f32_bf8_bf8:
4735 case Intrinsic::amdgcn_cvt_f32_fp8:
4736 case Intrinsic::amdgcn_cvt_f32_fp8_e5m3:
4737 case Intrinsic::amdgcn_cvt_f32_bf8:
4738 case Intrinsic::amdgcn_cvt_off_f32_i4:
4739 case Intrinsic::amdgcn_cvt_pk_f32_fp8:
4740 case Intrinsic::amdgcn_cvt_pk_f32_bf8:
4741 case Intrinsic::amdgcn_cvt_pk_fp8_f32:
4742 case Intrinsic::amdgcn_cvt_pk_fp8_f32_e5m3:
4743 case Intrinsic::amdgcn_cvt_pk_bf8_f32:
4744 case Intrinsic::amdgcn_cvt_sr_fp8_f32:
4745 case Intrinsic::amdgcn_cvt_sr_fp8_f32_e5m3:
4746 case Intrinsic::amdgcn_cvt_sr_bf8_f32:
4747 case Intrinsic::amdgcn_cvt_sr_bf16_f32:
4748 case Intrinsic::amdgcn_cvt_sr_f16_f32:
4749 case Intrinsic::amdgcn_cvt_f16_fp8:
4750 case Intrinsic::amdgcn_cvt_f16_bf8:
4751 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f16:
4752 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f16:
4753 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_bf16:
4754 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_bf16:
4755 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f32:
4756 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f32:
4757 case Intrinsic::amdgcn_cvt_scalef32_f16_fp8:
4758 case Intrinsic::amdgcn_cvt_scalef32_f16_bf8:
4759 case Intrinsic::amdgcn_cvt_scalef32_f32_fp8:
4760 case Intrinsic::amdgcn_cvt_scalef32_f32_bf8:
4761 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_f32:
4762 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_f32:
4763 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_fp8:
4764 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_bf8:
4765 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_f16:
4766 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_bf16:
4767 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_f16:
4768 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_bf16:
4769 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_fp4:
4770 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_f32:
4771 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_fp4:
4772 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_fp4:
4773 case Intrinsic::amdgcn_cvt_scalef32_pk32_f32_fp6:
4774 case Intrinsic::amdgcn_cvt_scalef32_pk32_f32_bf6:
4775 case Intrinsic::amdgcn_cvt_scalef32_pk32_f16_bf6:
4776 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf16_bf6:
4777 case Intrinsic::amdgcn_cvt_scalef32_pk32_f16_fp6:
4778 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf16_fp6:
4779 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_bf8:
4780 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_bf8:
4781 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_fp8:
4782 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_fp8:
4783 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_f16:
4784 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_bf16:
4785 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_f16:
4786 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_bf16:
4787 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_f32:
4788 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_bf16:
4789 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_f16:
4790 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_f32:
4791 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_bf16:
4792 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_f16:
4793 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_f32:
4794 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_bf16:
4795 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_f16:
4796 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_f32:
4797 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_bf16:
4798 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_f16:
4799 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_f32:
4800 case Intrinsic::amdgcn_ashr_pk_i8_i32:
4801 case Intrinsic::amdgcn_ashr_pk_u8_i32:
4802 case Intrinsic::amdgcn_cvt_scalef32_2xpk16_fp6_f32:
4803 case Intrinsic::amdgcn_cvt_scalef32_2xpk16_bf6_f32:
4804 case Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16:
4805 case Intrinsic::amdgcn_wmma_f16_16x16x16_f16:
4806 case Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied:
4807 case Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied:
4808 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf16:
4809 case Intrinsic::amdgcn_wmma_f32_16x16x16_f16:
4810 case Intrinsic::amdgcn_wmma_i32_16x16x16_iu4:
4811 case Intrinsic::amdgcn_wmma_i32_16x16x16_iu8:
4812 case Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8:
4813 case Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8:
4814 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8:
4815 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8:
4816 case Intrinsic::amdgcn_wmma_i32_16x16x32_iu4:
4817 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
4818 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
4819 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
4820 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
4821 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
4822 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
4823 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4:
4824 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
4825 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
4826 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
4827 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8:
4828 case Intrinsic::amdgcn_wmma_f64_16x16x4_f64:
4829 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
4830 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
4831 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
4832 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
4833 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
4834 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
4835 case Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_fp8:
4836 case Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_bf8:
4837 case Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_fp8:
4838 case Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_bf8:
4839 case Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_fp8:
4840 case Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_bf8:
4841 case Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_fp8:
4842 case Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_bf8:
4843 case Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_fp8:
4844 case Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_bf8:
4845 case Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_fp8:
4846 case Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_bf8:
4847 case Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_fp8:
4848 case Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_bf8:
4849 case Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_fp8:
4850 case Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_bf8:
4851 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
4852 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
4853 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
4854 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4:
4855 case Intrinsic::amdgcn_wmma_f32_32x16x128_f4:
4856 case Intrinsic::amdgcn_wmma_scale_f32_32x16x128_f4:
4857 case Intrinsic::amdgcn_wmma_scale16_f32_32x16x128_f4:
4858 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
4859 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
4860 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
4861 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
4862 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
4863 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
4864 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
4865 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
4866 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
4867 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
4868 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
4869 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
4870 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8:
4871 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
4872 case Intrinsic::amdgcn_perm_pk16_b4_u4:
4873 case Intrinsic::amdgcn_perm_pk16_b6_u4:
4874 case Intrinsic::amdgcn_perm_pk16_b8_u4:
4875 case Intrinsic::amdgcn_add_max_i32:
4876 case Intrinsic::amdgcn_add_max_u32:
4877 case Intrinsic::amdgcn_add_min_i32:
4878 case Intrinsic::amdgcn_add_min_u32:
4879 case Intrinsic::amdgcn_pk_add_max_i16:
4880 case Intrinsic::amdgcn_pk_add_max_u16:
4881 case Intrinsic::amdgcn_pk_add_min_i16:
4882 case Intrinsic::amdgcn_pk_add_min_u16:
4884 case Intrinsic::amdgcn_log:
4885 case Intrinsic::amdgcn_exp2:
4886 case Intrinsic::amdgcn_rcp:
4887 case Intrinsic::amdgcn_rsq:
4888 case Intrinsic::amdgcn_sqrt: {
4889 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
4890 unsigned Size = Ty.getSizeInBits();
4892 if (
Subtarget.hasPseudoScalarTrans() && !Ty.isBFloat16() &&
4897 case Intrinsic::amdgcn_sbfe:
4898 case Intrinsic::amdgcn_ubfe:
4902 case Intrinsic::amdgcn_ds_swizzle:
4903 case Intrinsic::amdgcn_ds_permute:
4904 case Intrinsic::amdgcn_ds_bpermute:
4905 case Intrinsic::amdgcn_update_dpp:
4906 case Intrinsic::amdgcn_mov_dpp8:
4907 case Intrinsic::amdgcn_mov_dpp:
4908 case Intrinsic::amdgcn_strict_wwm:
4909 case Intrinsic::amdgcn_wwm:
4910 case Intrinsic::amdgcn_strict_wqm:
4911 case Intrinsic::amdgcn_wqm:
4912 case Intrinsic::amdgcn_softwqm:
4913 case Intrinsic::amdgcn_set_inactive:
4914 case Intrinsic::amdgcn_set_inactive_chain_arg:
4915 case Intrinsic::amdgcn_permlane64:
4916 case Intrinsic::amdgcn_ds_bpermute_fi_b32:
4918 case Intrinsic::amdgcn_cvt_pkrtz:
4922 case Intrinsic::amdgcn_kernarg_segment_ptr:
4923 case Intrinsic::amdgcn_s_getpc:
4924 case Intrinsic::amdgcn_groupstaticsize:
4925 case Intrinsic::amdgcn_reloc_constant:
4926 case Intrinsic::returnaddress: {
4927 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4928 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID,
Size);
4931 case Intrinsic::amdgcn_wqm_vote: {
4932 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4933 OpdsMapping[0] = OpdsMapping[2]
4934 = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID,
Size);
4937 case Intrinsic::amdgcn_ps_live: {
4938 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4941 case Intrinsic::amdgcn_div_scale: {
4942 unsigned Dst0Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4943 unsigned Dst1Size =
MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits();
4944 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Dst0Size);
4945 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Dst1Size);
4947 unsigned SrcSize =
MRI.getType(
MI.getOperand(3).getReg()).getSizeInBits();
4948 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4949 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4952 case Intrinsic::amdgcn_class: {
4953 Register Src0Reg =
MI.getOperand(2).getReg();
4954 Register Src1Reg =
MI.getOperand(3).getReg();
4955 unsigned Src0Size =
MRI.getType(Src0Reg).getSizeInBits();
4956 unsigned Src1Size =
MRI.getType(Src1Reg).getSizeInBits();
4957 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4958 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize);
4959 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src0Size);
4960 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src1Size);
4963 case Intrinsic::amdgcn_readlane: {
4966 unsigned IdxSize =
MRI.getType(IdxReg).getSizeInBits();
4968 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize);
4971 case Intrinsic::amdgcn_readfirstlane: {
4972 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4973 unsigned SrcSize =
MRI.getType(
MI.getOperand(2).getReg()).getSizeInBits();
4974 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize);
4975 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4978 case Intrinsic::amdgcn_writelane: {
4979 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
4981 unsigned SrcSize =
MRI.getType(SrcReg).getSizeInBits();
4984 unsigned IdxSize =
MRI.getType(IdxReg).getSizeInBits();
4986 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
4990 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, SrcSize);
4991 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize);
4992 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize);
4995 case Intrinsic::amdgcn_if_break: {
4996 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
4997 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID,
Size);
4998 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
4999 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID,
Size);
5002 case Intrinsic::amdgcn_permlane16:
5003 case Intrinsic::amdgcn_permlanex16: {
5004 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
5005 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5006 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5007 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5012 case Intrinsic::amdgcn_permlane_bcast:
5013 case Intrinsic::amdgcn_permlane_up:
5014 case Intrinsic::amdgcn_permlane_down:
5015 case Intrinsic::amdgcn_permlane_xor: {
5016 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
5017 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5018 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5023 case Intrinsic::amdgcn_permlane_idx_gen: {
5024 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
5025 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5026 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5030 case Intrinsic::amdgcn_permlane16_var:
5031 case Intrinsic::amdgcn_permlanex16_var: {
5032 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
5033 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5034 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5035 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5036 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5039 case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
5040 case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
5041 case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
5042 case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
5043 case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
5044 case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
5045 case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
5046 case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
5047 case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
5048 case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
5049 case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
5050 case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
5051 case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
5052 case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
5053 case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
5054 case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
5055 case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
5056 case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
5057 case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
5058 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
5059 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
5060 case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
5061 case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
5062 case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
5063 case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
5064 case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
5065 case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
5066 case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
5067 case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
5068 case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
5069 case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
5070 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
5071 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
5072 case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
5073 case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
5074 case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
5075 case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
5076 case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
5077 case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
5078 case Intrinsic::amdgcn_mfma_f32_16x16x32_f16:
5079 case Intrinsic::amdgcn_mfma_f32_32x32x16_f16:
5080 case Intrinsic::amdgcn_mfma_i32_16x16x64_i8:
5081 case Intrinsic::amdgcn_mfma_i32_32x32x32_i8:
5082 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf16: {
5083 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5084 unsigned MinNumRegsRequired = DstSize / 32;
5094 bool UseAGPRForm = !
Subtarget.hasGFX90AInsts() ||
5095 Info->selectAGPRFormMFMA(MinNumRegsRequired);
5107 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
5108 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
5109 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5110 unsigned MinNumRegsRequired = DstSize / 32;
5129 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
5130 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
5131 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
5132 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
5133 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
5134 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
5135 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
5136 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
5137 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
5138 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
5139 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
5140 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
5141 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
5142 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
5143 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
5144 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
5145 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
5146 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
5147 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
5148 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
5149 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
5150 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
5151 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
5152 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
5153 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
5154 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
5155 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
5156 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8: {
5158 unsigned DstSize =
MRI.getType(DstReg).getSizeInBits();
5159 unsigned MinNumRegsRequired = DstSize / 32;
5175 case Intrinsic::amdgcn_interp_p1:
5176 case Intrinsic::amdgcn_interp_p2:
5177 case Intrinsic::amdgcn_interp_mov:
5178 case Intrinsic::amdgcn_interp_p1_f16:
5179 case Intrinsic::amdgcn_interp_p2_f16:
5180 case Intrinsic::amdgcn_lds_param_load: {
5181 const int M0Idx =
MI.getNumOperands() - 1;
5182 Register M0Reg =
MI.getOperand(M0Idx).getReg();
5184 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5186 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
5187 for (
int I = 2;
I != M0Idx &&
MI.getOperand(
I).
isReg(); ++
I)
5188 OpdsMapping[
I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5192 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(M0Bank, 32);
5195 case Intrinsic::amdgcn_interp_inreg_p10:
5196 case Intrinsic::amdgcn_interp_inreg_p2:
5197 case Intrinsic::amdgcn_interp_inreg_p10_f16:
5198 case Intrinsic::amdgcn_interp_inreg_p2_f16:
5199 case Intrinsic::amdgcn_interp_p10_rtz_f16:
5200 case Intrinsic::amdgcn_interp_p2_rtz_f16: {
5201 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5202 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
5203 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5204 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5205 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5208 case Intrinsic::amdgcn_permlane16_swap:
5209 case Intrinsic::amdgcn_permlane32_swap: {
5210 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5211 OpdsMapping[0] = OpdsMapping[1] = OpdsMapping[3] = OpdsMapping[4] =
5212 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
5215 case Intrinsic::amdgcn_ballot: {
5216 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5217 unsigned SrcSize =
MRI.getType(
MI.getOperand(2).getReg()).getSizeInBits();
5218 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize);
5219 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, SrcSize);
5222 case Intrinsic::amdgcn_inverse_ballot: {
5224 Register MaskReg =
MI.getOperand(2).getReg();
5225 unsigned MaskSize =
MRI.getType(MaskReg).getSizeInBits();
5226 unsigned MaskBank =
getRegBankID(MaskReg,
MRI, AMDGPU::SGPRRegBankID);
5227 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
5228 OpdsMapping[2] = AMDGPU::getValueMapping(MaskBank, MaskSize);
5231 case Intrinsic::amdgcn_bitop3: {
5232 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
5233 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5234 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5235 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5236 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5239 case Intrinsic::amdgcn_s_quadmask:
5240 case Intrinsic::amdgcn_s_wqm: {
5241 Register MaskReg =
MI.getOperand(2).getReg();
5242 unsigned MaskSize =
MRI.getType(MaskReg).getSizeInBits();
5243 unsigned MaskBank =
getRegBankID(MaskReg,
MRI, AMDGPU::SGPRRegBankID);
5244 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, MaskSize);
5245 OpdsMapping[2] = AMDGPU::getValueMapping(MaskBank, MaskSize);
5248 case Intrinsic::amdgcn_wave_reduce_add:
5249 case Intrinsic::amdgcn_wave_reduce_fadd:
5250 case Intrinsic::amdgcn_wave_reduce_sub:
5251 case Intrinsic::amdgcn_wave_reduce_fsub:
5252 case Intrinsic::amdgcn_wave_reduce_min:
5253 case Intrinsic::amdgcn_wave_reduce_umin:
5254 case Intrinsic::amdgcn_wave_reduce_fmin:
5255 case Intrinsic::amdgcn_wave_reduce_max:
5256 case Intrinsic::amdgcn_wave_reduce_umax:
5257 case Intrinsic::amdgcn_wave_reduce_fmax:
5258 case Intrinsic::amdgcn_wave_reduce_and:
5259 case Intrinsic::amdgcn_wave_reduce_or:
5260 case Intrinsic::amdgcn_wave_reduce_xor: {
5261 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5262 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize);
5263 unsigned OpSize =
MRI.getType(
MI.getOperand(2).getReg()).getSizeInBits();
5266 OpdsMapping[2] = AMDGPU::getValueMapping(regBankID, OpSize);
5269 case Intrinsic::amdgcn_s_bitreplicate: {
5270 Register MaskReg =
MI.getOperand(2).getReg();
5271 unsigned MaskBank =
getRegBankID(MaskReg,
MRI, AMDGPU::SGPRRegBankID);
5272 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 64);
5273 OpdsMapping[2] = AMDGPU::getValueMapping(MaskBank, 32);
5276 case Intrinsic::amdgcn_wave_shuffle: {
5277 unsigned OpSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5278 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize);
5279 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize);
5280 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize);
5286 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
5287 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
5288 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
5289 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
5290 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
5293 assert(RSrcIntrin &&
"missing RsrcIntrinsic for image intrinsic");
5300 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
5301 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
5302 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY: {
5304 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY ||
5305 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY;
5306 unsigned NumMods = IsDualOrBVH8 ? 0 : 1;
5307 unsigned LastRegOpIdx =
MI.getNumExplicitOperands() - 1 - NumMods;
5308 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5309 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
5311 OpdsMapping[1] = AMDGPU::getValueMapping(
5312 AMDGPU::VGPRRegBankID,
5313 MRI.getType(
MI.getOperand(1).getReg()).getSizeInBits());
5314 OpdsMapping[2] = AMDGPU::getValueMapping(
5315 AMDGPU::VGPRRegBankID,
5316 MRI.getType(
MI.getOperand(2).getReg()).getSizeInBits());
5318 OpdsMapping[LastRegOpIdx] =
5320 if (LastRegOpIdx == 3) {
5322 unsigned Size =
MRI.getType(
MI.getOperand(2).getReg()).getSizeInBits();
5325 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5328 unsigned FirstSrcOpIdx = IsDualOrBVH8 ? 4 : 2;
5329 for (
unsigned I = FirstSrcOpIdx;
I < LastRegOpIdx; ++
I) {
5330 unsigned Size =
MRI.getType(
MI.getOperand(
I).getReg()).getSizeInBits();
5331 OpdsMapping[
I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5336 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
5337 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
5340 case Intrinsic::amdgcn_s_getreg:
5341 case Intrinsic::amdgcn_s_memtime:
5342 case Intrinsic::amdgcn_s_memrealtime:
5343 case Intrinsic::amdgcn_s_get_waveid_in_workgroup:
5344 case Intrinsic::amdgcn_s_sendmsg_rtn: {
5345 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5346 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID,
Size);
5349 case Intrinsic::amdgcn_global_atomic_fmin_num:
5350 case Intrinsic::amdgcn_global_atomic_fmax_num:
5351 case Intrinsic::amdgcn_flat_atomic_fmin_num:
5352 case Intrinsic::amdgcn_flat_atomic_fmax_num:
5353 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
5354 case Intrinsic::amdgcn_global_load_tr_b64:
5355 case Intrinsic::amdgcn_global_load_tr_b128:
5356 case Intrinsic::amdgcn_global_load_tr4_b64:
5357 case Intrinsic::amdgcn_global_load_tr6_b96:
5358 case Intrinsic::amdgcn_ds_load_tr8_b64:
5359 case Intrinsic::amdgcn_ds_load_tr16_b128:
5360 case Intrinsic::amdgcn_ds_load_tr4_b64:
5361 case Intrinsic::amdgcn_ds_load_tr6_b96:
5362 case Intrinsic::amdgcn_ds_read_tr4_b64:
5363 case Intrinsic::amdgcn_ds_read_tr6_b96:
5364 case Intrinsic::amdgcn_ds_read_tr8_b64:
5365 case Intrinsic::amdgcn_ds_read_tr16_b64:
5366 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
5367 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
5369 case Intrinsic::amdgcn_ds_ordered_add:
5370 case Intrinsic::amdgcn_ds_ordered_swap: {
5371 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5372 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
5374 AMDGPU::SGPRRegBankID);
5375 OpdsMapping[2] = AMDGPU::getValueMapping(M0Bank, 32);
5376 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5379 case Intrinsic::amdgcn_ds_append:
5380 case Intrinsic::amdgcn_ds_consume: {
5381 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5382 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
5386 case Intrinsic::amdgcn_exp_compr:
5387 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5388 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5390 case Intrinsic::amdgcn_exp:
5392 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5393 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5394 OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5395 OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5397 case Intrinsic::amdgcn_exp_row:
5398 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5399 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5400 OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5401 OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5404 case Intrinsic::amdgcn_s_alloc_vgpr:
5405 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1);
5406 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
5408 case Intrinsic::amdgcn_s_sendmsg:
5409 case Intrinsic::amdgcn_s_sendmsghalt: {
5412 AMDGPU::SGPRRegBankID);
5413 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
5416 case Intrinsic::amdgcn_s_setreg: {
5419 AMDGPU::SGPRRegBankID);
5420 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
5423 case Intrinsic::amdgcn_s_ttracedata: {
5427 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, 32);
5430 case Intrinsic::amdgcn_end_cf: {
5431 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(1).getReg(),
MRI);
5432 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID,
Size);
5435 case Intrinsic::amdgcn_else: {
5436 unsigned WaveSize =
TRI->getRegSizeInBits(
MI.getOperand(1).getReg(),
MRI);
5437 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
5438 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize);
5439 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize);
5442 case Intrinsic::amdgcn_init_whole_wave:
5443 case Intrinsic::amdgcn_live_mask: {
5444 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
5447 case Intrinsic::amdgcn_wqm_demote:
5448 case Intrinsic::amdgcn_kill: {
5449 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
5452 case Intrinsic::amdgcn_ptr_s_buffer_load: {
5458 case Intrinsic::amdgcn_raw_buffer_load:
5459 case Intrinsic::amdgcn_raw_ptr_buffer_load:
5460 case Intrinsic::amdgcn_raw_atomic_buffer_load:
5461 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
5462 case Intrinsic::amdgcn_raw_tbuffer_load:
5463 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
5472 case Intrinsic::amdgcn_raw_buffer_load_lds:
5473 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
5474 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
5475 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds: {
5482 case Intrinsic::amdgcn_raw_buffer_store:
5483 case Intrinsic::amdgcn_raw_ptr_buffer_store:
5484 case Intrinsic::amdgcn_raw_buffer_store_format:
5485 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
5486 case Intrinsic::amdgcn_raw_tbuffer_store:
5487 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
5494 case Intrinsic::amdgcn_struct_buffer_load:
5495 case Intrinsic::amdgcn_struct_ptr_buffer_load:
5496 case Intrinsic::amdgcn_struct_tbuffer_load:
5497 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
5498 case Intrinsic::amdgcn_struct_atomic_buffer_load:
5499 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
5507 case Intrinsic::amdgcn_struct_buffer_load_lds:
5508 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
5509 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
5510 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
5518 case Intrinsic::amdgcn_struct_buffer_store:
5519 case Intrinsic::amdgcn_struct_ptr_buffer_store:
5520 case Intrinsic::amdgcn_struct_tbuffer_store:
5521 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
5529 case Intrinsic::amdgcn_init_exec_from_input: {
5530 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(1).getReg(),
MRI);
5531 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID,
Size);
5534 case Intrinsic::amdgcn_ds_gws_init:
5535 case Intrinsic::amdgcn_ds_gws_barrier:
5536 case Intrinsic::amdgcn_ds_gws_sema_br: {
5537 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5541 AMDGPU::SGPRRegBankID);
5542 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
5545 case Intrinsic::amdgcn_ds_gws_sema_v:
5546 case Intrinsic::amdgcn_ds_gws_sema_p:
5547 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
5550 AMDGPU::SGPRRegBankID);
5551 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, 32);
5554 case Intrinsic::amdgcn_cluster_load_b32:
5555 case Intrinsic::amdgcn_cluster_load_b64:
5556 case Intrinsic::amdgcn_cluster_load_b128: {
5561 OpdsMapping[4] = AMDGPU::getValueMapping(M0Bank, 32);
5564 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
5565 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
5566 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
5567 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
5573 OpdsMapping[5] = AMDGPU::getValueMapping(M0Bank, 32);
5576 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
5577 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
5578 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
5579 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
5580 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
5581 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
5582 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
5583 case Intrinsic::amdgcn_global_load_async_to_lds_b128: {
5589 case Intrinsic::amdgcn_load_to_lds:
5590 case Intrinsic::amdgcn_load_async_to_lds:
5591 case Intrinsic::amdgcn_global_load_lds:
5592 case Intrinsic::amdgcn_global_load_async_lds: {
5598 case Intrinsic::amdgcn_lds_direct_load: {
5599 const int M0Idx =
MI.getNumOperands() - 1;
5600 Register M0Reg =
MI.getOperand(M0Idx).getReg();
5602 unsigned DstSize =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5604 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize);
5605 for (
int I = 2;
I != M0Idx &&
MI.getOperand(
I).
isReg(); ++
I)
5606 OpdsMapping[
I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
5610 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(M0Bank, 32);
5613 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
5614 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn:
5618 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
5619 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
5620 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
5621 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
5634 case Intrinsic::amdgcn_s_sleep_var:
5637 case Intrinsic::amdgcn_s_barrier_join:
5638 case Intrinsic::amdgcn_s_wakeup_barrier:
5641 case Intrinsic::amdgcn_s_barrier_init:
5642 case Intrinsic::amdgcn_s_barrier_signal_var:
5646 case Intrinsic::amdgcn_s_barrier_signal_isfirst: {
5647 const unsigned ResultSize = 1;
5649 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, ResultSize);
5652 case Intrinsic::amdgcn_s_get_barrier_state:
5653 case Intrinsic::amdgcn_s_get_named_barrier_state: {
5658 case Intrinsic::amdgcn_pops_exiting_wave_id:
5660 case Intrinsic::amdgcn_tensor_load_to_lds:
5661 case Intrinsic::amdgcn_tensor_store_from_lds: {
5664 for (
unsigned I = 1;
I <
MI.getNumOperands(); ++
I) {
5665 if (
MI.getOperand(
I).isReg()) {
5668 unsigned Size =
TRI->getRegSizeInBits(Reg,
MRI);
5669 OpdsMapping[
I] = AMDGPU::getValueMapping(OpBank,
Size);
5674 case Intrinsic::amdgcn_s_prefetch_data:
5675 case Intrinsic::amdgcn_s_prefetch_inst: {
5680 case Intrinsic::amdgcn_flat_prefetch:
5681 case Intrinsic::amdgcn_global_prefetch:
5688 case AMDGPU::G_SELECT: {
5689 unsigned Size =
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits();
5691 AMDGPU::SGPRRegBankID);
5693 AMDGPU::SGPRRegBankID);
5694 bool SGPRSrcs = Op2Bank == AMDGPU::SGPRRegBankID &&
5695 Op3Bank == AMDGPU::SGPRRegBankID;
5697 unsigned CondBankDefault = SGPRSrcs ?
5698 AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
5701 if (CondBank == AMDGPU::SGPRRegBankID)
5702 CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
5703 else if (CondBank == AMDGPU::VGPRRegBankID)
5704 CondBank = AMDGPU::VCCRegBankID;
5706 unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SGPRRegBankID ?
5707 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
5709 assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SGPRRegBankID);
5713 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(Bank,
Size);
5714 OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1);
5715 OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(Bank,
Size);
5716 OpdsMapping[3] = AMDGPU::getValueMappingSGPR64Only(Bank,
Size);
5718 OpdsMapping[0] = AMDGPU::getValueMapping(Bank,
Size);
5719 OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1);
5720 OpdsMapping[2] = AMDGPU::getValueMapping(Bank,
Size);
5721 OpdsMapping[3] = AMDGPU::getValueMapping(Bank,
Size);
5727 case AMDGPU::G_SI_CALL: {
5728 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 64);
5734 for (
unsigned I = 4;
I <
MI.getNumOperands(); ++
I) {
5735 if (
MI.getOperand(
I).isReg()) {
5738 unsigned Size =
TRI->getRegSizeInBits(Reg,
MRI);
5739 OpdsMapping[
I] = AMDGPU::getValueMapping(OpBank,
Size);
5744 case AMDGPU::G_LOAD:
5745 case AMDGPU::G_ZEXTLOAD:
5746 case AMDGPU::G_SEXTLOAD:
5749 case AMDGPU::G_ATOMICRMW_XCHG:
5750 case AMDGPU::G_ATOMICRMW_ADD:
5751 case AMDGPU::G_ATOMICRMW_SUB:
5752 case AMDGPU::G_ATOMICRMW_AND:
5753 case AMDGPU::G_ATOMICRMW_OR:
5754 case AMDGPU::G_ATOMICRMW_XOR:
5755 case AMDGPU::G_ATOMICRMW_MAX:
5756 case AMDGPU::G_ATOMICRMW_MIN:
5757 case AMDGPU::G_ATOMICRMW_UMAX:
5758 case AMDGPU::G_ATOMICRMW_UMIN:
5759 case AMDGPU::G_ATOMICRMW_FADD:
5760 case AMDGPU::G_ATOMICRMW_FMIN:
5761 case AMDGPU::G_ATOMICRMW_FMAX:
5762 case AMDGPU::G_ATOMICRMW_UINC_WRAP:
5763 case AMDGPU::G_ATOMICRMW_UDEC_WRAP:
5764 case AMDGPU::G_ATOMICRMW_USUB_COND:
5765 case AMDGPU::G_ATOMICRMW_USUB_SAT:
5766 case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: {
5772 case AMDGPU::G_ATOMIC_CMPXCHG: {
5779 case AMDGPU::G_BRCOND: {
5781 AMDGPU::SGPRRegBankID);
5782 assert(
MRI.getType(
MI.getOperand(0).getReg()).getSizeInBits() == 1);
5783 if (Bank != AMDGPU::SGPRRegBankID)
5784 Bank = AMDGPU::VCCRegBankID;
5786 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1);
5789 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
5791 case AMDGPU::G_PREFETCH:
5794 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP:
5795 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN:
5796 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
5798 case AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR:
5799 case AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR: {
5800 unsigned Size =
TRI->getRegSizeInBits(
MI.getOperand(0).getReg(),
MRI);
5801 unsigned PtrSize =
TRI->getRegSizeInBits(
MI.getOperand(1).getReg(),
MRI);
5802 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID,
Size);
5803 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize);
5810 MI.getNumOperands());