27 #define DEBUG_TYPE "nvptx-isel"
31 cl::desc(
"NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
32 " IEEE Compliant F32 div.rnd if available."),
37 cl::desc(
"NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
42 cl::desc(
"NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
56 doMulWide = (OptLevel > 0);
64 int NVPTXDAGToDAGISel::getDivF32Level()
const {
77 bool NVPTXDAGToDAGISel::usePrecSqrtF32()
const {
87 bool NVPTXDAGToDAGISel::useF32FTZ()
const {
101 bool NVPTXDAGToDAGISel::allowFMA()
const {
108 void NVPTXDAGToDAGISel::Select(
SDNode *
N) {
126 if (tryLoadVector(N))
138 if (tryStoreVector(N))
150 if (tryStoreRetval(N))
158 if (tryStoreParam(N))
162 if (tryIntrinsicNoChain(N))
166 if (tryIntrinsicChain(N))
337 if (tryTextureIntrinsic(N))
505 if (trySurfaceIntrinsic(N))
516 SelectAddrSpaceCast(N);
524 bool NVPTXDAGToDAGISel::tryIntrinsicChain(
SDNode *N) {
525 unsigned IID = cast<ConstantSDNode>(N->
getOperand(1))->getZExtValue();
529 case Intrinsic::nvvm_ldg_global_f:
530 case Intrinsic::nvvm_ldg_global_i:
531 case Intrinsic::nvvm_ldg_global_p:
532 case Intrinsic::nvvm_ldu_global_f:
533 case Intrinsic::nvvm_ldu_global_i:
534 case Intrinsic::nvvm_ldu_global_p:
545 if (
auto *PT = dyn_cast<PointerType>(Src->
getType())) {
546 switch (PT->getAddressSpace()) {
593 for (
Value *Obj : Objs) {
595 if (!
A || !
A->onlyReadsMemory() || !
A->hasNoAliasAttr())
return false;
601 bool NVPTXDAGToDAGISel::tryIntrinsicNoChain(
SDNode *N) {
602 unsigned IID = cast<ConstantSDNode>(N->
getOperand(0))->getZExtValue();
606 case Intrinsic::nvvm_texsurf_handle_internal:
607 SelectTexSurfHandle(N);
612 void NVPTXDAGToDAGISel::SelectTexSurfHandle(
SDNode *N) {
620 void NVPTXDAGToDAGISel::SelectAddrSpaceCast(
SDNode *N) {
626 assert(SrcAddrSpace != DstAddrSpace &&
627 "addrspacecast must be between different address spaces");
632 switch (SrcAddrSpace) {
635 Opc = TM.
is64Bit() ? NVPTX::cvta_global_yes_64 : NVPTX::cvta_global_yes;
638 Opc = TM.
is64Bit() ? NVPTX::cvta_shared_yes_64 : NVPTX::cvta_shared_yes;
641 Opc = TM.
is64Bit() ? NVPTX::cvta_const_yes_64 : NVPTX::cvta_const_yes;
644 Opc = TM.
is64Bit() ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes;
652 if (SrcAddrSpace != 0)
655 switch (DstAddrSpace) {
658 Opc = TM.
is64Bit() ? NVPTX::cvta_to_global_yes_64
659 : NVPTX::cvta_to_global_yes;
662 Opc = TM.
is64Bit() ? NVPTX::cvta_to_shared_yes_64
663 : NVPTX::cvta_to_shared_yes;
667 TM.
is64Bit() ? NVPTX::cvta_to_const_yes_64 : NVPTX::cvta_to_const_yes;
671 TM.
is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes;
674 Opc = TM.
is64Bit() ? NVPTX::nvvm_ptr_gen_to_param_64
675 : NVPTX::nvvm_ptr_gen_to_param;
684 bool NVPTXDAGToDAGISel::tryLoad(
SDNode *N) {
688 SDNode *NVPTXLD =
nullptr;
694 if (!LoadedVT.isSimple())
713 MVT SimpleVT = LoadedVT.getSimpleVT();
733 unsigned fromTypeWidth = std::max(8U, ScalarVT.
getSizeInBits());
734 unsigned int fromType;
750 if (SelectDirectAddr(N1, Addr)) {
753 Opcode = NVPTX::LD_i8_avar;
756 Opcode = NVPTX::LD_i16_avar;
759 Opcode = NVPTX::LD_i32_avar;
762 Opcode = NVPTX::LD_i64_avar;
765 Opcode = NVPTX::LD_f32_avar;
768 Opcode = NVPTX::LD_f64_avar;
773 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
774 getI32Imm(vecType, dl), getI32Imm(fromType, dl),
775 getI32Imm(fromTypeWidth, dl), Addr, Chain };
781 Opcode = NVPTX::LD_i8_asi;
784 Opcode = NVPTX::LD_i16_asi;
787 Opcode = NVPTX::LD_i32_asi;
790 Opcode = NVPTX::LD_i64_asi;
793 Opcode = NVPTX::LD_f32_asi;
796 Opcode = NVPTX::LD_f64_asi;
801 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
802 getI32Imm(vecType, dl), getI32Imm(fromType, dl),
803 getI32Imm(fromTypeWidth, dl), Base,
Offset, Chain };
810 Opcode = NVPTX::LD_i8_ari_64;
813 Opcode = NVPTX::LD_i16_ari_64;
816 Opcode = NVPTX::LD_i32_ari_64;
819 Opcode = NVPTX::LD_i64_ari_64;
822 Opcode = NVPTX::LD_f32_ari_64;
825 Opcode = NVPTX::LD_f64_ari_64;
833 Opcode = NVPTX::LD_i8_ari;
836 Opcode = NVPTX::LD_i16_ari;
839 Opcode = NVPTX::LD_i32_ari;
842 Opcode = NVPTX::LD_i64_ari;
845 Opcode = NVPTX::LD_f32_ari;
848 Opcode = NVPTX::LD_f64_ari;
854 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
855 getI32Imm(vecType, dl), getI32Imm(fromType, dl),
856 getI32Imm(fromTypeWidth, dl), Base,
Offset, Chain };
862 Opcode = NVPTX::LD_i8_areg_64;
865 Opcode = NVPTX::LD_i16_areg_64;
868 Opcode = NVPTX::LD_i32_areg_64;
871 Opcode = NVPTX::LD_i64_areg_64;
874 Opcode = NVPTX::LD_f32_areg_64;
877 Opcode = NVPTX::LD_f64_areg_64;
885 Opcode = NVPTX::LD_i8_areg;
888 Opcode = NVPTX::LD_i16_areg;
891 Opcode = NVPTX::LD_i32_areg;
894 Opcode = NVPTX::LD_i64_areg;
897 Opcode = NVPTX::LD_f32_areg;
900 Opcode = NVPTX::LD_f64_areg;
906 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
907 getI32Imm(vecType, dl), getI32Imm(fromType, dl),
908 getI32Imm(fromTypeWidth, dl), N1, Chain };
916 MemRefs0[0] = cast<MemSDNode>(
N)->getMemOperand();
917 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
923 bool NVPTXDAGToDAGISel::tryLoadVector(
SDNode *N) {
934 if (!LoadedVT.isSimple())
953 MVT SimpleVT = LoadedVT.getSimpleVT();
963 unsigned FromTypeWidth = std::max(8U, ScalarVT.
getSizeInBits());
966 unsigned ExtensionType = cast<ConstantSDNode>(
990 if (SelectDirectAddr(Op1, Addr)) {
999 Opcode = NVPTX::LDV_i8_v2_avar;
1002 Opcode = NVPTX::LDV_i16_v2_avar;
1005 Opcode = NVPTX::LDV_i32_v2_avar;
1008 Opcode = NVPTX::LDV_i64_v2_avar;
1011 Opcode = NVPTX::LDV_f32_v2_avar;
1014 Opcode = NVPTX::LDV_f64_v2_avar;
1023 Opcode = NVPTX::LDV_i8_v4_avar;
1026 Opcode = NVPTX::LDV_i16_v4_avar;
1029 Opcode = NVPTX::LDV_i32_v4_avar;
1032 Opcode = NVPTX::LDV_f32_v4_avar;
1038 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1039 getI32Imm(VecType, DL), getI32Imm(FromType, DL),
1040 getI32Imm(FromTypeWidth, DL), Addr, Chain };
1052 Opcode = NVPTX::LDV_i8_v2_asi;
1055 Opcode = NVPTX::LDV_i16_v2_asi;
1058 Opcode = NVPTX::LDV_i32_v2_asi;
1061 Opcode = NVPTX::LDV_i64_v2_asi;
1064 Opcode = NVPTX::LDV_f32_v2_asi;
1067 Opcode = NVPTX::LDV_f64_v2_asi;
1076 Opcode = NVPTX::LDV_i8_v4_asi;
1079 Opcode = NVPTX::LDV_i16_v4_asi;
1082 Opcode = NVPTX::LDV_i32_v4_asi;
1085 Opcode = NVPTX::LDV_f32_v4_asi;
1091 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1092 getI32Imm(VecType, DL), getI32Imm(FromType, DL),
1093 getI32Imm(FromTypeWidth, DL), Base,
Offset, Chain };
1106 Opcode = NVPTX::LDV_i8_v2_ari_64;
1109 Opcode = NVPTX::LDV_i16_v2_ari_64;
1112 Opcode = NVPTX::LDV_i32_v2_ari_64;
1115 Opcode = NVPTX::LDV_i64_v2_ari_64;
1118 Opcode = NVPTX::LDV_f32_v2_ari_64;
1121 Opcode = NVPTX::LDV_f64_v2_ari_64;
1130 Opcode = NVPTX::LDV_i8_v4_ari_64;
1133 Opcode = NVPTX::LDV_i16_v4_ari_64;
1136 Opcode = NVPTX::LDV_i32_v4_ari_64;
1139 Opcode = NVPTX::LDV_f32_v4_ari_64;
1153 Opcode = NVPTX::LDV_i8_v2_ari;
1156 Opcode = NVPTX::LDV_i16_v2_ari;
1159 Opcode = NVPTX::LDV_i32_v2_ari;
1162 Opcode = NVPTX::LDV_i64_v2_ari;
1165 Opcode = NVPTX::LDV_f32_v2_ari;
1168 Opcode = NVPTX::LDV_f64_v2_ari;
1177 Opcode = NVPTX::LDV_i8_v4_ari;
1180 Opcode = NVPTX::LDV_i16_v4_ari;
1183 Opcode = NVPTX::LDV_i32_v4_ari;
1186 Opcode = NVPTX::LDV_f32_v4_ari;
1193 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1194 getI32Imm(VecType, DL), getI32Imm(FromType, DL),
1195 getI32Imm(FromTypeWidth, DL), Base,
Offset, Chain };
1208 Opcode = NVPTX::LDV_i8_v2_areg_64;
1211 Opcode = NVPTX::LDV_i16_v2_areg_64;
1214 Opcode = NVPTX::LDV_i32_v2_areg_64;
1217 Opcode = NVPTX::LDV_i64_v2_areg_64;
1220 Opcode = NVPTX::LDV_f32_v2_areg_64;
1223 Opcode = NVPTX::LDV_f64_v2_areg_64;
1232 Opcode = NVPTX::LDV_i8_v4_areg_64;
1235 Opcode = NVPTX::LDV_i16_v4_areg_64;
1238 Opcode = NVPTX::LDV_i32_v4_areg_64;
1241 Opcode = NVPTX::LDV_f32_v4_areg_64;
1255 Opcode = NVPTX::LDV_i8_v2_areg;
1258 Opcode = NVPTX::LDV_i16_v2_areg;
1261 Opcode = NVPTX::LDV_i32_v2_areg;
1264 Opcode = NVPTX::LDV_i64_v2_areg;
1267 Opcode = NVPTX::LDV_f32_v2_areg;
1270 Opcode = NVPTX::LDV_f64_v2_areg;
1279 Opcode = NVPTX::LDV_i8_v4_areg;
1282 Opcode = NVPTX::LDV_i16_v4_areg;
1285 Opcode = NVPTX::LDV_i32_v4_areg;
1288 Opcode = NVPTX::LDV_f32_v4_areg;
1295 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1296 getI32Imm(VecType, DL), getI32Imm(FromType, DL),
1297 getI32Imm(FromTypeWidth, DL), Op1, Chain };
1302 MemRefs0[0] = cast<MemSDNode>(
N)->getMemOperand();
1303 cast<MachineSDNode>(
LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1309 bool NVPTXDAGToDAGISel::tryLDGLDU(
SDNode *N) {
1320 Mem = cast<MemIntrinsicSDNode>(
N);
1321 unsigned IID = cast<ConstantSDNode>(N->
getOperand(1))->getZExtValue();
1325 case Intrinsic::nvvm_ldg_global_f:
1326 case Intrinsic::nvvm_ldg_global_i:
1327 case Intrinsic::nvvm_ldg_global_p:
1330 case Intrinsic::nvvm_ldu_global_f:
1331 case Intrinsic::nvvm_ldu_global_i:
1332 case Intrinsic::nvvm_ldu_global_p:
1338 Mem = cast<MemSDNode>(
N);
1347 unsigned NumElts = 1;
1358 for (
unsigned i = 0;
i != NumElts; ++
i) {
1364 if (SelectDirectAddr(Op1, Addr)) {
1374 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
1377 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
1380 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
1383 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
1386 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
1389 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
1397 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
1400 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
1403 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
1406 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
1409 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
1412 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
1422 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
1425 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
1428 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
1431 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
1434 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
1437 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
1446 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
1449 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
1452 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
1455 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
1458 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
1461 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
1470 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
1473 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
1476 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
1479 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
1488 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
1491 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
1494 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
1497 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
1503 SDValue Ops[] = { Addr, Chain };
1518 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
1521 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
1524 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
1527 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
1530 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
1533 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
1541 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
1544 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
1547 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
1550 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
1553 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
1556 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
1567 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
1570 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
1573 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
1576 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
1579 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
1582 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
1591 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
1594 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
1597 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
1600 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
1603 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
1606 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
1616 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
1619 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
1622 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
1625 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
1634 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
1637 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
1640 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1643 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1659 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
1662 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
1665 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
1668 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
1671 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
1674 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
1682 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
1685 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
1688 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
1691 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
1694 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
1697 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
1708 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1711 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1714 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1717 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1720 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1723 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1732 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1735 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1738 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1741 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1744 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1747 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1757 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1760 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1763 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1766 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1775 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1778 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1781 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1784 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1806 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
1809 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
1812 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
1815 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
1818 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
1821 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
1829 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
1832 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
1835 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
1838 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
1841 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
1844 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
1855 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1858 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1861 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1864 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1867 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1870 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1879 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1882 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1885 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1888 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1891 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1894 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1904 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1907 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1910 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1913 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1922 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1925 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1928 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1931 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1947 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
1950 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
1953 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
1956 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
1959 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
1962 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
1970 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
1973 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
1976 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
1979 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
1982 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
1985 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
1996 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1999 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
2002 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
2005 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
2008 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
2011 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
2020 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
2023 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
2026 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
2029 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
2032 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
2035 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
2045 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
2048 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
2051 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
2054 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
2063 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
2066 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
2069 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
2072 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
2079 SDValue Ops[] = { Op1, Chain };
2085 cast<MachineSDNode>(
LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
2101 if (OrigType != EltVT && LdNode) {
2106 unsigned CvtOpc = GetConvertOpcode(OrigType.
getSimpleVT(),
2111 for (
unsigned i = 0;
i != NumElts; ++
i) {
2127 bool NVPTXDAGToDAGISel::tryStore(
SDNode *N) {
2131 SDNode *NVPTXST =
nullptr;
2137 if (!StoreVT.isSimple())
2152 MVT SimpleVT = StoreVT.getSimpleVT();
2169 unsigned int toType;
2184 if (SelectDirectAddr(N2, Addr)) {
2187 Opcode = NVPTX::ST_i8_avar;
2190 Opcode = NVPTX::ST_i16_avar;
2193 Opcode = NVPTX::ST_i32_avar;
2196 Opcode = NVPTX::ST_i64_avar;
2199 Opcode = NVPTX::ST_f32_avar;
2202 Opcode = NVPTX::ST_f64_avar;
2207 SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
2208 getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
2209 getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Addr,
2216 Opcode = NVPTX::ST_i8_asi;
2219 Opcode = NVPTX::ST_i16_asi;
2222 Opcode = NVPTX::ST_i32_asi;
2225 Opcode = NVPTX::ST_i64_asi;
2228 Opcode = NVPTX::ST_f32_asi;
2231 Opcode = NVPTX::ST_f64_asi;
2236 SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
2237 getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
2238 getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
2246 Opcode = NVPTX::ST_i8_ari_64;
2249 Opcode = NVPTX::ST_i16_ari_64;
2252 Opcode = NVPTX::ST_i32_ari_64;
2255 Opcode = NVPTX::ST_i64_ari_64;
2258 Opcode = NVPTX::ST_f32_ari_64;
2261 Opcode = NVPTX::ST_f64_ari_64;
2269 Opcode = NVPTX::ST_i8_ari;
2272 Opcode = NVPTX::ST_i16_ari;
2275 Opcode = NVPTX::ST_i32_ari;
2278 Opcode = NVPTX::ST_i64_ari;
2281 Opcode = NVPTX::ST_f32_ari;
2284 Opcode = NVPTX::ST_f64_ari;
2290 SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
2291 getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
2292 getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
2299 Opcode = NVPTX::ST_i8_areg_64;
2302 Opcode = NVPTX::ST_i16_areg_64;
2305 Opcode = NVPTX::ST_i32_areg_64;
2308 Opcode = NVPTX::ST_i64_areg_64;
2311 Opcode = NVPTX::ST_f32_areg_64;
2314 Opcode = NVPTX::ST_f64_areg_64;
2322 Opcode = NVPTX::ST_i8_areg;
2325 Opcode = NVPTX::ST_i16_areg;
2328 Opcode = NVPTX::ST_i32_areg;
2331 Opcode = NVPTX::ST_i64_areg;
2334 Opcode = NVPTX::ST_f32_areg;
2337 Opcode = NVPTX::ST_f64_areg;
2343 SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
2344 getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
2345 getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), N2,
2354 MemRefs0[0] = cast<MemSDNode>(
N)->getMemOperand();
2355 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2360 bool NVPTXDAGToDAGISel::tryStoreVector(
SDNode *N) {
2389 assert(StoreVT.isSimple() &&
"Store value is not simple");
2393 if (ScalarVT.isFloatingPoint())
2421 StOps.
push_back(getI32Imm(IsVolatile, DL));
2422 StOps.
push_back(getI32Imm(CodeAddrSpace, DL));
2423 StOps.
push_back(getI32Imm(VecType, DL));
2425 StOps.
push_back(getI32Imm(ToTypeWidth, DL));
2427 if (SelectDirectAddr(N2, Addr)) {
2436 Opcode = NVPTX::STV_i8_v2_avar;
2439 Opcode = NVPTX::STV_i16_v2_avar;
2442 Opcode = NVPTX::STV_i32_v2_avar;
2445 Opcode = NVPTX::STV_i64_v2_avar;
2448 Opcode = NVPTX::STV_f32_v2_avar;
2451 Opcode = NVPTX::STV_f64_v2_avar;
2460 Opcode = NVPTX::STV_i8_v4_avar;
2463 Opcode = NVPTX::STV_i16_v4_avar;
2466 Opcode = NVPTX::STV_i32_v4_avar;
2469 Opcode = NVPTX::STV_f32_v4_avar;
2485 Opcode = NVPTX::STV_i8_v2_asi;
2488 Opcode = NVPTX::STV_i16_v2_asi;
2491 Opcode = NVPTX::STV_i32_v2_asi;
2494 Opcode = NVPTX::STV_i64_v2_asi;
2497 Opcode = NVPTX::STV_f32_v2_asi;
2500 Opcode = NVPTX::STV_f64_v2_asi;
2509 Opcode = NVPTX::STV_i8_v4_asi;
2512 Opcode = NVPTX::STV_i16_v4_asi;
2515 Opcode = NVPTX::STV_i32_v4_asi;
2518 Opcode = NVPTX::STV_f32_v4_asi;
2536 Opcode = NVPTX::STV_i8_v2_ari_64;
2539 Opcode = NVPTX::STV_i16_v2_ari_64;
2542 Opcode = NVPTX::STV_i32_v2_ari_64;
2545 Opcode = NVPTX::STV_i64_v2_ari_64;
2548 Opcode = NVPTX::STV_f32_v2_ari_64;
2551 Opcode = NVPTX::STV_f64_v2_ari_64;
2560 Opcode = NVPTX::STV_i8_v4_ari_64;
2563 Opcode = NVPTX::STV_i16_v4_ari_64;
2566 Opcode = NVPTX::STV_i32_v4_ari_64;
2569 Opcode = NVPTX::STV_f32_v4_ari_64;
2583 Opcode = NVPTX::STV_i8_v2_ari;
2586 Opcode = NVPTX::STV_i16_v2_ari;
2589 Opcode = NVPTX::STV_i32_v2_ari;
2592 Opcode = NVPTX::STV_i64_v2_ari;
2595 Opcode = NVPTX::STV_f32_v2_ari;
2598 Opcode = NVPTX::STV_f64_v2_ari;
2607 Opcode = NVPTX::STV_i8_v4_ari;
2610 Opcode = NVPTX::STV_i16_v4_ari;
2613 Opcode = NVPTX::STV_i32_v4_ari;
2616 Opcode = NVPTX::STV_f32_v4_ari;
2634 Opcode = NVPTX::STV_i8_v2_areg_64;
2637 Opcode = NVPTX::STV_i16_v2_areg_64;
2640 Opcode = NVPTX::STV_i32_v2_areg_64;
2643 Opcode = NVPTX::STV_i64_v2_areg_64;
2646 Opcode = NVPTX::STV_f32_v2_areg_64;
2649 Opcode = NVPTX::STV_f64_v2_areg_64;
2658 Opcode = NVPTX::STV_i8_v4_areg_64;
2661 Opcode = NVPTX::STV_i16_v4_areg_64;
2664 Opcode = NVPTX::STV_i32_v4_areg_64;
2667 Opcode = NVPTX::STV_f32_v4_areg_64;
2681 Opcode = NVPTX::STV_i8_v2_areg;
2684 Opcode = NVPTX::STV_i16_v2_areg;
2687 Opcode = NVPTX::STV_i32_v2_areg;
2690 Opcode = NVPTX::STV_i64_v2_areg;
2693 Opcode = NVPTX::STV_f32_v2_areg;
2696 Opcode = NVPTX::STV_f64_v2_areg;
2705 Opcode = NVPTX::STV_i8_v4_areg;
2708 Opcode = NVPTX::STV_i16_v4_areg;
2711 Opcode = NVPTX::STV_i32_v4_areg;
2714 Opcode = NVPTX::STV_f32_v4_areg;
2728 MemRefs0[0] = cast<MemSDNode>(
N)->getMemOperand();
2729 cast<MachineSDNode>(
ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2735 bool NVPTXDAGToDAGISel::tryLoadParam(
SDNode *Node) {
2770 Opc = NVPTX::LoadParamMemI8;
2773 Opc = NVPTX::LoadParamMemI8;
2776 Opc = NVPTX::LoadParamMemI16;
2779 Opc = NVPTX::LoadParamMemI32;
2782 Opc = NVPTX::LoadParamMemI64;
2785 Opc = NVPTX::LoadParamMemF32;
2788 Opc = NVPTX::LoadParamMemF64;
2797 Opc = NVPTX::LoadParamMemV2I8;
2800 Opc = NVPTX::LoadParamMemV2I8;
2803 Opc = NVPTX::LoadParamMemV2I16;
2806 Opc = NVPTX::LoadParamMemV2I32;
2809 Opc = NVPTX::LoadParamMemV2I64;
2812 Opc = NVPTX::LoadParamMemV2F32;
2815 Opc = NVPTX::LoadParamMemV2F64;
2824 Opc = NVPTX::LoadParamMemV4I8;
2827 Opc = NVPTX::LoadParamMemV4I8;
2830 Opc = NVPTX::LoadParamMemV4I16;
2833 Opc = NVPTX::LoadParamMemV4I32;
2836 Opc = NVPTX::LoadParamMemV4F32;
2845 }
else if (VecSize == 2) {
2852 unsigned OffsetVal = cast<ConstantSDNode>(
Offset)->getZExtValue();
2863 bool NVPTXDAGToDAGISel::tryStoreRetval(
SDNode *N) {
2867 unsigned OffsetVal = cast<ConstantSDNode>(
Offset)->getZExtValue();
2871 unsigned NumElts = 1;
2888 for (
unsigned i = 0;
i < NumElts; ++
i)
2896 unsigned Opcode = 0;
2905 Opcode = NVPTX::StoreRetvalI8;
2908 Opcode = NVPTX::StoreRetvalI8;
2911 Opcode = NVPTX::StoreRetvalI16;
2914 Opcode = NVPTX::StoreRetvalI32;
2917 Opcode = NVPTX::StoreRetvalI64;
2920 Opcode = NVPTX::StoreRetvalF32;
2923 Opcode = NVPTX::StoreRetvalF64;
2932 Opcode = NVPTX::StoreRetvalV2I8;
2935 Opcode = NVPTX::StoreRetvalV2I8;
2938 Opcode = NVPTX::StoreRetvalV2I16;
2941 Opcode = NVPTX::StoreRetvalV2I32;
2944 Opcode = NVPTX::StoreRetvalV2I64;
2947 Opcode = NVPTX::StoreRetvalV2F32;
2950 Opcode = NVPTX::StoreRetvalV2F64;
2959 Opcode = NVPTX::StoreRetvalV4I8;
2962 Opcode = NVPTX::StoreRetvalV4I8;
2965 Opcode = NVPTX::StoreRetvalV4I16;
2968 Opcode = NVPTX::StoreRetvalV4I32;
2971 Opcode = NVPTX::StoreRetvalV4F32;
2980 MemRefs0[0] = cast<MemSDNode>(
N)->getMemOperand();
2981 cast<MachineSDNode>(
Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2987 bool NVPTXDAGToDAGISel::tryStoreParam(
SDNode *N) {
2991 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2993 unsigned OffsetVal = cast<ConstantSDNode>(
Offset)->getZExtValue();
2998 unsigned NumElts = 1;
3017 for (
unsigned i = 0;
i < NumElts; ++
i)
3027 unsigned Opcode = 0;
3038 Opcode = NVPTX::StoreParamI8;
3041 Opcode = NVPTX::StoreParamI8;
3044 Opcode = NVPTX::StoreParamI16;
3047 Opcode = NVPTX::StoreParamI32;
3050 Opcode = NVPTX::StoreParamI64;
3053 Opcode = NVPTX::StoreParamF32;
3056 Opcode = NVPTX::StoreParamF64;
3065 Opcode = NVPTX::StoreParamV2I8;
3068 Opcode = NVPTX::StoreParamV2I8;
3071 Opcode = NVPTX::StoreParamV2I16;
3074 Opcode = NVPTX::StoreParamV2I32;
3077 Opcode = NVPTX::StoreParamV2I64;
3080 Opcode = NVPTX::StoreParamV2F32;
3083 Opcode = NVPTX::StoreParamV2F64;
3092 Opcode = NVPTX::StoreParamV4I8;
3095 Opcode = NVPTX::StoreParamV4I8;
3098 Opcode = NVPTX::StoreParamV4I16;
3101 Opcode = NVPTX::StoreParamV4I32;
3104 Opcode = NVPTX::StoreParamV4F32;
3114 Opcode = NVPTX::StoreParamI32;
3123 Opcode = NVPTX::StoreParamI32;
3137 MemRefs0[0] = cast<MemSDNode>(
N)->getMemOperand();
3138 cast<MachineSDNode>(
Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
3144 bool NVPTXDAGToDAGISel::tryTextureIntrinsic(
SDNode *N) {
3150 default:
return false;
3152 Opc = NVPTX::TEX_1D_F32_S32;
3155 Opc = NVPTX::TEX_1D_F32_F32;
3158 Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
3161 Opc = NVPTX::TEX_1D_F32_F32_GRAD;
3164 Opc = NVPTX::TEX_1D_S32_S32;
3167 Opc = NVPTX::TEX_1D_S32_F32;
3170 Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
3173 Opc = NVPTX::TEX_1D_S32_F32_GRAD;
3176 Opc = NVPTX::TEX_1D_U32_S32;
3179 Opc = NVPTX::TEX_1D_U32_F32;
3182 Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
3185 Opc = NVPTX::TEX_1D_U32_F32_GRAD;
3188 Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
3191 Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
3194 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
3197 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
3200 Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
3203 Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
3206 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
3209 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
3212 Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
3215 Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
3218 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
3221 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
3224 Opc = NVPTX::TEX_2D_F32_S32;
3227 Opc = NVPTX::TEX_2D_F32_F32;
3230 Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
3233 Opc = NVPTX::TEX_2D_F32_F32_GRAD;
3236 Opc = NVPTX::TEX_2D_S32_S32;
3239 Opc = NVPTX::TEX_2D_S32_F32;
3242 Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
3245 Opc = NVPTX::TEX_2D_S32_F32_GRAD;
3248 Opc = NVPTX::TEX_2D_U32_S32;
3251 Opc = NVPTX::TEX_2D_U32_F32;
3254 Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
3257 Opc = NVPTX::TEX_2D_U32_F32_GRAD;
3260 Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
3263 Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
3266 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
3269 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
3272 Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
3275 Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
3278 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
3281 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
3284 Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
3287 Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
3290 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
3293 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
3296 Opc = NVPTX::TEX_3D_F32_S32;
3299 Opc = NVPTX::TEX_3D_F32_F32;
3302 Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
3305 Opc = NVPTX::TEX_3D_F32_F32_GRAD;
3308 Opc = NVPTX::TEX_3D_S32_S32;
3311 Opc = NVPTX::TEX_3D_S32_F32;
3314 Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
3317 Opc = NVPTX::TEX_3D_S32_F32_GRAD;
3320 Opc = NVPTX::TEX_3D_U32_S32;
3323 Opc = NVPTX::TEX_3D_U32_F32;
3326 Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
3329 Opc = NVPTX::TEX_3D_U32_F32_GRAD;
3332 Opc = NVPTX::TEX_CUBE_F32_F32;
3335 Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
3338 Opc = NVPTX::TEX_CUBE_S32_F32;
3341 Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
3344 Opc = NVPTX::TEX_CUBE_U32_F32;
3347 Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
3350 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
3353 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
3356 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
3359 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
3362 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
3365 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
3368 Opc = NVPTX::TLD4_R_2D_F32_F32;
3371 Opc = NVPTX::TLD4_G_2D_F32_F32;
3374 Opc = NVPTX::TLD4_B_2D_F32_F32;
3377 Opc = NVPTX::TLD4_A_2D_F32_F32;
3380 Opc = NVPTX::TLD4_R_2D_S32_F32;
3383 Opc = NVPTX::TLD4_G_2D_S32_F32;
3386 Opc = NVPTX::TLD4_B_2D_S32_F32;
3389 Opc = NVPTX::TLD4_A_2D_S32_F32;
3392 Opc = NVPTX::TLD4_R_2D_U32_F32;
3395 Opc = NVPTX::TLD4_G_2D_U32_F32;
3398 Opc = NVPTX::TLD4_B_2D_U32_F32;
3401 Opc = NVPTX::TLD4_A_2D_U32_F32;
3404 Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
3407 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
3410 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
3413 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
3416 Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
3419 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
3422 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
3425 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
3428 Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
3431 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
3434 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
3437 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
3440 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
3443 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
3446 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
3449 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
3452 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
3455 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
3458 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
3461 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
3464 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
3467 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
3470 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
3473 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
3476 Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
3479 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
3482 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
3485 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
3488 Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
3491 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
3494 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
3497 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
3500 Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
3503 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
3506 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
3509 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
3512 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
3515 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
3518 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
3521 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
3524 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
3527 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
3530 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
3533 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
3536 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
3539 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
3542 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
3545 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
3548 Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
3551 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
3554 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
3557 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
3560 Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
3563 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
3566 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
3569 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
3572 Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
3575 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
3578 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
3581 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
3584 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
3587 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
3590 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
3593 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
3596 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
3599 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
3602 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
3605 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
3608 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
3611 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
3614 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
3617 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
3620 Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
3623 Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
3626 Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
3629 Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
3632 Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
3635 Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
3638 Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
3641 Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
3644 Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
3647 Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
3650 Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
3653 Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
3667 bool NVPTXDAGToDAGISel::trySurfaceIntrinsic(
SDNode *N) {
3673 default:
return false;
3675 Opc = NVPTX::SULD_1D_I8_CLAMP;
3681 Opc = NVPTX::SULD_1D_I16_CLAMP;
3687 Opc = NVPTX::SULD_1D_I32_CLAMP;
3693 Opc = NVPTX::SULD_1D_I64_CLAMP;
3699 Opc = NVPTX::SULD_1D_V2I8_CLAMP;
3705 Opc = NVPTX::SULD_1D_V2I16_CLAMP;
3711 Opc = NVPTX::SULD_1D_V2I32_CLAMP;
3717 Opc = NVPTX::SULD_1D_V2I64_CLAMP;
3723 Opc = NVPTX::SULD_1D_V4I8_CLAMP;
3729 Opc = NVPTX::SULD_1D_V4I16_CLAMP;
3735 Opc = NVPTX::SULD_1D_V4I32_CLAMP;
3741 Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
3748 Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
3755 Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
3762 Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
3769 Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
3776 Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
3783 Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
3790 Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
3797 Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
3804 Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
3811 Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
3818 Opc = NVPTX::SULD_2D_I8_CLAMP;
3825 Opc = NVPTX::SULD_2D_I16_CLAMP;
3832 Opc = NVPTX::SULD_2D_I32_CLAMP;
3839 Opc = NVPTX::SULD_2D_I64_CLAMP;
3846 Opc = NVPTX::SULD_2D_V2I8_CLAMP;
3853 Opc = NVPTX::SULD_2D_V2I16_CLAMP;
3860 Opc = NVPTX::SULD_2D_V2I32_CLAMP;
3867 Opc = NVPTX::SULD_2D_V2I64_CLAMP;
3874 Opc = NVPTX::SULD_2D_V4I8_CLAMP;
3881 Opc = NVPTX::SULD_2D_V4I16_CLAMP;
3888 Opc = NVPTX::SULD_2D_V4I32_CLAMP;
3895 Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
3903 Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
3911 Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
3919 Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
3927 Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
3935 Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
3943 Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
3951 Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
3959 Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
3967 Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
3975 Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
3983 Opc = NVPTX::SULD_3D_I8_CLAMP;
3991 Opc = NVPTX::SULD_3D_I16_CLAMP;
3999 Opc = NVPTX::SULD_3D_I32_CLAMP;
4007 Opc = NVPTX::SULD_3D_I64_CLAMP;
4015 Opc = NVPTX::SULD_3D_V2I8_CLAMP;
4023 Opc = NVPTX::SULD_3D_V2I16_CLAMP;
4031 Opc = NVPTX::SULD_3D_V2I32_CLAMP;
4039 Opc = NVPTX::SULD_3D_V2I64_CLAMP;
4047 Opc = NVPTX::SULD_3D_V4I8_CLAMP;
4055 Opc = NVPTX::SULD_3D_V4I16_CLAMP;
4063 Opc = NVPTX::SULD_3D_V4I32_CLAMP;
4071 Opc = NVPTX::SULD_1D_I8_TRAP;
4077 Opc = NVPTX::SULD_1D_I16_TRAP;
4083 Opc = NVPTX::SULD_1D_I32_TRAP;
4089 Opc = NVPTX::SULD_1D_I64_TRAP;
4095 Opc = NVPTX::SULD_1D_V2I8_TRAP;
4101 Opc = NVPTX::SULD_1D_V2I16_TRAP;
4107 Opc = NVPTX::SULD_1D_V2I32_TRAP;
4113 Opc = NVPTX::SULD_1D_V2I64_TRAP;
4119 Opc = NVPTX::SULD_1D_V4I8_TRAP;
4125 Opc = NVPTX::SULD_1D_V4I16_TRAP;
4131 Opc = NVPTX::SULD_1D_V4I32_TRAP;
4137 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
4144 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
4151 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
4158 Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
4165 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
4172 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
4179 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
4186 Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
4193 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
4200 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
4207 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
4214 Opc = NVPTX::SULD_2D_I8_TRAP;
4221 Opc = NVPTX::SULD_2D_I16_TRAP;
4228 Opc = NVPTX::SULD_2D_I32_TRAP;
4235 Opc = NVPTX::SULD_2D_I64_TRAP;
4242 Opc = NVPTX::SULD_2D_V2I8_TRAP;
4249 Opc = NVPTX::SULD_2D_V2I16_TRAP;
4256 Opc = NVPTX::SULD_2D_V2I32_TRAP;
4263 Opc = NVPTX::SULD_2D_V2I64_TRAP;
4270 Opc = NVPTX::SULD_2D_V4I8_TRAP;
4277 Opc = NVPTX::SULD_2D_V4I16_TRAP;
4284 Opc = NVPTX::SULD_2D_V4I32_TRAP;
4291 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
4299 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
4307 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
4315 Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
4323 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
4331 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
4339 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
4347 Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
4355 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
4363 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
4371 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
4379 Opc = NVPTX::SULD_3D_I8_TRAP;
4387 Opc = NVPTX::SULD_3D_I16_TRAP;
4395 Opc = NVPTX::SULD_3D_I32_TRAP;
4403 Opc = NVPTX::SULD_3D_I64_TRAP;
4411 Opc = NVPTX::SULD_3D_V2I8_TRAP;
4419 Opc = NVPTX::SULD_3D_V2I16_TRAP;
4427 Opc = NVPTX::SULD_3D_V2I32_TRAP;
4435 Opc = NVPTX::SULD_3D_V2I64_TRAP;
4443 Opc = NVPTX::SULD_3D_V4I8_TRAP;
4451 Opc = NVPTX::SULD_3D_V4I16_TRAP;
4459 Opc = NVPTX::SULD_3D_V4I32_TRAP;
4467 Opc = NVPTX::SULD_1D_I8_ZERO;
4473 Opc = NVPTX::SULD_1D_I16_ZERO;
4479 Opc = NVPTX::SULD_1D_I32_ZERO;
4485 Opc = NVPTX::SULD_1D_I64_ZERO;
4491 Opc = NVPTX::SULD_1D_V2I8_ZERO;
4497 Opc = NVPTX::SULD_1D_V2I16_ZERO;
4503 Opc = NVPTX::SULD_1D_V2I32_ZERO;
4509 Opc = NVPTX::SULD_1D_V2I64_ZERO;
4515 Opc = NVPTX::SULD_1D_V4I8_ZERO;
4521 Opc = NVPTX::SULD_1D_V4I16_ZERO;
4527 Opc = NVPTX::SULD_1D_V4I32_ZERO;
4533 Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
4540 Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
4547 Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
4554 Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
4561 Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
4568 Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
4575 Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
4582 Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
4589 Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
4596 Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
4603 Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
4610 Opc = NVPTX::SULD_2D_I8_ZERO;
4617 Opc = NVPTX::SULD_2D_I16_ZERO;
4624 Opc = NVPTX::SULD_2D_I32_ZERO;
4631 Opc = NVPTX::SULD_2D_I64_ZERO;
4638 Opc = NVPTX::SULD_2D_V2I8_ZERO;
4645 Opc = NVPTX::SULD_2D_V2I16_ZERO;
4652 Opc = NVPTX::SULD_2D_V2I32_ZERO;
4659 Opc = NVPTX::SULD_2D_V2I64_ZERO;
4666 Opc = NVPTX::SULD_2D_V4I8_ZERO;
4673 Opc = NVPTX::SULD_2D_V4I16_ZERO;
4680 Opc = NVPTX::SULD_2D_V4I32_ZERO;
4687 Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
4695 Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
4703 Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
4711 Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
4719 Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
4727 Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
4735 Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
4743 Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
4751 Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
4759 Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
4767 Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
4775 Opc = NVPTX::SULD_3D_I8_ZERO;
4783 Opc = NVPTX::SULD_3D_I16_ZERO;
4791 Opc = NVPTX::SULD_3D_I32_ZERO;
4799 Opc = NVPTX::SULD_3D_I64_ZERO;
4807 Opc = NVPTX::SULD_3D_V2I8_ZERO;
4815 Opc = NVPTX::SULD_3D_V2I16_ZERO;
4823 Opc = NVPTX::SULD_3D_V2I32_ZERO;
4831 Opc = NVPTX::SULD_3D_V2I64_ZERO;
4839 Opc = NVPTX::SULD_3D_V4I8_ZERO;
4847 Opc = NVPTX::SULD_3D_V4I16_ZERO;
4855 Opc = NVPTX::SULD_3D_V4I32_ZERO;
4870 bool NVPTXDAGToDAGISel::tryBFE(
SDNode *N) {
4877 bool IsSigned =
false;
4882 if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
4915 if (NumBits > GoodBits) {
4949 if (isa<ConstantSDNode>(AndLHS)) {
4973 NumBits = NumZeros + NumOnes - ShiftAmt;
4979 if (ShiftAmt < NumZeros) {
5015 if (OuterShiftAmt < InnerShiftAmt) {
5051 Opc = NVPTX::BFE_S32rii;
5053 Opc = NVPTX::BFE_U32rii;
5057 Opc = NVPTX::BFE_S64rii;
5059 Opc = NVPTX::BFE_U64rii;
5098 bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
5103 if (SelectDirectAddr(base, Base)) {
5114 bool NVPTXDAGToDAGISel::SelectADDRsi(
SDNode *OpNode,
SDValue Addr,
5116 return SelectADDRsi_imp(OpNode, Addr, Base, Offset,
MVT::i32);
5120 bool NVPTXDAGToDAGISel::SelectADDRsi64(
SDNode *OpNode,
SDValue Addr,
5122 return SelectADDRsi_imp(OpNode, Addr, Base, Offset,
MVT::i64);
5126 bool NVPTXDAGToDAGISel::SelectADDRri_imp(
5138 if (SelectDirectAddr(Addr.
getOperand(0), Addr)) {
5143 dyn_cast<FrameIndexSDNode>(Addr.
getOperand(0)))
5157 bool NVPTXDAGToDAGISel::SelectADDRri(
SDNode *OpNode,
SDValue Addr,
5159 return SelectADDRri_imp(OpNode, Addr, Base, Offset,
MVT::i32);
5163 bool NVPTXDAGToDAGISel::SelectADDRri64(
SDNode *OpNode,
SDValue Addr,
5165 return SelectADDRri_imp(OpNode, Addr, Base, Offset,
MVT::i64);
5168 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(
SDNode *N,
5169 unsigned int spN)
const {
5170 const Value *Src =
nullptr;
5171 if (
MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
5172 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
5174 Src = mN->getMemOperand()->getValue();
5178 if (
auto *PT = dyn_cast<PointerType>(Src->
getType()))
5179 return (PT->getAddressSpace() == spN);
5186 const SDValue &
Op,
unsigned ConstraintID, std::vector<SDValue> &OutOps) {
5188 switch (ConstraintID) {
5192 if (SelectDirectAddr(Op, Op0)) {
5193 OutOps.push_back(Op0);
5197 if (SelectADDRri(Op.
getNode(),
Op, Op0, Op1)) {
5198 OutOps.push_back(Op0);
5199 OutOps.push_back(Op1);
5209 unsigned NVPTXDAGToDAGISel::GetConvertOpcode(
MVT DestTy,
MVT SrcTy,
5219 return IsSigned ? NVPTX::CVT_s16_s8 : NVPTX::CVT_u16_u8;
5221 return IsSigned ? NVPTX::CVT_s32_s8 : NVPTX::CVT_u32_u8;
5223 return IsSigned ? NVPTX::CVT_s64_s8 : NVPTX::CVT_u64_u8;
5230 return IsSigned ? NVPTX::CVT_s8_s16 : NVPTX::CVT_u8_u16;
5232 return IsSigned ? NVPTX::CVT_s32_s16 : NVPTX::CVT_u32_u16;
5234 return IsSigned ? NVPTX::CVT_s64_s16 : NVPTX::CVT_u64_u16;
5241 return IsSigned ? NVPTX::CVT_s8_s32 : NVPTX::CVT_u8_u32;
5243 return IsSigned ? NVPTX::CVT_s16_s32 : NVPTX::CVT_u16_u32;
5245 return IsSigned ? NVPTX::CVT_s64_s32 : NVPTX::CVT_u64_u32;
5252 return IsSigned ? NVPTX::CVT_s8_s64 : NVPTX::CVT_u8_u64;
5254 return IsSigned ? NVPTX::CVT_s16_s64 : NVPTX::CVT_u16_u64;
5256 return IsSigned ? NVPTX::CVT_s32_s64 : NVPTX::CVT_u32_u64;
void push_back(const T &Elt)
LLVM Argument representation.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
unsigned getDestAddressSpace() const
SDVTList getVTList() const
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
unsigned getSrcAddressSpace() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getSizeInBits() const
static unsigned int getCodeAddrSpace(MemSDNode *N)
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
void GetUnderlyingObjects(Value *V, SmallVectorImpl< Value * > &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
void setNodeId(int Id)
Set unique node id.
const NVPTXSubtarget * Subtarget
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, CodeGenOpt::Level OptLevel)
bool isVector() const
isVector - Return true if this is a vector value type.
A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Shift and rotation operations.
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
MVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
bool isKernelFunction(const Function &F)
constexpr bool isMask_64(uint64_t Value)
isMask_64 - This function returns true if the argument is a non-empty sequence of ones starting at th...
SDValue getTargetFrameIndex(int FI, EVT VT)
Simple integer binary arithmetic operators.
bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Implement addressing mode selection for inline asm expressions...
EVT getMemoryVT() const
Return the type of the in-memory value.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
This class is used to represent ISD::STORE nodes.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
SDNode * getNode() const
get the SDNode which holds the desired result
initializer< Ty > init(const Ty &Val)
CodeGenOpt::Level OptLevel
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
bool allowFMA(MachineFunction &MF, CodeGenOpt::Level OptLevel) const
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
unsigned getVectorNumElements() const
MVT - Machine Value Type.
const SDValue & getOperand(unsigned i) const
bool isVector() const
isVector - Return true if this is a vector value type.
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
const NVPTXTargetLowering * getTargetLowering() const override
static cl::opt< bool > UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden, cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), cl::init(true))
static bool canLowerToLDG(MemSDNode *N, const NVPTXSubtarget &Subtarget, unsigned CodeAddrSpace, MachineFunction *F)
static ManagedStatic< std::set< EVT, EVT::compareRawBits > > EVTs
unsigned getOpcode() const
FunctionPass class - This class is used to implement most global optimizations.
EVT - Extended Value Type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Type * getType() const
All values are typed, get the type of this value.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
This is an abstract virtual class for memory operations.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
const Value * getValue() const
Return the base address of the memory access.
static cl::opt< bool > FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden, cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."), cl::init(false))
static cl::opt< int > UsePrecDivF32("nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden, cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"" IEEE Compliant F32 div.rnd if available."), cl::init(2))
constexpr bool isShiftedMask_64(uint64_t Value)
isShiftedMask_64 - This function returns true if the argument contains a non-empty sequence of ones w...
FunctionPass * createNVPTXISelDag(NVPTXTargetMachine &TM, llvm::CodeGenOpt::Level OptLevel)
createNVPTXISelDag - This pass converts a legalized DAG into a NVPTX-specific DAG, ready for instruction scheduling.
Bitwise operators - logical and, logical or, logical xor.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
EVT getValueType() const
Return the ValueType of the referenced return value.
StringRef getValueAsString() const
Return the attribute's value as a string.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
static bool isVolatile(Instruction *Inst)
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num)
allocateMemRefsArray - Allocate an array to hold MachineMemOperand pointers.
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
uint64_t getZExtValue() const
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
This class is used to represent ISD::LOAD nodes.