20#include "llvm/IR/IntrinsicsNVPTX.h"
31#define DEBUG_TYPE "nvptx-isel"
32#define PASS_NAME "NVPTX DAG->DAG Pattern Instruction Selection"
36 cl::desc(
"Enable reciprocal sqrt optimization"));
65int NVPTXDAGToDAGISel::getDivF32Level()
const {
69bool NVPTXDAGToDAGISel::usePrecSqrtF32()
const {
73bool NVPTXDAGToDAGISel::useF32FTZ()
const {
77bool NVPTXDAGToDAGISel::allowFMA()
const {
82bool NVPTXDAGToDAGISel::allowUnsafeFPMath()
const {
87bool NVPTXDAGToDAGISel::doRsqrtOpt()
const {
return EnableRsqrtOpt; }
91void NVPTXDAGToDAGISel::Select(
SDNode *
N) {
93 if (
N->isMachineOpcode()) {
98 switch (
N->getOpcode()) {
110 if (tryEXTRACT_VECTOR_ELEMENT(
N))
117 SelectSETP_BF16X2(
N);
121 if (tryLoadVector(
N))
133 if (tryStoreVector(
N))
145 if (tryStoreRetval(
N))
153 if (tryStoreParam(
N))
157 if (tryIntrinsicNoChain(
N))
161 if (tryIntrinsicChain(
N))
338 if (tryTextureIntrinsic(
N))
506 if (trySurfaceIntrinsic(
N))
517 SelectAddrSpaceCast(
N);
520 if (tryConstantFP(
N))
524 if (
N->getOperand(1).getValueType() == MVT::i128) {
525 SelectV2I64toI128(
N);
531 if (
N->getOperand(1).getValueType() == MVT::i128) {
532 SelectI128toV2I64(
N);
543bool NVPTXDAGToDAGISel::tryIntrinsicChain(
SDNode *
N) {
544 unsigned IID =
N->getConstantOperandVal(1);
548 case Intrinsic::nvvm_ldg_global_f:
549 case Intrinsic::nvvm_ldg_global_i:
550 case Intrinsic::nvvm_ldg_global_p:
551 case Intrinsic::nvvm_ldu_global_f:
552 case Intrinsic::nvvm_ldu_global_i:
553 case Intrinsic::nvvm_ldu_global_p:
560bool NVPTXDAGToDAGISel::tryConstantFP(
SDNode *
N) {
561 if (
N->getValueType(0) != MVT::f16 &&
N->getValueType(0) != MVT::bf16)
564 cast<ConstantFPSDNode>(
N)->getValueAPF(),
SDLoc(
N),
N->getValueType(0));
566 (
N->getValueType(0) == MVT::f16 ? NVPTX::LOAD_CONST_F16
567 : NVPTX::LOAD_CONST_BF16),
568 SDLoc(
N),
N->getValueType(0), Val);
596 return CmpMode::NotANumber;
630bool NVPTXDAGToDAGISel::SelectSETP_F16X2(
SDNode *
N) {
631 unsigned PTXCmpMode =
632 getPTXCmpMode(*cast<CondCodeSDNode>(
N->getOperand(2)), useF32FTZ());
635 NVPTX::SETP_f16x2rr,
DL, MVT::i1, MVT::i1,
N->getOperand(0),
641bool NVPTXDAGToDAGISel::SelectSETP_BF16X2(
SDNode *
N) {
642 unsigned PTXCmpMode =
643 getPTXCmpMode(*cast<CondCodeSDNode>(
N->getOperand(2)), useF32FTZ());
646 NVPTX::SETP_bf16x2rr,
DL, MVT::i1, MVT::i1,
N->getOperand(0),
654bool NVPTXDAGToDAGISel::tryEXTRACT_VECTOR_ELEMENT(
SDNode *
N) {
664 for (
auto *U :
Vector.getNode()->uses()) {
667 if (
U->getOperand(0) !=
Vector)
670 dyn_cast<ConstantSDNode>(
U->getOperand(1))) {
671 if (IdxConst->getZExtValue() == 0)
673 else if (IdxConst->getZExtValue() == 1)
690 for (
auto *
Node : E0)
692 for (
auto *
Node : E1)
699 const Value *Src =
N->getMemOperand()->getValue();
704 if (
auto *PT = dyn_cast<PointerType>(Src->getType())) {
705 switch (PT->getAddressSpace()) {
720struct OperationOrderings {
727static OperationOrderings
829 !HasMemoryOrdering) {
831 formatv(
"PTX does not support \"atomic\" for orderings different than"
832 "\"NotAtomic\" or \"Monotonic\" for sm_60 or older, but order "
844 bool AddrGenericOrGlobalOrShared =
848 if (!AddrGenericOrGlobalOrShared)
851 bool UseRelaxedMMIO =
873 formatv(
"PTX only supports Acquire Ordering on reads: {}",
874 N->getOperationName()));
879 formatv(
"PTX only supports Release Ordering on writes: {}",
880 N->getOperationName()));
884 formatv(
"NVPTX does not support AcquireRelease Ordering on "
886 "yet and PTX does not support it on loads or stores: {}",
887 N->getOperationName()));
900 else if (
N->writeMem())
904 formatv(
"NVPTX does not support SequentiallyConsistent Ordering on "
905 "read-modify-writes yet: {}",
906 N->getOperationName()));
907 return OperationOrderings(InstrOrder,
912 formatv(
"NVPTX backend does not support AtomicOrdering \"{}\" yet.",
940 if (
N->isInvariant())
952 if (
auto *
A = dyn_cast<const Argument>(V))
953 return IsKernelFn &&
A->onlyReadsMemory() &&
A->hasNoAliasAttr();
954 if (
auto *GV = dyn_cast<const GlobalVariable>(V))
955 return GV->isConstant();
975 ? NVPTX::atomic_thread_fence_seq_cst_sys
976 : NVPTX::INT_MEMBAR_SYS;
982 formatv(
"Unexpected fence ordering: \"{}\".",
989bool NVPTXDAGToDAGISel::tryIntrinsicNoChain(
SDNode *
N) {
990 unsigned IID =
N->getConstantOperandVal(0);
994 case Intrinsic::nvvm_texsurf_handle_internal:
995 SelectTexSurfHandle(
N);
1000void NVPTXDAGToDAGISel::SelectTexSurfHandle(
SDNode *
N) {
1005 MVT::i64, GlobalVal));
1008void NVPTXDAGToDAGISel::SelectAddrSpaceCast(
SDNode *
N) {
1013 assert(SrcAddrSpace != DstAddrSpace &&
1014 "addrspacecast must be between different address spaces");
1019 switch (SrcAddrSpace) {
1022 Opc = TM.
is64Bit() ? NVPTX::cvta_global_64 : NVPTX::cvta_global;
1026 ? NVPTX::cvta_shared_6432
1027 : NVPTX::cvta_shared_64)
1028 : NVPTX::cvta_shared;
1032 ? NVPTX::cvta_const_6432
1033 : NVPTX::cvta_const_64)
1034 : NVPTX::cvta_const;
1038 ? NVPTX::cvta_local_6432
1039 : NVPTX::cvta_local_64)
1040 : NVPTX::cvta_local;
1048 if (SrcAddrSpace != 0)
1051 switch (DstAddrSpace) {
1054 Opc = TM.
is64Bit() ? NVPTX::cvta_to_global_64 : NVPTX::cvta_to_global;
1058 ? NVPTX::cvta_to_shared_3264
1059 : NVPTX::cvta_to_shared_64)
1060 : NVPTX::cvta_to_shared;
1064 ? NVPTX::cvta_to_const_3264
1065 : NVPTX::cvta_to_const_64)
1066 : NVPTX::cvta_to_const;
1070 ? NVPTX::cvta_to_local_3264
1071 : NVPTX::cvta_to_local_64)
1072 : NVPTX::cvta_to_local;
1075 Opc = TM.
is64Bit() ? NVPTX::nvvm_ptr_gen_to_param_64
1076 : NVPTX::nvvm_ptr_gen_to_param;
1087static std::optional<unsigned>
1089 unsigned Opcode_i16,
unsigned Opcode_i32,
1090 std::optional<unsigned> Opcode_i64,
unsigned Opcode_f32,
1091 std::optional<unsigned> Opcode_f64) {
1115 return std::nullopt;
1134bool NVPTXDAGToDAGISel::tryLoad(
SDNode *
N) {
1136 assert(
LD->readMem() &&
"Expected load");
1140 if (PlainLoad && PlainLoad->
isIndexed())
1143 EVT LoadedVT =
LD->getMemoryVT();
1150 return tryLDGLDU(
N);
1168 unsigned FromTypeWidth = std::max(8U, (
unsigned)ScalarVT.
getSizeInBits());
1175 "Unexpected vector type");
1189 std::optional<unsigned> Opcode;
1193 getI32Imm(CodeAddrSpace,
DL),
1194 getI32Imm(VecType,
DL), getI32Imm(FromType,
DL),
1195 getI32Imm(FromTypeWidth,
DL)});
1197 if (SelectDirectAddr(N1,
Addr)) {
1198 Opcode =
pickOpcodeForVT(TargetVT, NVPTX::LD_i8_avar, NVPTX::LD_i16_avar,
1199 NVPTX::LD_i32_avar, NVPTX::LD_i64_avar,
1200 NVPTX::LD_f32_avar, NVPTX::LD_f64_avar);
1203 Ops.append({
Addr, Chain});
1206 Opcode =
pickOpcodeForVT(TargetVT, NVPTX::LD_i8_asi, NVPTX::LD_i16_asi,
1207 NVPTX::LD_i32_asi, NVPTX::LD_i64_asi,
1208 NVPTX::LD_f32_asi, NVPTX::LD_f64_asi);
1214 if (PointerSize == 64)
1217 NVPTX::LD_i32_ari_64, NVPTX::LD_i64_ari_64,
1218 NVPTX::LD_f32_ari_64, NVPTX::LD_f64_ari_64);
1220 Opcode =
pickOpcodeForVT(TargetVT, NVPTX::LD_i8_ari, NVPTX::LD_i16_ari,
1221 NVPTX::LD_i32_ari, NVPTX::LD_i64_ari,
1222 NVPTX::LD_f32_ari, NVPTX::LD_f64_ari);
1227 if (PointerSize == 64)
1229 pickOpcodeForVT(TargetVT, NVPTX::LD_i8_areg_64, NVPTX::LD_i16_areg_64,
1230 NVPTX::LD_i32_areg_64, NVPTX::LD_i64_areg_64,
1231 NVPTX::LD_f32_areg_64, NVPTX::LD_f64_areg_64);
1233 Opcode =
pickOpcodeForVT(TargetVT, NVPTX::LD_i8_areg, NVPTX::LD_i16_areg,
1234 NVPTX::LD_i32_areg, NVPTX::LD_i64_areg,
1235 NVPTX::LD_f32_areg, NVPTX::LD_f64_areg);
1238 Ops.append({N1, Chain});
1253bool NVPTXDAGToDAGISel::tryLoadVector(
SDNode *
N) {
1262 return tryLDGLDU(
N);
1282 unsigned FromTypeWidth = std::max(8U, (
unsigned)ScalarVT.
getSizeInBits());
1285 unsigned ExtensionType = cast<ConstantSDNode>(
1286 N->getOperand(
N->getNumOperands() - 1))->getZExtValue();
1294 switch (
N->getOpcode()) {
1305 EVT EltVT =
N->getValueType(0);
1319 std::optional<unsigned> Opcode;
1323 getI32Imm(CodeAddrSpace,
DL),
1324 getI32Imm(VecType,
DL), getI32Imm(FromType,
DL),
1325 getI32Imm(FromTypeWidth,
DL)});
1327 if (SelectDirectAddr(Op1,
Addr)) {
1328 switch (
N->getOpcode()) {
1333 NVPTX::LDV_i8_v2_avar, NVPTX::LDV_i16_v2_avar,
1334 NVPTX::LDV_i32_v2_avar, NVPTX::LDV_i64_v2_avar,
1335 NVPTX::LDV_f32_v2_avar, NVPTX::LDV_f64_v2_avar);
1340 NVPTX::LDV_i16_v4_avar, NVPTX::LDV_i32_v4_avar,
1341 std::nullopt, NVPTX::LDV_f32_v4_avar, std::nullopt);
1346 Ops.append({
Addr, Chain});
1347 }
else if (PointerSize == 64
1350 switch (
N->getOpcode()) {
1355 NVPTX::LDV_i8_v2_asi, NVPTX::LDV_i16_v2_asi,
1356 NVPTX::LDV_i32_v2_asi, NVPTX::LDV_i64_v2_asi,
1357 NVPTX::LDV_f32_v2_asi, NVPTX::LDV_f64_v2_asi);
1362 NVPTX::LDV_i16_v4_asi, NVPTX::LDV_i32_v4_asi,
1363 std::nullopt, NVPTX::LDV_f32_v4_asi, std::nullopt);
1369 }
else if (PointerSize == 64
1372 if (PointerSize == 64) {
1373 switch (
N->getOpcode()) {
1379 NVPTX::LDV_i8_v2_ari_64, NVPTX::LDV_i16_v2_ari_64,
1380 NVPTX::LDV_i32_v2_ari_64, NVPTX::LDV_i64_v2_ari_64,
1381 NVPTX::LDV_f32_v2_ari_64, NVPTX::LDV_f64_v2_ari_64);
1386 NVPTX::LDV_i16_v4_ari_64, NVPTX::LDV_i32_v4_ari_64, std::nullopt,
1387 NVPTX::LDV_f32_v4_ari_64, std::nullopt);
1391 switch (
N->getOpcode()) {
1396 NVPTX::LDV_i8_v2_ari, NVPTX::LDV_i16_v2_ari,
1397 NVPTX::LDV_i32_v2_ari, NVPTX::LDV_i64_v2_ari,
1398 NVPTX::LDV_f32_v2_ari, NVPTX::LDV_f64_v2_ari);
1403 NVPTX::LDV_i16_v4_ari, NVPTX::LDV_i32_v4_ari,
1404 std::nullopt, NVPTX::LDV_f32_v4_ari, std::nullopt);
1412 if (PointerSize == 64) {
1413 switch (
N->getOpcode()) {
1419 NVPTX::LDV_i16_v2_areg_64, NVPTX::LDV_i32_v2_areg_64,
1420 NVPTX::LDV_i64_v2_areg_64, NVPTX::LDV_f32_v2_areg_64,
1421 NVPTX::LDV_f64_v2_areg_64);
1426 NVPTX::LDV_i16_v4_areg_64, NVPTX::LDV_i32_v4_areg_64, std::nullopt,
1427 NVPTX::LDV_f32_v4_areg_64, std::nullopt);
1431 switch (
N->getOpcode()) {
1437 NVPTX::LDV_i16_v2_areg, NVPTX::LDV_i32_v2_areg,
1438 NVPTX::LDV_i64_v2_areg, NVPTX::LDV_f32_v2_areg,
1439 NVPTX::LDV_f64_v2_areg);
1444 NVPTX::LDV_i16_v4_areg, NVPTX::LDV_i32_v4_areg,
1445 std::nullopt, NVPTX::LDV_f32_v4_areg, std::nullopt);
1451 Ops.append({Op1, Chain});
1462bool NVPTXDAGToDAGISel::tryLDGLDU(
SDNode *
N) {
1470 Op1 =
N->getOperand(2);
1471 Mem = cast<MemIntrinsicSDNode>(
N);
1472 unsigned IID =
N->getConstantOperandVal(1);
1476 case Intrinsic::nvvm_ldg_global_f:
1477 case Intrinsic::nvvm_ldg_global_i:
1478 case Intrinsic::nvvm_ldg_global_p:
1481 case Intrinsic::nvvm_ldu_global_f:
1482 case Intrinsic::nvvm_ldu_global_i:
1483 case Intrinsic::nvvm_ldu_global_p:
1488 Op1 =
N->getOperand(1);
1489 Mem = cast<MemSDNode>(
N);
1492 EVT OrigType =
N->getValueType(0);
1494 unsigned NumElts = 1;
1499 if ((EltVT == MVT::f16 && OrigType == MVT::v2f16) ||
1500 (EltVT == MVT::bf16 && OrigType == MVT::v2bf16) ||
1501 (EltVT == MVT::i16 && OrigType == MVT::v2i16)) {
1502 assert(NumElts % 2 == 0 &&
"Vector must have even number of elements");
1505 }
else if (OrigType == MVT::v4i8) {
1514 EVT NodeVT = (EltVT == MVT::i8) ? MVT::i16 : EltVT;
1516 for (
unsigned i = 0; i != NumElts; ++i) {
1523 std::optional<unsigned> Opcode;
1528 if (SelectDirectAddr(Op1,
Addr)) {
1529 switch (
N->getOpcode()) {
1536 NVPTX::INT_PTX_LDG_GLOBAL_i8avar,
1537 NVPTX::INT_PTX_LDG_GLOBAL_i16avar,
1538 NVPTX::INT_PTX_LDG_GLOBAL_i32avar,
1539 NVPTX::INT_PTX_LDG_GLOBAL_i64avar,
1540 NVPTX::INT_PTX_LDG_GLOBAL_f32avar,
1541 NVPTX::INT_PTX_LDG_GLOBAL_f64avar);
1544 NVPTX::INT_PTX_LDU_GLOBAL_i8avar,
1545 NVPTX::INT_PTX_LDU_GLOBAL_i16avar,
1546 NVPTX::INT_PTX_LDU_GLOBAL_i32avar,
1547 NVPTX::INT_PTX_LDU_GLOBAL_i64avar,
1548 NVPTX::INT_PTX_LDU_GLOBAL_f32avar,
1549 NVPTX::INT_PTX_LDU_GLOBAL_f64avar);
1554 NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar,
1555 NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar,
1556 NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar,
1557 NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar,
1558 NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar,
1559 NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar);
1563 NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar,
1564 NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar,
1565 NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar,
1566 NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar,
1567 NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar,
1568 NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar);
1574 NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar,
1575 NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar, std::nullopt,
1576 NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar, std::nullopt);
1581 NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar,
1582 NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar, std::nullopt,
1583 NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar, std::nullopt);
1593 switch (
N->getOpcode()) {
1600 NVPTX::INT_PTX_LDG_GLOBAL_i8ari64,
1601 NVPTX::INT_PTX_LDG_GLOBAL_i16ari64,
1602 NVPTX::INT_PTX_LDG_GLOBAL_i32ari64,
1603 NVPTX::INT_PTX_LDG_GLOBAL_i64ari64,
1604 NVPTX::INT_PTX_LDG_GLOBAL_f32ari64,
1605 NVPTX::INT_PTX_LDG_GLOBAL_f64ari64);
1608 NVPTX::INT_PTX_LDU_GLOBAL_i8ari64,
1609 NVPTX::INT_PTX_LDU_GLOBAL_i16ari64,
1610 NVPTX::INT_PTX_LDU_GLOBAL_i32ari64,
1611 NVPTX::INT_PTX_LDU_GLOBAL_i64ari64,
1612 NVPTX::INT_PTX_LDU_GLOBAL_f32ari64,
1613 NVPTX::INT_PTX_LDU_GLOBAL_f64ari64);
1618 NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64,
1619 NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64,
1620 NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64,
1621 NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64,
1622 NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64,
1623 NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64);
1627 NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64,
1628 NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64,
1629 NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64,
1630 NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64,
1631 NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64,
1632 NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64);
1638 NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64,
1639 NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64, std::nullopt,
1640 NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64, std::nullopt);
1645 NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64,
1646 NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64, std::nullopt,
1647 NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64, std::nullopt);
1651 switch (
N->getOpcode()) {
1658 NVPTX::INT_PTX_LDG_GLOBAL_i8ari,
1659 NVPTX::INT_PTX_LDG_GLOBAL_i16ari,
1660 NVPTX::INT_PTX_LDG_GLOBAL_i32ari,
1661 NVPTX::INT_PTX_LDG_GLOBAL_i64ari,
1662 NVPTX::INT_PTX_LDG_GLOBAL_f32ari,
1663 NVPTX::INT_PTX_LDG_GLOBAL_f64ari);
1666 NVPTX::INT_PTX_LDU_GLOBAL_i8ari,
1667 NVPTX::INT_PTX_LDU_GLOBAL_i16ari,
1668 NVPTX::INT_PTX_LDU_GLOBAL_i32ari,
1669 NVPTX::INT_PTX_LDU_GLOBAL_i64ari,
1670 NVPTX::INT_PTX_LDU_GLOBAL_f32ari,
1671 NVPTX::INT_PTX_LDU_GLOBAL_f64ari);
1676 NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32,
1677 NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32,
1678 NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32,
1679 NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32,
1680 NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32,
1681 NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32);
1685 NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32,
1686 NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32,
1687 NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32,
1688 NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32,
1689 NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32,
1690 NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32);
1696 NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32,
1697 NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32, std::nullopt,
1698 NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32, std::nullopt);
1703 NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32,
1704 NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32, std::nullopt,
1705 NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32, std::nullopt);
1715 switch (
N->getOpcode()) {
1722 NVPTX::INT_PTX_LDG_GLOBAL_i8areg64,
1723 NVPTX::INT_PTX_LDG_GLOBAL_i16areg64,
1724 NVPTX::INT_PTX_LDG_GLOBAL_i32areg64,
1725 NVPTX::INT_PTX_LDG_GLOBAL_i64areg64,
1726 NVPTX::INT_PTX_LDG_GLOBAL_f32areg64,
1727 NVPTX::INT_PTX_LDG_GLOBAL_f64areg64);
1730 NVPTX::INT_PTX_LDU_GLOBAL_i8areg64,
1731 NVPTX::INT_PTX_LDU_GLOBAL_i16areg64,
1732 NVPTX::INT_PTX_LDU_GLOBAL_i32areg64,
1733 NVPTX::INT_PTX_LDU_GLOBAL_i64areg64,
1734 NVPTX::INT_PTX_LDU_GLOBAL_f32areg64,
1735 NVPTX::INT_PTX_LDU_GLOBAL_f64areg64);
1740 NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64,
1741 NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64,
1742 NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64,
1743 NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64,
1744 NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64,
1745 NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64);
1749 NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64,
1750 NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64,
1751 NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64,
1752 NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64,
1753 NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64,
1754 NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64);
1760 NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64,
1761 NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64, std::nullopt,
1762 NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64, std::nullopt);
1767 NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64,
1768 NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64, std::nullopt,
1769 NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64, std::nullopt);
1773 switch (
N->getOpcode()) {
1780 NVPTX::INT_PTX_LDG_GLOBAL_i8areg,
1781 NVPTX::INT_PTX_LDG_GLOBAL_i16areg,
1782 NVPTX::INT_PTX_LDG_GLOBAL_i32areg,
1783 NVPTX::INT_PTX_LDG_GLOBAL_i64areg,
1784 NVPTX::INT_PTX_LDG_GLOBAL_f32areg,
1785 NVPTX::INT_PTX_LDG_GLOBAL_f64areg);
1788 NVPTX::INT_PTX_LDU_GLOBAL_i8areg,
1789 NVPTX::INT_PTX_LDU_GLOBAL_i16areg,
1790 NVPTX::INT_PTX_LDU_GLOBAL_i32areg,
1791 NVPTX::INT_PTX_LDU_GLOBAL_i64areg,
1792 NVPTX::INT_PTX_LDU_GLOBAL_f32areg,
1793 NVPTX::INT_PTX_LDU_GLOBAL_f64areg);
1798 NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32,
1799 NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32,
1800 NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32,
1801 NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32,
1802 NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32,
1803 NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32);
1807 NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32,
1808 NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32,
1809 NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32,
1810 NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32,
1811 NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32,
1812 NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32);
1818 NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32,
1819 NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32, std::nullopt,
1820 NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32, std::nullopt);
1825 NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32,
1826 NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32, std::nullopt,
1827 NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32, std::nullopt);
1833 SDValue Ops[] = { Op1, Chain };
1850 if (OrigType != EltVT &&
1860 for (
unsigned i = 0; i != NumElts; ++i) {
1876bool NVPTXDAGToDAGISel::tryStore(
SDNode *
N) {
1878 assert(
ST->writeMem() &&
"Expected store");
1881 assert((PlainStore || AtomicStore) &&
"Expected store");
1884 if (PlainStore && PlainStore->
isIndexed())
1887 EVT StoreVT =
ST->getMemoryVT();
1910 "Unexpected vector type");
1922 std::optional<unsigned> Opcode;
1924 Value.getNode()->getSimpleValueType(0).SimpleTy;
1927 getI32Imm(CodeAddrSpace,
DL),
1928 getI32Imm(VecType,
DL), getI32Imm(ToType,
DL),
1929 getI32Imm(ToTypeWidth,
DL)});
1931 if (SelectDirectAddr(BasePtr,
Addr)) {
1932 Opcode =
pickOpcodeForVT(SourceVT, NVPTX::ST_i8_avar, NVPTX::ST_i16_avar,
1933 NVPTX::ST_i32_avar, NVPTX::ST_i64_avar,
1934 NVPTX::ST_f32_avar, NVPTX::ST_f64_avar);
1937 Ops.append({
Addr, Chain});
1938 }
else if (PointerSize == 64
1941 Opcode =
pickOpcodeForVT(SourceVT, NVPTX::ST_i8_asi, NVPTX::ST_i16_asi,
1942 NVPTX::ST_i32_asi, NVPTX::ST_i64_asi,
1943 NVPTX::ST_f32_asi, NVPTX::ST_f64_asi);
1947 }
else if (PointerSize == 64
1950 if (PointerSize == 64)
1953 NVPTX::ST_i32_ari_64, NVPTX::ST_i64_ari_64,
1954 NVPTX::ST_f32_ari_64, NVPTX::ST_f64_ari_64);
1956 Opcode =
pickOpcodeForVT(SourceVT, NVPTX::ST_i8_ari, NVPTX::ST_i16_ari,
1957 NVPTX::ST_i32_ari, NVPTX::ST_i64_ari,
1958 NVPTX::ST_f32_ari, NVPTX::ST_f64_ari);
1963 if (PointerSize == 64)
1965 pickOpcodeForVT(SourceVT, NVPTX::ST_i8_areg_64, NVPTX::ST_i16_areg_64,
1966 NVPTX::ST_i32_areg_64, NVPTX::ST_i64_areg_64,
1967 NVPTX::ST_f32_areg_64, NVPTX::ST_f64_areg_64);
1969 Opcode =
pickOpcodeForVT(SourceVT, NVPTX::ST_i8_areg, NVPTX::ST_i16_areg,
1970 NVPTX::ST_i32_areg, NVPTX::ST_i64_areg,
1971 NVPTX::ST_f32_areg, NVPTX::ST_f64_areg);
1977 SDNode *NVPTXST = NVPTXST =
1989bool NVPTXDAGToDAGISel::tryStoreVector(
SDNode *
N) {
1992 std::optional<unsigned> Opcode;
2022 switch (
N->getOpcode()) {
2025 Ops.
append({
N->getOperand(1),
N->getOperand(2)});
2026 N2 =
N->getOperand(3);
2030 Ops.
append({
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2032 N2 =
N->getOperand(5);
2049 getI32Imm(VecType,
DL), getI32Imm(ToType,
DL),
2050 getI32Imm(ToTypeWidth,
DL)});
2052 if (SelectDirectAddr(N2,
Addr)) {
2053 switch (
N->getOpcode()) {
2058 NVPTX::STV_i8_v2_avar, NVPTX::STV_i16_v2_avar,
2059 NVPTX::STV_i32_v2_avar, NVPTX::STV_i64_v2_avar,
2060 NVPTX::STV_f32_v2_avar, NVPTX::STV_f64_v2_avar);
2064 NVPTX::STV_i8_v4_avar, NVPTX::STV_i16_v4_avar,
2065 NVPTX::STV_i32_v4_avar, std::nullopt,
2066 NVPTX::STV_f32_v4_avar, std::nullopt);
2072 switch (
N->getOpcode()) {
2077 NVPTX::STV_i8_v2_asi, NVPTX::STV_i16_v2_asi,
2078 NVPTX::STV_i32_v2_asi, NVPTX::STV_i64_v2_asi,
2079 NVPTX::STV_f32_v2_asi, NVPTX::STV_f64_v2_asi);
2084 NVPTX::STV_i16_v4_asi, NVPTX::STV_i32_v4_asi,
2085 std::nullopt, NVPTX::STV_f32_v4_asi, std::nullopt);
2091 if (PointerSize == 64) {
2092 switch (
N->getOpcode()) {
2098 NVPTX::STV_i8_v2_ari_64, NVPTX::STV_i16_v2_ari_64,
2099 NVPTX::STV_i32_v2_ari_64, NVPTX::STV_i64_v2_ari_64,
2100 NVPTX::STV_f32_v2_ari_64, NVPTX::STV_f64_v2_ari_64);
2105 NVPTX::STV_i16_v4_ari_64, NVPTX::STV_i32_v4_ari_64, std::nullopt,
2106 NVPTX::STV_f32_v4_ari_64, std::nullopt);
2110 switch (
N->getOpcode()) {
2115 NVPTX::STV_i8_v2_ari, NVPTX::STV_i16_v2_ari,
2116 NVPTX::STV_i32_v2_ari, NVPTX::STV_i64_v2_ari,
2117 NVPTX::STV_f32_v2_ari, NVPTX::STV_f64_v2_ari);
2121 NVPTX::STV_i8_v4_ari, NVPTX::STV_i16_v4_ari,
2122 NVPTX::STV_i32_v4_ari, std::nullopt,
2123 NVPTX::STV_f32_v4_ari, std::nullopt);
2129 if (PointerSize == 64) {
2130 switch (
N->getOpcode()) {
2136 NVPTX::STV_i16_v2_areg_64, NVPTX::STV_i32_v2_areg_64,
2137 NVPTX::STV_i64_v2_areg_64, NVPTX::STV_f32_v2_areg_64,
2138 NVPTX::STV_f64_v2_areg_64);
2143 NVPTX::STV_i16_v4_areg_64, NVPTX::STV_i32_v4_areg_64, std::nullopt,
2144 NVPTX::STV_f32_v4_areg_64, std::nullopt);
2148 switch (
N->getOpcode()) {
2154 NVPTX::STV_i16_v2_areg, NVPTX::STV_i32_v2_areg,
2155 NVPTX::STV_i64_v2_areg, NVPTX::STV_f32_v2_areg,
2156 NVPTX::STV_f64_v2_areg);
2161 NVPTX::STV_i16_v4_areg, NVPTX::STV_i32_v4_areg,
2162 std::nullopt, NVPTX::STV_f32_v4_areg, std::nullopt);
2183bool NVPTXDAGToDAGISel::tryLoadParam(
SDNode *
Node) {
2191 switch (
Node->getOpcode()) {
2205 EVT EltVT =
Node->getValueType(0);
2208 std::optional<unsigned> Opcode;
2215 NVPTX::LoadParamMemI8, NVPTX::LoadParamMemI16,
2216 NVPTX::LoadParamMemI32, NVPTX::LoadParamMemI64,
2217 NVPTX::LoadParamMemF32, NVPTX::LoadParamMemF64);
2222 NVPTX::LoadParamMemV2I16, NVPTX::LoadParamMemV2I32,
2223 NVPTX::LoadParamMemV2I64, NVPTX::LoadParamMemV2F32,
2224 NVPTX::LoadParamMemV2F64);
2229 NVPTX::LoadParamMemV4I16, NVPTX::LoadParamMemV4I32,
2230 std::nullopt, NVPTX::LoadParamMemV4F32, std::nullopt);
2239 }
else if (VecSize == 2) {
2242 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
2246 unsigned OffsetVal =
Offset->getAsZExtVal();
2255bool NVPTXDAGToDAGISel::tryStoreRetval(
SDNode *
N) {
2259 unsigned OffsetVal =
Offset->getAsZExtVal();
2263 unsigned NumElts = 1;
2264 switch (
N->getOpcode()) {
2280 for (
unsigned i = 0; i < NumElts; ++i)
2287 std::optional<unsigned> Opcode = 0;
2293 NVPTX::StoreRetvalI8, NVPTX::StoreRetvalI16,
2294 NVPTX::StoreRetvalI32, NVPTX::StoreRetvalI64,
2295 NVPTX::StoreRetvalF32, NVPTX::StoreRetvalF64);
2296 if (Opcode == NVPTX::StoreRetvalI8) {
2300 switch (Ops[0].getSimpleValueType().SimpleTy) {
2304 Opcode = NVPTX::StoreRetvalI8TruncI32;
2307 Opcode = NVPTX::StoreRetvalI8TruncI64;
2314 NVPTX::StoreRetvalV2I8, NVPTX::StoreRetvalV2I16,
2315 NVPTX::StoreRetvalV2I32, NVPTX::StoreRetvalV2I64,
2316 NVPTX::StoreRetvalV2F32, NVPTX::StoreRetvalV2F64);
2320 NVPTX::StoreRetvalV4I8, NVPTX::StoreRetvalV4I16,
2321 NVPTX::StoreRetvalV4I32, std::nullopt,
2322 NVPTX::StoreRetvalV4F32, std::nullopt);
2337#define getOpcV2H(ty, opKind0, opKind1) \
2338 NVPTX::StoreParamV2##ty##_##opKind0##opKind1
2340#define getOpcV2H1(ty, opKind0, isImm1) \
2341 (isImm1) ? getOpcV2H(ty, opKind0, i) : getOpcV2H(ty, opKind0, r)
2343#define getOpcodeForVectorStParamV2(ty, isimm) \
2344 (isimm[0]) ? getOpcV2H1(ty, i, isimm[1]) : getOpcV2H1(ty, r, isimm[1])
2346#define getOpcV4H(ty, opKind0, opKind1, opKind2, opKind3) \
2347 NVPTX::StoreParamV4##ty##_##opKind0##opKind1##opKind2##opKind3
2349#define getOpcV4H3(ty, opKind0, opKind1, opKind2, isImm3) \
2350 (isImm3) ? getOpcV4H(ty, opKind0, opKind1, opKind2, i) \
2351 : getOpcV4H(ty, opKind0, opKind1, opKind2, r)
2353#define getOpcV4H2(ty, opKind0, opKind1, isImm2, isImm3) \
2354 (isImm2) ? getOpcV4H3(ty, opKind0, opKind1, i, isImm3) \
2355 : getOpcV4H3(ty, opKind0, opKind1, r, isImm3)
2357#define getOpcV4H1(ty, opKind0, isImm1, isImm2, isImm3) \
2358 (isImm1) ? getOpcV4H2(ty, opKind0, i, isImm2, isImm3) \
2359 : getOpcV4H2(ty, opKind0, r, isImm2, isImm3)
2361#define getOpcodeForVectorStParamV4(ty, isimm) \
2362 (isimm[0]) ? getOpcV4H1(ty, i, isimm[1], isimm[2], isimm[3]) \
2363 : getOpcV4H1(ty, r, isimm[1], isimm[2], isimm[3])
2365#define getOpcodeForVectorStParam(n, ty, isimm) \
2366 (n == 2) ? getOpcodeForVectorStParamV2(ty, isimm) \
2367 : getOpcodeForVectorStParamV4(ty, isimm)
2376 for (
unsigned i = 0; i < NumElts; i++) {
2377 IsImm[i] = (isa<ConstantSDNode>(Ops[i]) || isa<ConstantFPSDNode>(Ops[i]));
2380 if (MemTy == MVT::f32 || MemTy == MVT::f64) {
2402 assert(NumElts == 2 &&
"MVT too large for NumElts > 2");
2407 assert(NumElts == 2 &&
"MVT too large for NumElts > 2");
2413 return (NumElts == 2) ? NVPTX::StoreParamV2I8_rr
2414 : NVPTX::StoreParamV4I8_rrrr;
2417 return (NumElts == 2) ? NVPTX::StoreParamV2I16_rr
2418 : NVPTX::StoreParamV4I16_rrrr;
2423 return (NumElts == 2) ? NVPTX::StoreParamV2I32_rr
2424 : NVPTX::StoreParamV4I32_rrrr;
2430bool NVPTXDAGToDAGISel::tryStoreParam(
SDNode *
N) {
2434 unsigned ParamVal =
Param->getAsZExtVal();
2436 unsigned OffsetVal =
Offset->getAsZExtVal();
2438 SDValue Glue =
N->getOperand(
N->getNumOperands() - 1);
2442 switch (
N->getOpcode()) {
2460 for (
unsigned i = 0; i < NumElts; ++i)
2468 std::optional<unsigned> Opcode;
2469 switch (
N->getOpcode()) {
2477 if (MemTy != MVT::f16 && MemTy != MVT::v2f16 &&
2478 (isa<ConstantSDNode>(Imm) || isa<ConstantFPSDNode>(Imm))) {
2480 if (MemTy == MVT::f32 || MemTy == MVT::f64) {
2492 NVPTX::StoreParamI16_i, NVPTX::StoreParamI32_i,
2493 NVPTX::StoreParamI64_i, NVPTX::StoreParamF32_i,
2494 NVPTX::StoreParamF64_i);
2498 NVPTX::StoreParamI8_r, NVPTX::StoreParamI16_r,
2499 NVPTX::StoreParamI32_r, NVPTX::StoreParamI64_r,
2500 NVPTX::StoreParamF32_r, NVPTX::StoreParamF64_r);
2501 if (Opcode == NVPTX::StoreParamI8_r) {
2505 switch (Ops[0].getSimpleValueType().SimpleTy) {
2509 Opcode = NVPTX::StoreParamI8TruncI32_r;
2512 Opcode = NVPTX::StoreParamI8TruncI64_r;
2530 Opcode = NVPTX::StoreParamI32_r;
2534 MVT::i32, Ops[0], CvtNone);
2539 Opcode = NVPTX::StoreParamI32_r;
2543 MVT::i32, Ops[0], CvtNone);
2558bool NVPTXDAGToDAGISel::tryTextureIntrinsic(
SDNode *
N) {
2561 switch (
N->getOpcode()) {
2562 default:
return false;
2564 Opc = NVPTX::TEX_1D_F32_S32_RR;
2567 Opc = NVPTX::TEX_1D_F32_F32_RR;
2570 Opc = NVPTX::TEX_1D_F32_F32_LEVEL_RR;
2573 Opc = NVPTX::TEX_1D_F32_F32_GRAD_RR;
2576 Opc = NVPTX::TEX_1D_S32_S32_RR;
2579 Opc = NVPTX::TEX_1D_S32_F32_RR;
2582 Opc = NVPTX::TEX_1D_S32_F32_LEVEL_RR;
2585 Opc = NVPTX::TEX_1D_S32_F32_GRAD_RR;
2588 Opc = NVPTX::TEX_1D_U32_S32_RR;
2591 Opc = NVPTX::TEX_1D_U32_F32_RR;
2594 Opc = NVPTX::TEX_1D_U32_F32_LEVEL_RR;
2597 Opc = NVPTX::TEX_1D_U32_F32_GRAD_RR;
2600 Opc = NVPTX::TEX_1D_ARRAY_F32_S32_RR;
2603 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_RR;
2606 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL_RR;
2609 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD_RR;
2612 Opc = NVPTX::TEX_1D_ARRAY_S32_S32_RR;
2615 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_RR;
2618 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL_RR;
2621 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD_RR;
2624 Opc = NVPTX::TEX_1D_ARRAY_U32_S32_RR;
2627 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_RR;
2630 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL_RR;
2633 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD_RR;
2636 Opc = NVPTX::TEX_2D_F32_S32_RR;
2639 Opc = NVPTX::TEX_2D_F32_F32_RR;
2642 Opc = NVPTX::TEX_2D_F32_F32_LEVEL_RR;
2645 Opc = NVPTX::TEX_2D_F32_F32_GRAD_RR;
2648 Opc = NVPTX::TEX_2D_S32_S32_RR;
2651 Opc = NVPTX::TEX_2D_S32_F32_RR;
2654 Opc = NVPTX::TEX_2D_S32_F32_LEVEL_RR;
2657 Opc = NVPTX::TEX_2D_S32_F32_GRAD_RR;
2660 Opc = NVPTX::TEX_2D_U32_S32_RR;
2663 Opc = NVPTX::TEX_2D_U32_F32_RR;
2666 Opc = NVPTX::TEX_2D_U32_F32_LEVEL_RR;
2669 Opc = NVPTX::TEX_2D_U32_F32_GRAD_RR;
2672 Opc = NVPTX::TEX_2D_ARRAY_F32_S32_RR;
2675 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_RR;
2678 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL_RR;
2681 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD_RR;
2684 Opc = NVPTX::TEX_2D_ARRAY_S32_S32_RR;
2687 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_RR;
2690 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL_RR;
2693 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD_RR;
2696 Opc = NVPTX::TEX_2D_ARRAY_U32_S32_RR;
2699 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_RR;
2702 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL_RR;
2705 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD_RR;
2708 Opc = NVPTX::TEX_3D_F32_S32_RR;
2711 Opc = NVPTX::TEX_3D_F32_F32_RR;
2714 Opc = NVPTX::TEX_3D_F32_F32_LEVEL_RR;
2717 Opc = NVPTX::TEX_3D_F32_F32_GRAD_RR;
2720 Opc = NVPTX::TEX_3D_S32_S32_RR;
2723 Opc = NVPTX::TEX_3D_S32_F32_RR;
2726 Opc = NVPTX::TEX_3D_S32_F32_LEVEL_RR;
2729 Opc = NVPTX::TEX_3D_S32_F32_GRAD_RR;
2732 Opc = NVPTX::TEX_3D_U32_S32_RR;
2735 Opc = NVPTX::TEX_3D_U32_F32_RR;
2738 Opc = NVPTX::TEX_3D_U32_F32_LEVEL_RR;
2741 Opc = NVPTX::TEX_3D_U32_F32_GRAD_RR;
2744 Opc = NVPTX::TEX_CUBE_F32_F32_RR;
2747 Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL_RR;
2750 Opc = NVPTX::TEX_CUBE_S32_F32_RR;
2753 Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL_RR;
2756 Opc = NVPTX::TEX_CUBE_U32_F32_RR;
2759 Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL_RR;
2762 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_RR;
2765 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL_RR;
2768 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_RR;
2771 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL_RR;
2774 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_RR;
2777 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL_RR;
2780 Opc = NVPTX::TLD4_R_2D_F32_F32_RR;
2783 Opc = NVPTX::TLD4_G_2D_F32_F32_RR;
2786 Opc = NVPTX::TLD4_B_2D_F32_F32_RR;
2789 Opc = NVPTX::TLD4_A_2D_F32_F32_RR;
2792 Opc = NVPTX::TLD4_R_2D_S32_F32_RR;
2795 Opc = NVPTX::TLD4_G_2D_S32_F32_RR;
2798 Opc = NVPTX::TLD4_B_2D_S32_F32_RR;
2801 Opc = NVPTX::TLD4_A_2D_S32_F32_RR;
2804 Opc = NVPTX::TLD4_R_2D_U32_F32_RR;
2807 Opc = NVPTX::TLD4_G_2D_U32_F32_RR;
2810 Opc = NVPTX::TLD4_B_2D_U32_F32_RR;
2813 Opc = NVPTX::TLD4_A_2D_U32_F32_RR;
2816 Opc = NVPTX::TEX_UNIFIED_1D_F32_S32_R;
2819 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_R;
2822 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL_R;
2825 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD_R;
2828 Opc = NVPTX::TEX_UNIFIED_1D_S32_S32_R;
2831 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_R;
2834 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL_R;
2837 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD_R;
2840 Opc = NVPTX::TEX_UNIFIED_1D_U32_S32_R;
2843 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_R;
2846 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL_R;
2849 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD_R;
2852 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32_R;
2855 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_R;
2858 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL_R;
2861 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD_R;
2864 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32_R;
2867 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_R;
2870 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL_R;
2873 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD_R;
2876 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32_R;
2879 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_R;
2882 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL_R;
2885 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD_R;
2888 Opc = NVPTX::TEX_UNIFIED_2D_F32_S32_R;
2891 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_R;
2894 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL_R;
2897 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD_R;
2900 Opc = NVPTX::TEX_UNIFIED_2D_S32_S32_R;
2903 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_R;
2906 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL_R;
2909 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD_R;
2912 Opc = NVPTX::TEX_UNIFIED_2D_U32_S32_R;
2915 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_R;
2918 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL_R;
2921 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD_R;
2924 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32_R;
2927 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_R;
2930 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL_R;
2933 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD_R;
2936 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32_R;
2939 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_R;
2942 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL_R;
2945 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD_R;
2948 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32_R;
2951 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_R;
2954 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL_R;
2957 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD_R;
2960 Opc = NVPTX::TEX_UNIFIED_3D_F32_S32_R;
2963 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_R;
2966 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL_R;
2969 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD_R;
2972 Opc = NVPTX::TEX_UNIFIED_3D_S32_S32_R;
2975 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_R;
2978 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL_R;
2981 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD_R;
2984 Opc = NVPTX::TEX_UNIFIED_3D_U32_S32_R;
2987 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_R;
2990 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL_R;
2993 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD_R;
2996 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_R;
2999 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL_R;
3002 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_R;
3005 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL_R;
3008 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_R;
3011 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL_R;
3014 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_R;
3017 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL_R;
3020 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_R;
3023 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL_R;
3026 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_R;
3029 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL_R;
3032 Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32_R;
3035 Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32_R;
3038 Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32_R;
3041 Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32_R;
3044 Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32_R;
3047 Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32_R;
3050 Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32_R;
3053 Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32_R;
3056 Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32_R;
3059 Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32_R;
3062 Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32_R;
3065 Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32_R;
3068 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_GRAD_R;
3071 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_GRAD_R;
3074 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_GRAD_R;
3077 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_GRAD_R;
3080 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_GRAD_R;
3083 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_GRAD_R;
3095bool NVPTXDAGToDAGISel::trySurfaceIntrinsic(
SDNode *
N) {
3097 switch (
N->getOpcode()) {
3098 default:
return false;
3100 Opc = NVPTX::SULD_1D_I8_CLAMP_R;
3103 Opc = NVPTX::SULD_1D_I16_CLAMP_R;
3106 Opc = NVPTX::SULD_1D_I32_CLAMP_R;
3109 Opc = NVPTX::SULD_1D_I64_CLAMP_R;
3112 Opc = NVPTX::SULD_1D_V2I8_CLAMP_R;
3115 Opc = NVPTX::SULD_1D_V2I16_CLAMP_R;
3118 Opc = NVPTX::SULD_1D_V2I32_CLAMP_R;
3121 Opc = NVPTX::SULD_1D_V2I64_CLAMP_R;
3124 Opc = NVPTX::SULD_1D_V4I8_CLAMP_R;
3127 Opc = NVPTX::SULD_1D_V4I16_CLAMP_R;
3130 Opc = NVPTX::SULD_1D_V4I32_CLAMP_R;
3133 Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP_R;
3136 Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP_R;
3139 Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP_R;
3142 Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP_R;
3145 Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP_R;
3148 Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP_R;
3151 Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP_R;
3154 Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP_R;
3157 Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP_R;
3160 Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP_R;
3163 Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP_R;
3166 Opc = NVPTX::SULD_2D_I8_CLAMP_R;
3169 Opc = NVPTX::SULD_2D_I16_CLAMP_R;
3172 Opc = NVPTX::SULD_2D_I32_CLAMP_R;
3175 Opc = NVPTX::SULD_2D_I64_CLAMP_R;
3178 Opc = NVPTX::SULD_2D_V2I8_CLAMP_R;
3181 Opc = NVPTX::SULD_2D_V2I16_CLAMP_R;
3184 Opc = NVPTX::SULD_2D_V2I32_CLAMP_R;
3187 Opc = NVPTX::SULD_2D_V2I64_CLAMP_R;
3190 Opc = NVPTX::SULD_2D_V4I8_CLAMP_R;
3193 Opc = NVPTX::SULD_2D_V4I16_CLAMP_R;
3196 Opc = NVPTX::SULD_2D_V4I32_CLAMP_R;
3199 Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP_R;
3202 Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP_R;
3205 Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP_R;
3208 Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP_R;
3211 Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP_R;
3214 Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP_R;
3217 Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP_R;
3220 Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP_R;
3223 Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP_R;
3226 Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP_R;
3229 Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP_R;
3232 Opc = NVPTX::SULD_3D_I8_CLAMP_R;
3235 Opc = NVPTX::SULD_3D_I16_CLAMP_R;
3238 Opc = NVPTX::SULD_3D_I32_CLAMP_R;
3241 Opc = NVPTX::SULD_3D_I64_CLAMP_R;
3244 Opc = NVPTX::SULD_3D_V2I8_CLAMP_R;
3247 Opc = NVPTX::SULD_3D_V2I16_CLAMP_R;
3250 Opc = NVPTX::SULD_3D_V2I32_CLAMP_R;
3253 Opc = NVPTX::SULD_3D_V2I64_CLAMP_R;
3256 Opc = NVPTX::SULD_3D_V4I8_CLAMP_R;
3259 Opc = NVPTX::SULD_3D_V4I16_CLAMP_R;
3262 Opc = NVPTX::SULD_3D_V4I32_CLAMP_R;
3265 Opc = NVPTX::SULD_1D_I8_TRAP_R;
3268 Opc = NVPTX::SULD_1D_I16_TRAP_R;
3271 Opc = NVPTX::SULD_1D_I32_TRAP_R;
3274 Opc = NVPTX::SULD_1D_I64_TRAP_R;
3277 Opc = NVPTX::SULD_1D_V2I8_TRAP_R;
3280 Opc = NVPTX::SULD_1D_V2I16_TRAP_R;
3283 Opc = NVPTX::SULD_1D_V2I32_TRAP_R;
3286 Opc = NVPTX::SULD_1D_V2I64_TRAP_R;
3289 Opc = NVPTX::SULD_1D_V4I8_TRAP_R;
3292 Opc = NVPTX::SULD_1D_V4I16_TRAP_R;
3295 Opc = NVPTX::SULD_1D_V4I32_TRAP_R;
3298 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP_R;
3301 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP_R;
3304 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP_R;
3307 Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP_R;
3310 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP_R;
3313 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP_R;
3316 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP_R;
3319 Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP_R;
3322 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP_R;
3325 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP_R;
3328 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP_R;
3331 Opc = NVPTX::SULD_2D_I8_TRAP_R;
3334 Opc = NVPTX::SULD_2D_I16_TRAP_R;
3337 Opc = NVPTX::SULD_2D_I32_TRAP_R;
3340 Opc = NVPTX::SULD_2D_I64_TRAP_R;
3343 Opc = NVPTX::SULD_2D_V2I8_TRAP_R;
3346 Opc = NVPTX::SULD_2D_V2I16_TRAP_R;
3349 Opc = NVPTX::SULD_2D_V2I32_TRAP_R;
3352 Opc = NVPTX::SULD_2D_V2I64_TRAP_R;
3355 Opc = NVPTX::SULD_2D_V4I8_TRAP_R;
3358 Opc = NVPTX::SULD_2D_V4I16_TRAP_R;
3361 Opc = NVPTX::SULD_2D_V4I32_TRAP_R;
3364 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP_R;
3367 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP_R;
3370 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP_R;
3373 Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP_R;
3376 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP_R;
3379 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP_R;
3382 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP_R;
3385 Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP_R;
3388 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP_R;
3391 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP_R;
3394 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP_R;
3397 Opc = NVPTX::SULD_3D_I8_TRAP_R;
3400 Opc = NVPTX::SULD_3D_I16_TRAP_R;
3403 Opc = NVPTX::SULD_3D_I32_TRAP_R;
3406 Opc = NVPTX::SULD_3D_I64_TRAP_R;
3409 Opc = NVPTX::SULD_3D_V2I8_TRAP_R;
3412 Opc = NVPTX::SULD_3D_V2I16_TRAP_R;
3415 Opc = NVPTX::SULD_3D_V2I32_TRAP_R;
3418 Opc = NVPTX::SULD_3D_V2I64_TRAP_R;
3421 Opc = NVPTX::SULD_3D_V4I8_TRAP_R;
3424 Opc = NVPTX::SULD_3D_V4I16_TRAP_R;
3427 Opc = NVPTX::SULD_3D_V4I32_TRAP_R;
3430 Opc = NVPTX::SULD_1D_I8_ZERO_R;
3433 Opc = NVPTX::SULD_1D_I16_ZERO_R;
3436 Opc = NVPTX::SULD_1D_I32_ZERO_R;
3439 Opc = NVPTX::SULD_1D_I64_ZERO_R;
3442 Opc = NVPTX::SULD_1D_V2I8_ZERO_R;
3445 Opc = NVPTX::SULD_1D_V2I16_ZERO_R;
3448 Opc = NVPTX::SULD_1D_V2I32_ZERO_R;
3451 Opc = NVPTX::SULD_1D_V2I64_ZERO_R;
3454 Opc = NVPTX::SULD_1D_V4I8_ZERO_R;
3457 Opc = NVPTX::SULD_1D_V4I16_ZERO_R;
3460 Opc = NVPTX::SULD_1D_V4I32_ZERO_R;
3463 Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO_R;
3466 Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO_R;
3469 Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO_R;
3472 Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO_R;
3475 Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO_R;
3478 Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO_R;
3481 Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO_R;
3484 Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO_R;
3487 Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO_R;
3490 Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO_R;
3493 Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO_R;
3496 Opc = NVPTX::SULD_2D_I8_ZERO_R;
3499 Opc = NVPTX::SULD_2D_I16_ZERO_R;
3502 Opc = NVPTX::SULD_2D_I32_ZERO_R;
3505 Opc = NVPTX::SULD_2D_I64_ZERO_R;
3508 Opc = NVPTX::SULD_2D_V2I8_ZERO_R;
3511 Opc = NVPTX::SULD_2D_V2I16_ZERO_R;
3514 Opc = NVPTX::SULD_2D_V2I32_ZERO_R;
3517 Opc = NVPTX::SULD_2D_V2I64_ZERO_R;
3520 Opc = NVPTX::SULD_2D_V4I8_ZERO_R;
3523 Opc = NVPTX::SULD_2D_V4I16_ZERO_R;
3526 Opc = NVPTX::SULD_2D_V4I32_ZERO_R;
3529 Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO_R;
3532 Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO_R;
3535 Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO_R;
3538 Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO_R;
3541 Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO_R;
3544 Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO_R;
3547 Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO_R;
3550 Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO_R;
3553 Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO_R;
3556 Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO_R;
3559 Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO_R;
3562 Opc = NVPTX::SULD_3D_I8_ZERO_R;
3565 Opc = NVPTX::SULD_3D_I16_ZERO_R;
3568 Opc = NVPTX::SULD_3D_I32_ZERO_R;
3571 Opc = NVPTX::SULD_3D_I64_ZERO_R;
3574 Opc = NVPTX::SULD_3D_V2I8_ZERO_R;
3577 Opc = NVPTX::SULD_3D_V2I16_ZERO_R;
3580 Opc = NVPTX::SULD_3D_V2I32_ZERO_R;
3583 Opc = NVPTX::SULD_3D_V2I64_ZERO_R;
3586 Opc = NVPTX::SULD_3D_V4I8_ZERO_R;
3589 Opc = NVPTX::SULD_3D_V4I16_ZERO_R;
3592 Opc = NVPTX::SULD_3D_V4I32_ZERO_R;
3607bool NVPTXDAGToDAGISel::tryBFE(
SDNode *
N) {
3614 bool IsSigned =
false;
3619 if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
3644 Val =
LHS.getNode()->getOperand(0);
3645 Start =
LHS.getNode()->getOperand(1);
3651 int64_t GoodBits = Start.getValueSizeInBits() - StartVal;
3652 if (NumBits > GoodBits) {
3686 if (isa<ConstantSDNode>(AndLHS)) {
3710 NumBits = NumZeros + NumOnes - ShiftAmt;
3716 if (ShiftAmt < NumZeros) {
3733 Val =
LHS->getOperand(0);
3752 if (OuterShiftAmt < InnerShiftAmt) {
3788 Opc = NVPTX::BFE_S32rii;
3790 Opc = NVPTX::BFE_U32rii;
3794 Opc = NVPTX::BFE_S64rii;
3796 Opc = NVPTX::BFE_U64rii;
3835bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
3840 if (SelectDirectAddr(base,
Base)) {
3863bool NVPTXDAGToDAGISel::SelectADDRri_imp(
3875 if (SelectDirectAddr(
Addr.getOperand(0),
Addr)) {
3880 dyn_cast<FrameIndexSDNode>(
Addr.getOperand(0)))
3888 if (!CN->getAPIntValue().isSignedIntN(32))
3911bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(
SDNode *
N,
3912 unsigned int spN)
const {
3913 const Value *Src =
nullptr;
3914 if (
MemSDNode *mN = dyn_cast<MemSDNode>(
N)) {
3915 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
3917 Src = mN->getMemOperand()->getValue();
3921 if (
auto *PT = dyn_cast<PointerType>(Src->getType()))
3922 return (PT->getAddressSpace() == spN);
3930 std::vector<SDValue> &OutOps) {
3932 switch (ConstraintID) {
3936 if (SelectDirectAddr(
Op, Op0)) {
3937 OutOps.push_back(Op0);
3941 if (SelectADDRri(
Op.getNode(),
Op, Op0, Op1)) {
3942 OutOps.push_back(Op0);
3943 OutOps.push_back(Op1);
3951void NVPTXDAGToDAGISel::SelectV2I64toI128(
SDNode *
N) {
3970 NewOps[0] =
N->getOperand(0);
3973 if (
N->getNumOperands() == 5)
3974 NewOps[3] =
N->getOperand(4);
3980void NVPTXDAGToDAGISel::SelectI128toV2I64(
SDNode *
N) {
3998 NVPTX::I128toV2I64,
DL,
4007unsigned NVPTXDAGToDAGISel::GetConvertOpcode(
MVT DestTy,
MVT SrcTy,
4018 return IsSigned ? NVPTX::CVT_s16_s8 : NVPTX::CVT_u16_u8;
4020 return IsSigned ? NVPTX::CVT_s32_s8 : NVPTX::CVT_u32_u8;
4022 return IsSigned ? NVPTX::CVT_s64_s8 : NVPTX::CVT_u64_u8;
4029 return IsSigned ? NVPTX::CVT_s8_s16 : NVPTX::CVT_u8_u16;
4031 return IsSigned ? NVPTX::CVT_s32_s16 : NVPTX::CVT_u32_u16;
4033 return IsSigned ? NVPTX::CVT_s64_s16 : NVPTX::CVT_u64_u16;
4040 return IsSigned ? NVPTX::CVT_s8_s32 : NVPTX::CVT_u8_u32;
4042 return IsSigned ? NVPTX::CVT_s16_s32 : NVPTX::CVT_u16_u32;
4044 return IsSigned ? NVPTX::CVT_s64_s32 : NVPTX::CVT_u64_u32;
4051 return IsSigned ? NVPTX::CVT_s8_s64 : NVPTX::CVT_u8_u64;
4053 return IsSigned ? NVPTX::CVT_s16_s64 : NVPTX::CVT_u16_u64;
4055 return IsSigned ? NVPTX::CVT_s32_s64 : NVPTX::CVT_u32_u64;
4062 return NVPTX::CVT_f32_f16;
4064 return NVPTX::CVT_f64_f16;
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Atomic ordering constants.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define getOpcodeForVectorStParam(n, ty, isimm)
static unsigned int getCodeAddrSpace(MemSDNode *N)
static int getLdStRegType(EVT VT)
static unsigned pickOpcodeForVectorStParam(SmallVector< SDValue, 8 > &Ops, unsigned NumElts, MVT::SimpleValueType MemTy, SelectionDAG *CurDAG, SDLoc DL)
#define getOpcodeForVectorStParamV2(ty, isimm)
static cl::opt< bool > EnableRsqrtOpt("nvptx-rsqrt-approx-opt", cl::init(true), cl::Hidden, cl::desc("Enable reciprocal sqrt optimization"))
static unsigned getPTXCmpMode(const CondCodeSDNode &CondCode, bool FTZ)
static std::optional< unsigned > pickOpcodeForVT(MVT::SimpleValueType VT, unsigned Opcode_i8, unsigned Opcode_i16, unsigned Opcode_i32, std::optional< unsigned > Opcode_i64, unsigned Opcode_f32, std::optional< unsigned > Opcode_f64)
static bool canLowerToLDG(MemSDNode *N, const NVPTXSubtarget &Subtarget, unsigned CodeAddrSpace, MachineFunction *F)
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned getSrcAddressSpace() const
unsigned getDestAddressSpace() const
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This class represents an Operation in the Expression.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
FunctionPass class - This class is used to implement most global optimizations.
Record instruction ordering so we can query their relative positions within a function.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
This class is used to represent ISD::LOAD nodes.
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
bool isVector() const
Return true if this is a vector value type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getVectorElementType() const
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
EVT getMemoryVT() const
Return the type of the in-memory value.
NVPTXDAGToDAGISelLegacy(NVPTXTargetMachine &tm, CodeGenOptLevel OptLevel)
bool runOnMachineFunction(MachineFunction &MF) override
bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Implement addressing mode selection for inline asm expressions.
const NVPTXSubtarget * Subtarget
const NVPTXTargetLowering * getTargetLowering() const override
bool hasRelaxedMMIO() const
bool hasMemoryOrdering() const
bool useF32FTZ(const MachineFunction &MF) const
bool allowFMA(MachineFunction &MF, CodeGenOptLevel OptLevel) const
bool usePrecSqrtF32() const
bool allowUnsafeFPMath(MachineFunction &MF) const
int getDivF32Level() const
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const SDValue & getOperand(unsigned Num) const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getValue() const
unsigned getPointerSizeInBits(unsigned AS) const
LLVM Value Representation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
@ TexUnified1DS32FloatLevel
@ Tex1DArrayFloatFloatLevel
@ TexUnified2DU32FloatGrad
@ Tld4UnifiedG2DFloatFloat
@ TexUnifiedCubeArrayFloatFloatLevel
@ Tld4UnifiedR2DFloatFloat
@ Tex2DArrayS32FloatLevel
@ TexUnified1DArrayFloatFloatLevel
@ TexUnified2DFloatFloatLevel
@ TexUnified3DFloatFloatLevel
@ TexUnified1DFloatFloatLevel
@ TexUnified2DArrayU32Float
@ TexUnified1DArrayFloatFloat
@ Tex1DArrayFloatFloatGrad
@ TexUnifiedCubeArrayU32FloatGrad
@ TexUnified1DFloatFloatGrad
@ TexUnifiedCubeFloatFloatGrad
@ TexUnified2DArrayFloatFloat
@ TexUnified3DU32FloatLevel
@ TexUnified1DArrayU32Float
@ TexUnified2DArrayFloatFloatLevel
@ TexUnified2DFloatFloatGrad
@ TexUnified2DArrayU32S32
@ TexUnifiedCubeArrayS32FloatLevel
@ TexUnified1DArrayS32Float
@ TexUnified1DArrayS32FloatLevel
@ TexUnified2DS32FloatLevel
@ TexUnified3DU32FloatGrad
@ TexUnifiedCubeU32FloatLevel
@ TexUnified2DArrayU32FloatGrad
@ TexUnifiedCubeFloatFloatLevel
@ TexUnified1DArrayFloatS32
@ TexUnifiedCubeS32FloatLevel
@ TexUnified1DS32FloatGrad
@ Tex2DArrayFloatFloatLevel
@ TexUnifiedCubeArrayFloatFloat
@ TexUnifiedCubeArrayFloatFloatGrad
@ TexUnifiedCubeFloatFloat
@ TexUnified1DArrayU32S32
@ TexUnified3DFloatFloatGrad
@ Tld4UnifiedA2DFloatFloat
@ TexUnified3DS32FloatGrad
@ TexUnified2DU32FloatLevel
@ TexUnified1DArrayS32S32
@ TexCubeArrayFloatFloatLevel
@ TexUnified1DU32FloatGrad
@ TexCubeArrayS32FloatLevel
@ Tex2DArrayU32FloatLevel
@ Tex1DArrayU32FloatLevel
@ TexUnified2DArrayU32FloatLevel
@ TexUnified1DArrayFloatFloatGrad
@ TexUnifiedCubeS32FloatGrad
@ TexCubeArrayU32FloatLevel
@ TexUnified3DS32FloatLevel
@ TexUnifiedCubeArrayS32FloatGrad
@ TexUnified2DArrayS32Float
@ Tex2DArrayFloatFloatGrad
@ TexUnifiedCubeArrayS32Float
@ TexUnified2DArrayS32FloatLevel
@ Tex1DArrayS32FloatLevel
@ TexUnifiedCubeArrayU32FloatLevel
@ TexUnified2DArrayS32S32
@ TexUnified2DArrayFloatFloatGrad
@ TexUnifiedCubeU32FloatGrad
@ Tld4UnifiedB2DFloatFloat
@ TexUnified1DArrayU32FloatLevel
@ TexUnified1DArrayS32FloatGrad
@ TexUnified2DS32FloatGrad
@ TexUnified2DArrayS32FloatGrad
@ TexUnified1DU32FloatLevel
@ TexUnifiedCubeArrayU32Float
@ TexUnified2DArrayFloatS32
@ TexUnified1DArrayU32FloatGrad
std::string OrderingToCString(Ordering Order)
initializer< Ty > init(const Ty &Val)
constexpr uint64_t PointerSize
aarch64 pointer size.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
FunctionPass * createNVPTXISelDag(NVPTXTargetMachine &TM, llvm::CodeGenOptLevel OptLevel)
createNVPTXISelDag - This pass converts a legalized DAG into a NVPTX-specific DAG,...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
const char * toIRString(AtomicOrdering ao)
String used by LLVM IR to represent atomic ordering.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
CodeGenOptLevel
Code generation optimization level.
AtomicOrdering
Atomic ordering for LLVM's memory model.
bool isKernelFunction(const Function &F)
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, const LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isVector() const
Return true if this is a vector value type.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.