LLVM API Documentation

NVPTXISelDAGToDAG.cpp
Go to the documentation of this file.
00001 //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file defines an instruction selector for the NVPTX target.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "NVPTXISelDAGToDAG.h"
00015 #include "llvm/IR/GlobalValue.h"
00016 #include "llvm/IR/Instructions.h"
00017 #include "llvm/Support/CommandLine.h"
00018 #include "llvm/Support/Debug.h"
00019 #include "llvm/Support/ErrorHandling.h"
00020 #include "llvm/Support/raw_ostream.h"
00021 #include "llvm/Target/TargetIntrinsicInfo.h"
00022 
00023 #undef DEBUG_TYPE
00024 #define DEBUG_TYPE "nvptx-isel"
00025 
00026 using namespace llvm;
00027 
00028 static cl::opt<int>
00029 FMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden,
00030                  cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
00031                           " 1: do it  2: do it aggressively"),
00032                  cl::init(2));
00033 
00034 static cl::opt<int> UsePrecDivF32(
00035     "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
00036     cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
00037              " IEEE Compliant F32 div.rnd if avaiable."),
00038     cl::init(2));
00039 
00040 static cl::opt<bool>
00041 UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
00042           cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
00043           cl::init(true));
00044 
00045 static cl::opt<bool>
00046 FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
00047            cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
00048            cl::init(false));
00049 
00050 
00051 /// createNVPTXISelDag - This pass converts a legalized DAG into a
00052 /// NVPTX-specific DAG, ready for instruction scheduling.
00053 FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
00054                                        llvm::CodeGenOpt::Level OptLevel) {
00055   return new NVPTXDAGToDAGISel(TM, OptLevel);
00056 }
00057 
00058 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
00059                                      CodeGenOpt::Level OptLevel)
00060     : SelectionDAGISel(tm, OptLevel),
00061       Subtarget(tm.getSubtarget<NVPTXSubtarget>()) {
00062 
00063   doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1);
00064   doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1);
00065   doFMAF32AGG =
00066       (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2);
00067   doFMAF64AGG =
00068       (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2);
00069 
00070   allowFMA = (FMAContractLevel >= 1);
00071 
00072   doMulWide = (OptLevel > 0);
00073 }
00074 
00075 int NVPTXDAGToDAGISel::getDivF32Level() const {
00076   if (UsePrecDivF32.getNumOccurrences() > 0) {
00077     // If nvptx-prec-div32=N is used on the command-line, always honor it
00078     return UsePrecDivF32;
00079   } else {
00080     // Otherwise, use div.approx if fast math is enabled
00081     if (TM.Options.UnsafeFPMath)
00082       return 0;
00083     else
00084       return 2;
00085   }
00086 }
00087 
00088 bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
00089   if (UsePrecSqrtF32.getNumOccurrences() > 0) {
00090     // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
00091     return UsePrecSqrtF32;
00092   } else {
00093     // Otherwise, use sqrt.approx if fast math is enabled
00094     if (TM.Options.UnsafeFPMath)
00095       return false;
00096     else
00097       return true;
00098   }
00099 }
00100 
00101 bool NVPTXDAGToDAGISel::useF32FTZ() const {
00102   if (FtzEnabled.getNumOccurrences() > 0) {
00103     // If nvptx-f32ftz is used on the command-line, always honor it
00104     return FtzEnabled;
00105   } else {
00106     const Function *F = MF->getFunction();
00107     // Otherwise, check for an nvptx-f32ftz attribute on the function
00108     if (F->hasFnAttribute("nvptx-f32ftz"))
00109       return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex,
00110                                               "nvptx-f32ftz")
00111                                               .getValueAsString() == "true");
00112     else
00113       return false;
00114   }
00115 }
00116 
00117 /// Select - Select instructions not customized! Used for
00118 /// expanded, promoted and normal instructions.
00119 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
00120 
00121   if (N->isMachineOpcode()) {
00122     N->setNodeId(-1);
00123     return NULL; // Already selected.
00124   }
00125 
00126   SDNode *ResNode = NULL;
00127   switch (N->getOpcode()) {
00128   case ISD::LOAD:
00129     ResNode = SelectLoad(N);
00130     break;
00131   case ISD::STORE:
00132     ResNode = SelectStore(N);
00133     break;
00134   case NVPTXISD::LoadV2:
00135   case NVPTXISD::LoadV4:
00136     ResNode = SelectLoadVector(N);
00137     break;
00138   case NVPTXISD::LDGV2:
00139   case NVPTXISD::LDGV4:
00140   case NVPTXISD::LDUV2:
00141   case NVPTXISD::LDUV4:
00142     ResNode = SelectLDGLDUVector(N);
00143     break;
00144   case NVPTXISD::StoreV2:
00145   case NVPTXISD::StoreV4:
00146     ResNode = SelectStoreVector(N);
00147     break;
00148   case NVPTXISD::LoadParam:
00149   case NVPTXISD::LoadParamV2:
00150   case NVPTXISD::LoadParamV4:
00151     ResNode = SelectLoadParam(N);
00152     break;
00153   case NVPTXISD::StoreRetval:
00154   case NVPTXISD::StoreRetvalV2:
00155   case NVPTXISD::StoreRetvalV4:
00156     ResNode = SelectStoreRetval(N);
00157     break;
00158   case NVPTXISD::StoreParam:
00159   case NVPTXISD::StoreParamV2:
00160   case NVPTXISD::StoreParamV4:
00161   case NVPTXISD::StoreParamS32:
00162   case NVPTXISD::StoreParamU32:
00163     ResNode = SelectStoreParam(N);
00164     break;
00165   case ISD::INTRINSIC_WO_CHAIN:
00166     ResNode = SelectIntrinsicNoChain(N);
00167     break;
00168   case NVPTXISD::Tex1DFloatI32:
00169   case NVPTXISD::Tex1DFloatFloat:
00170   case NVPTXISD::Tex1DFloatFloatLevel:
00171   case NVPTXISD::Tex1DFloatFloatGrad:
00172   case NVPTXISD::Tex1DI32I32:
00173   case NVPTXISD::Tex1DI32Float:
00174   case NVPTXISD::Tex1DI32FloatLevel:
00175   case NVPTXISD::Tex1DI32FloatGrad:
00176   case NVPTXISD::Tex1DArrayFloatI32:
00177   case NVPTXISD::Tex1DArrayFloatFloat:
00178   case NVPTXISD::Tex1DArrayFloatFloatLevel:
00179   case NVPTXISD::Tex1DArrayFloatFloatGrad:
00180   case NVPTXISD::Tex1DArrayI32I32:
00181   case NVPTXISD::Tex1DArrayI32Float:
00182   case NVPTXISD::Tex1DArrayI32FloatLevel:
00183   case NVPTXISD::Tex1DArrayI32FloatGrad:
00184   case NVPTXISD::Tex2DFloatI32:
00185   case NVPTXISD::Tex2DFloatFloat:
00186   case NVPTXISD::Tex2DFloatFloatLevel:
00187   case NVPTXISD::Tex2DFloatFloatGrad:
00188   case NVPTXISD::Tex2DI32I32:
00189   case NVPTXISD::Tex2DI32Float:
00190   case NVPTXISD::Tex2DI32FloatLevel:
00191   case NVPTXISD::Tex2DI32FloatGrad:
00192   case NVPTXISD::Tex2DArrayFloatI32:
00193   case NVPTXISD::Tex2DArrayFloatFloat:
00194   case NVPTXISD::Tex2DArrayFloatFloatLevel:
00195   case NVPTXISD::Tex2DArrayFloatFloatGrad:
00196   case NVPTXISD::Tex2DArrayI32I32:
00197   case NVPTXISD::Tex2DArrayI32Float:
00198   case NVPTXISD::Tex2DArrayI32FloatLevel:
00199   case NVPTXISD::Tex2DArrayI32FloatGrad:
00200   case NVPTXISD::Tex3DFloatI32:
00201   case NVPTXISD::Tex3DFloatFloat:
00202   case NVPTXISD::Tex3DFloatFloatLevel:
00203   case NVPTXISD::Tex3DFloatFloatGrad:
00204   case NVPTXISD::Tex3DI32I32:
00205   case NVPTXISD::Tex3DI32Float:
00206   case NVPTXISD::Tex3DI32FloatLevel:
00207   case NVPTXISD::Tex3DI32FloatGrad:
00208     ResNode = SelectTextureIntrinsic(N);
00209     break;
00210   case NVPTXISD::Suld1DI8Trap:
00211   case NVPTXISD::Suld1DI16Trap:
00212   case NVPTXISD::Suld1DI32Trap:
00213   case NVPTXISD::Suld1DV2I8Trap:
00214   case NVPTXISD::Suld1DV2I16Trap:
00215   case NVPTXISD::Suld1DV2I32Trap:
00216   case NVPTXISD::Suld1DV4I8Trap:
00217   case NVPTXISD::Suld1DV4I16Trap:
00218   case NVPTXISD::Suld1DV4I32Trap:
00219   case NVPTXISD::Suld1DArrayI8Trap:
00220   case NVPTXISD::Suld1DArrayI16Trap:
00221   case NVPTXISD::Suld1DArrayI32Trap:
00222   case NVPTXISD::Suld1DArrayV2I8Trap:
00223   case NVPTXISD::Suld1DArrayV2I16Trap:
00224   case NVPTXISD::Suld1DArrayV2I32Trap:
00225   case NVPTXISD::Suld1DArrayV4I8Trap:
00226   case NVPTXISD::Suld1DArrayV4I16Trap:
00227   case NVPTXISD::Suld1DArrayV4I32Trap:
00228   case NVPTXISD::Suld2DI8Trap:
00229   case NVPTXISD::Suld2DI16Trap:
00230   case NVPTXISD::Suld2DI32Trap:
00231   case NVPTXISD::Suld2DV2I8Trap:
00232   case NVPTXISD::Suld2DV2I16Trap:
00233   case NVPTXISD::Suld2DV2I32Trap:
00234   case NVPTXISD::Suld2DV4I8Trap:
00235   case NVPTXISD::Suld2DV4I16Trap:
00236   case NVPTXISD::Suld2DV4I32Trap:
00237   case NVPTXISD::Suld2DArrayI8Trap:
00238   case NVPTXISD::Suld2DArrayI16Trap:
00239   case NVPTXISD::Suld2DArrayI32Trap:
00240   case NVPTXISD::Suld2DArrayV2I8Trap:
00241   case NVPTXISD::Suld2DArrayV2I16Trap:
00242   case NVPTXISD::Suld2DArrayV2I32Trap:
00243   case NVPTXISD::Suld2DArrayV4I8Trap:
00244   case NVPTXISD::Suld2DArrayV4I16Trap:
00245   case NVPTXISD::Suld2DArrayV4I32Trap:
00246   case NVPTXISD::Suld3DI8Trap:
00247   case NVPTXISD::Suld3DI16Trap:
00248   case NVPTXISD::Suld3DI32Trap:
00249   case NVPTXISD::Suld3DV2I8Trap:
00250   case NVPTXISD::Suld3DV2I16Trap:
00251   case NVPTXISD::Suld3DV2I32Trap:
00252   case NVPTXISD::Suld3DV4I8Trap:
00253   case NVPTXISD::Suld3DV4I16Trap:
00254   case NVPTXISD::Suld3DV4I32Trap:
00255     ResNode = SelectSurfaceIntrinsic(N);
00256     break;
00257   case ISD::ADDRSPACECAST:
00258     ResNode = SelectAddrSpaceCast(N);
00259     break;
00260   default:
00261     break;
00262   }
00263   if (ResNode)
00264     return ResNode;
00265   return SelectCode(N);
00266 }
00267 
00268 static unsigned int getCodeAddrSpace(MemSDNode *N,
00269                                      const NVPTXSubtarget &Subtarget) {
00270   const Value *Src = N->getMemOperand()->getValue();
00271 
00272   if (!Src)
00273     return NVPTX::PTXLdStInstCode::GENERIC;
00274 
00275   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
00276     switch (PT->getAddressSpace()) {
00277     case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
00278     case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
00279     case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
00280     case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
00281     case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
00282     case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
00283     default: break;
00284     }
00285   }
00286   return NVPTX::PTXLdStInstCode::GENERIC;
00287 }
00288 
00289 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
00290   unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
00291   switch (IID) {
00292   default:
00293     return NULL;
00294   case Intrinsic::nvvm_texsurf_handle_internal:
00295     return SelectTexSurfHandle(N);
00296   }
00297 }
00298 
00299 SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
00300   // Op 0 is the intrinsic ID
00301   SDValue Wrapper = N->getOperand(1);
00302   SDValue GlobalVal = Wrapper.getOperand(0);
00303   return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
00304                                 GlobalVal);
00305 }
00306 
00307 SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
00308   SDValue Src = N->getOperand(0);
00309   AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
00310   unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
00311   unsigned DstAddrSpace = CastN->getDestAddressSpace();
00312 
00313   assert(SrcAddrSpace != DstAddrSpace &&
00314          "addrspacecast must be between different address spaces");
00315 
00316   if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
00317     // Specific to generic
00318     unsigned Opc;
00319     switch (SrcAddrSpace) {
00320     default: report_fatal_error("Bad address space in addrspacecast");
00321     case ADDRESS_SPACE_GLOBAL:
00322       Opc = Subtarget.is64Bit() ? NVPTX::cvta_global_yes_64
00323                                 : NVPTX::cvta_global_yes;
00324       break;
00325     case ADDRESS_SPACE_SHARED:
00326       Opc = Subtarget.is64Bit() ? NVPTX::cvta_shared_yes_64
00327                                 : NVPTX::cvta_shared_yes;
00328       break;
00329     case ADDRESS_SPACE_CONST:
00330       Opc = Subtarget.is64Bit() ? NVPTX::cvta_const_yes_64
00331                                 : NVPTX::cvta_const_yes;
00332       break;
00333     case ADDRESS_SPACE_LOCAL:
00334       Opc = Subtarget.is64Bit() ? NVPTX::cvta_local_yes_64
00335                                 : NVPTX::cvta_local_yes;
00336       break;
00337     }
00338     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00339   } else {
00340     // Generic to specific
00341     if (SrcAddrSpace != 0)
00342       report_fatal_error("Cannot cast between two non-generic address spaces");
00343     unsigned Opc;
00344     switch (DstAddrSpace) {
00345     default: report_fatal_error("Bad address space in addrspacecast");
00346     case ADDRESS_SPACE_GLOBAL:
00347       Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_global_yes_64
00348                                 : NVPTX::cvta_to_global_yes;
00349       break;
00350     case ADDRESS_SPACE_SHARED:
00351       Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_shared_yes_64
00352                                 : NVPTX::cvta_to_shared_yes;
00353       break;
00354     case ADDRESS_SPACE_CONST:
00355       Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_const_yes_64
00356                                 : NVPTX::cvta_to_const_yes;
00357       break;
00358     case ADDRESS_SPACE_LOCAL:
00359       Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_local_yes_64
00360                                 : NVPTX::cvta_to_local_yes;
00361       break;
00362     }
00363     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00364   }
00365 }
00366 
00367 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
00368   SDLoc dl(N);
00369   LoadSDNode *LD = cast<LoadSDNode>(N);
00370   EVT LoadedVT = LD->getMemoryVT();
00371   SDNode *NVPTXLD = NULL;
00372 
00373   // do not support pre/post inc/dec
00374   if (LD->isIndexed())
00375     return NULL;
00376 
00377   if (!LoadedVT.isSimple())
00378     return NULL;
00379 
00380   // Address Space Setting
00381   unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget);
00382 
00383   // Volatile Setting
00384   // - .volatile is only availalble for .global and .shared
00385   bool isVolatile = LD->isVolatile();
00386   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00387       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00388       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00389     isVolatile = false;
00390 
00391   // Vector Setting
00392   MVT SimpleVT = LoadedVT.getSimpleVT();
00393   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
00394   if (SimpleVT.isVector()) {
00395     unsigned num = SimpleVT.getVectorNumElements();
00396     if (num == 2)
00397       vecType = NVPTX::PTXLdStInstCode::V2;
00398     else if (num == 4)
00399       vecType = NVPTX::PTXLdStInstCode::V4;
00400     else
00401       return NULL;
00402   }
00403 
00404   // Type Setting: fromType + fromTypeWidth
00405   //
00406   // Sign   : ISD::SEXTLOAD
00407   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00408   //          type is integer
00409   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00410   MVT ScalarVT = SimpleVT.getScalarType();
00411   // Read at least 8 bits (predicates are stored as 8-bit values)
00412   unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00413   unsigned int fromType;
00414   if ((LD->getExtensionType() == ISD::SEXTLOAD))
00415     fromType = NVPTX::PTXLdStInstCode::Signed;
00416   else if (ScalarVT.isFloatingPoint())
00417     fromType = NVPTX::PTXLdStInstCode::Float;
00418   else
00419     fromType = NVPTX::PTXLdStInstCode::Unsigned;
00420 
00421   // Create the machine instruction DAG
00422   SDValue Chain = N->getOperand(0);
00423   SDValue N1 = N->getOperand(1);
00424   SDValue Addr;
00425   SDValue Offset, Base;
00426   unsigned Opcode;
00427   MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
00428 
00429   if (SelectDirectAddr(N1, Addr)) {
00430     switch (TargetVT) {
00431     case MVT::i8:
00432       Opcode = NVPTX::LD_i8_avar;
00433       break;
00434     case MVT::i16:
00435       Opcode = NVPTX::LD_i16_avar;
00436       break;
00437     case MVT::i32:
00438       Opcode = NVPTX::LD_i32_avar;
00439       break;
00440     case MVT::i64:
00441       Opcode = NVPTX::LD_i64_avar;
00442       break;
00443     case MVT::f32:
00444       Opcode = NVPTX::LD_f32_avar;
00445       break;
00446     case MVT::f64:
00447       Opcode = NVPTX::LD_f64_avar;
00448       break;
00449     default:
00450       return NULL;
00451     }
00452     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00453                       getI32Imm(vecType), getI32Imm(fromType),
00454                       getI32Imm(fromTypeWidth), Addr, Chain };
00455     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00456   } else if (Subtarget.is64Bit()
00457                  ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
00458                  : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
00459     switch (TargetVT) {
00460     case MVT::i8:
00461       Opcode = NVPTX::LD_i8_asi;
00462       break;
00463     case MVT::i16:
00464       Opcode = NVPTX::LD_i16_asi;
00465       break;
00466     case MVT::i32:
00467       Opcode = NVPTX::LD_i32_asi;
00468       break;
00469     case MVT::i64:
00470       Opcode = NVPTX::LD_i64_asi;
00471       break;
00472     case MVT::f32:
00473       Opcode = NVPTX::LD_f32_asi;
00474       break;
00475     case MVT::f64:
00476       Opcode = NVPTX::LD_f64_asi;
00477       break;
00478     default:
00479       return NULL;
00480     }
00481     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00482                       getI32Imm(vecType), getI32Imm(fromType),
00483                       getI32Imm(fromTypeWidth), Base, Offset, Chain };
00484     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00485   } else if (Subtarget.is64Bit()
00486                  ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
00487                  : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
00488     if (Subtarget.is64Bit()) {
00489       switch (TargetVT) {
00490       case MVT::i8:
00491         Opcode = NVPTX::LD_i8_ari_64;
00492         break;
00493       case MVT::i16:
00494         Opcode = NVPTX::LD_i16_ari_64;
00495         break;
00496       case MVT::i32:
00497         Opcode = NVPTX::LD_i32_ari_64;
00498         break;
00499       case MVT::i64:
00500         Opcode = NVPTX::LD_i64_ari_64;
00501         break;
00502       case MVT::f32:
00503         Opcode = NVPTX::LD_f32_ari_64;
00504         break;
00505       case MVT::f64:
00506         Opcode = NVPTX::LD_f64_ari_64;
00507         break;
00508       default:
00509         return NULL;
00510       }
00511     } else {
00512       switch (TargetVT) {
00513       case MVT::i8:
00514         Opcode = NVPTX::LD_i8_ari;
00515         break;
00516       case MVT::i16:
00517         Opcode = NVPTX::LD_i16_ari;
00518         break;
00519       case MVT::i32:
00520         Opcode = NVPTX::LD_i32_ari;
00521         break;
00522       case MVT::i64:
00523         Opcode = NVPTX::LD_i64_ari;
00524         break;
00525       case MVT::f32:
00526         Opcode = NVPTX::LD_f32_ari;
00527         break;
00528       case MVT::f64:
00529         Opcode = NVPTX::LD_f64_ari;
00530         break;
00531       default:
00532         return NULL;
00533       }
00534     }
00535     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00536                       getI32Imm(vecType), getI32Imm(fromType),
00537                       getI32Imm(fromTypeWidth), Base, Offset, Chain };
00538     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00539   } else {
00540     if (Subtarget.is64Bit()) {
00541       switch (TargetVT) {
00542       case MVT::i8:
00543         Opcode = NVPTX::LD_i8_areg_64;
00544         break;
00545       case MVT::i16:
00546         Opcode = NVPTX::LD_i16_areg_64;
00547         break;
00548       case MVT::i32:
00549         Opcode = NVPTX::LD_i32_areg_64;
00550         break;
00551       case MVT::i64:
00552         Opcode = NVPTX::LD_i64_areg_64;
00553         break;
00554       case MVT::f32:
00555         Opcode = NVPTX::LD_f32_areg_64;
00556         break;
00557       case MVT::f64:
00558         Opcode = NVPTX::LD_f64_areg_64;
00559         break;
00560       default:
00561         return NULL;
00562       }
00563     } else {
00564       switch (TargetVT) {
00565       case MVT::i8:
00566         Opcode = NVPTX::LD_i8_areg;
00567         break;
00568       case MVT::i16:
00569         Opcode = NVPTX::LD_i16_areg;
00570         break;
00571       case MVT::i32:
00572         Opcode = NVPTX::LD_i32_areg;
00573         break;
00574       case MVT::i64:
00575         Opcode = NVPTX::LD_i64_areg;
00576         break;
00577       case MVT::f32:
00578         Opcode = NVPTX::LD_f32_areg;
00579         break;
00580       case MVT::f64:
00581         Opcode = NVPTX::LD_f64_areg;
00582         break;
00583       default:
00584         return NULL;
00585       }
00586     }
00587     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00588                       getI32Imm(vecType), getI32Imm(fromType),
00589                       getI32Imm(fromTypeWidth), N1, Chain };
00590     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00591   }
00592 
00593   if (NVPTXLD != NULL) {
00594     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
00595     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
00596     cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
00597   }
00598 
00599   return NVPTXLD;
00600 }
00601 
00602 SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
00603 
00604   SDValue Chain = N->getOperand(0);
00605   SDValue Op1 = N->getOperand(1);
00606   SDValue Addr, Offset, Base;
00607   unsigned Opcode;
00608   SDLoc DL(N);
00609   SDNode *LD;
00610   MemSDNode *MemSD = cast<MemSDNode>(N);
00611   EVT LoadedVT = MemSD->getMemoryVT();
00612 
00613   if (!LoadedVT.isSimple())
00614     return NULL;
00615 
00616   // Address Space Setting
00617   unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
00618 
00619   // Volatile Setting
00620   // - .volatile is only availalble for .global and .shared
00621   bool IsVolatile = MemSD->isVolatile();
00622   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00623       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00624       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00625     IsVolatile = false;
00626 
00627   // Vector Setting
00628   MVT SimpleVT = LoadedVT.getSimpleVT();
00629 
00630   // Type Setting: fromType + fromTypeWidth
00631   //
00632   // Sign   : ISD::SEXTLOAD
00633   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00634   //          type is integer
00635   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00636   MVT ScalarVT = SimpleVT.getScalarType();
00637   // Read at least 8 bits (predicates are stored as 8-bit values)
00638   unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00639   unsigned int FromType;
00640   // The last operand holds the original LoadSDNode::getExtensionType() value
00641   unsigned ExtensionType = cast<ConstantSDNode>(
00642       N->getOperand(N->getNumOperands() - 1))->getZExtValue();
00643   if (ExtensionType == ISD::SEXTLOAD)
00644     FromType = NVPTX::PTXLdStInstCode::Signed;
00645   else if (ScalarVT.isFloatingPoint())
00646     FromType = NVPTX::PTXLdStInstCode::Float;
00647   else
00648     FromType = NVPTX::PTXLdStInstCode::Unsigned;
00649 
00650   unsigned VecType;
00651 
00652   switch (N->getOpcode()) {
00653   case NVPTXISD::LoadV2:
00654     VecType = NVPTX::PTXLdStInstCode::V2;
00655     break;
00656   case NVPTXISD::LoadV4:
00657     VecType = NVPTX::PTXLdStInstCode::V4;
00658     break;
00659   default:
00660     return NULL;
00661   }
00662 
00663   EVT EltVT = N->getValueType(0);
00664 
00665   if (SelectDirectAddr(Op1, Addr)) {
00666     switch (N->getOpcode()) {
00667     default:
00668       return NULL;
00669     case NVPTXISD::LoadV2:
00670       switch (EltVT.getSimpleVT().SimpleTy) {
00671       default:
00672         return NULL;
00673       case MVT::i8:
00674         Opcode = NVPTX::LDV_i8_v2_avar;
00675         break;
00676       case MVT::i16:
00677         Opcode = NVPTX::LDV_i16_v2_avar;
00678         break;
00679       case MVT::i32:
00680         Opcode = NVPTX::LDV_i32_v2_avar;
00681         break;
00682       case MVT::i64:
00683         Opcode = NVPTX::LDV_i64_v2_avar;
00684         break;
00685       case MVT::f32:
00686         Opcode = NVPTX::LDV_f32_v2_avar;
00687         break;
00688       case MVT::f64:
00689         Opcode = NVPTX::LDV_f64_v2_avar;
00690         break;
00691       }
00692       break;
00693     case NVPTXISD::LoadV4:
00694       switch (EltVT.getSimpleVT().SimpleTy) {
00695       default:
00696         return NULL;
00697       case MVT::i8:
00698         Opcode = NVPTX::LDV_i8_v4_avar;
00699         break;
00700       case MVT::i16:
00701         Opcode = NVPTX::LDV_i16_v4_avar;
00702         break;
00703       case MVT::i32:
00704         Opcode = NVPTX::LDV_i32_v4_avar;
00705         break;
00706       case MVT::f32:
00707         Opcode = NVPTX::LDV_f32_v4_avar;
00708         break;
00709       }
00710       break;
00711     }
00712 
00713     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
00714                       getI32Imm(VecType), getI32Imm(FromType),
00715                       getI32Imm(FromTypeWidth), Addr, Chain };
00716     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
00717   } else if (Subtarget.is64Bit()
00718                  ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
00719                  : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
00720     switch (N->getOpcode()) {
00721     default:
00722       return NULL;
00723     case NVPTXISD::LoadV2:
00724       switch (EltVT.getSimpleVT().SimpleTy) {
00725       default:
00726         return NULL;
00727       case MVT::i8:
00728         Opcode = NVPTX::LDV_i8_v2_asi;
00729         break;
00730       case MVT::i16:
00731         Opcode = NVPTX::LDV_i16_v2_asi;
00732         break;
00733       case MVT::i32:
00734         Opcode = NVPTX::LDV_i32_v2_asi;
00735         break;
00736       case MVT::i64:
00737         Opcode = NVPTX::LDV_i64_v2_asi;
00738         break;
00739       case MVT::f32:
00740         Opcode = NVPTX::LDV_f32_v2_asi;
00741         break;
00742       case MVT::f64:
00743         Opcode = NVPTX::LDV_f64_v2_asi;
00744         break;
00745       }
00746       break;
00747     case NVPTXISD::LoadV4:
00748       switch (EltVT.getSimpleVT().SimpleTy) {
00749       default:
00750         return NULL;
00751       case MVT::i8:
00752         Opcode = NVPTX::LDV_i8_v4_asi;
00753         break;
00754       case MVT::i16:
00755         Opcode = NVPTX::LDV_i16_v4_asi;
00756         break;
00757       case MVT::i32:
00758         Opcode = NVPTX::LDV_i32_v4_asi;
00759         break;
00760       case MVT::f32:
00761         Opcode = NVPTX::LDV_f32_v4_asi;
00762         break;
00763       }
00764       break;
00765     }
00766 
00767     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
00768                       getI32Imm(VecType), getI32Imm(FromType),
00769                       getI32Imm(FromTypeWidth), Base, Offset, Chain };
00770     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
00771   } else if (Subtarget.is64Bit()
00772                  ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
00773                  : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
00774     if (Subtarget.is64Bit()) {
00775       switch (N->getOpcode()) {
00776       default:
00777         return NULL;
00778       case NVPTXISD::LoadV2:
00779         switch (EltVT.getSimpleVT().SimpleTy) {
00780         default:
00781           return NULL;
00782         case MVT::i8:
00783           Opcode = NVPTX::LDV_i8_v2_ari_64;
00784           break;
00785         case MVT::i16:
00786           Opcode = NVPTX::LDV_i16_v2_ari_64;
00787           break;
00788         case MVT::i32:
00789           Opcode = NVPTX::LDV_i32_v2_ari_64;
00790           break;
00791         case MVT::i64:
00792           Opcode = NVPTX::LDV_i64_v2_ari_64;
00793           break;
00794         case MVT::f32:
00795           Opcode = NVPTX::LDV_f32_v2_ari_64;
00796           break;
00797         case MVT::f64:
00798           Opcode = NVPTX::LDV_f64_v2_ari_64;
00799           break;
00800         }
00801         break;
00802       case NVPTXISD::LoadV4:
00803         switch (EltVT.getSimpleVT().SimpleTy) {
00804         default:
00805           return NULL;
00806         case MVT::i8:
00807           Opcode = NVPTX::LDV_i8_v4_ari_64;
00808           break;
00809         case MVT::i16:
00810           Opcode = NVPTX::LDV_i16_v4_ari_64;
00811           break;
00812         case MVT::i32:
00813           Opcode = NVPTX::LDV_i32_v4_ari_64;
00814           break;
00815         case MVT::f32:
00816           Opcode = NVPTX::LDV_f32_v4_ari_64;
00817           break;
00818         }
00819         break;
00820       }
00821     } else {
00822       switch (N->getOpcode()) {
00823       default:
00824         return NULL;
00825       case NVPTXISD::LoadV2:
00826         switch (EltVT.getSimpleVT().SimpleTy) {
00827         default:
00828           return NULL;
00829         case MVT::i8:
00830           Opcode = NVPTX::LDV_i8_v2_ari;
00831           break;
00832         case MVT::i16:
00833           Opcode = NVPTX::LDV_i16_v2_ari;
00834           break;
00835         case MVT::i32:
00836           Opcode = NVPTX::LDV_i32_v2_ari;
00837           break;
00838         case MVT::i64:
00839           Opcode = NVPTX::LDV_i64_v2_ari;
00840           break;
00841         case MVT::f32:
00842           Opcode = NVPTX::LDV_f32_v2_ari;
00843           break;
00844         case MVT::f64:
00845           Opcode = NVPTX::LDV_f64_v2_ari;
00846           break;
00847         }
00848         break;
00849       case NVPTXISD::LoadV4:
00850         switch (EltVT.getSimpleVT().SimpleTy) {
00851         default:
00852           return NULL;
00853         case MVT::i8:
00854           Opcode = NVPTX::LDV_i8_v4_ari;
00855           break;
00856         case MVT::i16:
00857           Opcode = NVPTX::LDV_i16_v4_ari;
00858           break;
00859         case MVT::i32:
00860           Opcode = NVPTX::LDV_i32_v4_ari;
00861           break;
00862         case MVT::f32:
00863           Opcode = NVPTX::LDV_f32_v4_ari;
00864           break;
00865         }
00866         break;
00867       }
00868     }
00869 
00870     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
00871                       getI32Imm(VecType), getI32Imm(FromType),
00872                       getI32Imm(FromTypeWidth), Base, Offset, Chain };
00873 
00874     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
00875   } else {
00876     if (Subtarget.is64Bit()) {
00877       switch (N->getOpcode()) {
00878       default:
00879         return NULL;
00880       case NVPTXISD::LoadV2:
00881         switch (EltVT.getSimpleVT().SimpleTy) {
00882         default:
00883           return NULL;
00884         case MVT::i8:
00885           Opcode = NVPTX::LDV_i8_v2_areg_64;
00886           break;
00887         case MVT::i16:
00888           Opcode = NVPTX::LDV_i16_v2_areg_64;
00889           break;
00890         case MVT::i32:
00891           Opcode = NVPTX::LDV_i32_v2_areg_64;
00892           break;
00893         case MVT::i64:
00894           Opcode = NVPTX::LDV_i64_v2_areg_64;
00895           break;
00896         case MVT::f32:
00897           Opcode = NVPTX::LDV_f32_v2_areg_64;
00898           break;
00899         case MVT::f64:
00900           Opcode = NVPTX::LDV_f64_v2_areg_64;
00901           break;
00902         }
00903         break;
00904       case NVPTXISD::LoadV4:
00905         switch (EltVT.getSimpleVT().SimpleTy) {
00906         default:
00907           return NULL;
00908         case MVT::i8:
00909           Opcode = NVPTX::LDV_i8_v4_areg_64;
00910           break;
00911         case MVT::i16:
00912           Opcode = NVPTX::LDV_i16_v4_areg_64;
00913           break;
00914         case MVT::i32:
00915           Opcode = NVPTX::LDV_i32_v4_areg_64;
00916           break;
00917         case MVT::f32:
00918           Opcode = NVPTX::LDV_f32_v4_areg_64;
00919           break;
00920         }
00921         break;
00922       }
00923     } else {
00924       switch (N->getOpcode()) {
00925       default:
00926         return NULL;
00927       case NVPTXISD::LoadV2:
00928         switch (EltVT.getSimpleVT().SimpleTy) {
00929         default:
00930           return NULL;
00931         case MVT::i8:
00932           Opcode = NVPTX::LDV_i8_v2_areg;
00933           break;
00934         case MVT::i16:
00935           Opcode = NVPTX::LDV_i16_v2_areg;
00936           break;
00937         case MVT::i32:
00938           Opcode = NVPTX::LDV_i32_v2_areg;
00939           break;
00940         case MVT::i64:
00941           Opcode = NVPTX::LDV_i64_v2_areg;
00942           break;
00943         case MVT::f32:
00944           Opcode = NVPTX::LDV_f32_v2_areg;
00945           break;
00946         case MVT::f64:
00947           Opcode = NVPTX::LDV_f64_v2_areg;
00948           break;
00949         }
00950         break;
00951       case NVPTXISD::LoadV4:
00952         switch (EltVT.getSimpleVT().SimpleTy) {
00953         default:
00954           return NULL;
00955         case MVT::i8:
00956           Opcode = NVPTX::LDV_i8_v4_areg;
00957           break;
00958         case MVT::i16:
00959           Opcode = NVPTX::LDV_i16_v4_areg;
00960           break;
00961         case MVT::i32:
00962           Opcode = NVPTX::LDV_i32_v4_areg;
00963           break;
00964         case MVT::f32:
00965           Opcode = NVPTX::LDV_f32_v4_areg;
00966           break;
00967         }
00968         break;
00969       }
00970     }
00971 
00972     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
00973                       getI32Imm(VecType), getI32Imm(FromType),
00974                       getI32Imm(FromTypeWidth), Op1, Chain };
00975     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
00976   }
00977 
00978   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
00979   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
00980   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
00981 
00982   return LD;
00983 }
00984 
00985 SDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) {
00986 
00987   SDValue Chain = N->getOperand(0);
00988   SDValue Op1 = N->getOperand(1);
00989   unsigned Opcode;
00990   SDLoc DL(N);
00991   SDNode *LD;
00992   MemSDNode *Mem = cast<MemSDNode>(N);
00993   SDValue Base, Offset, Addr;
00994 
00995   EVT EltVT = Mem->getMemoryVT().getVectorElementType();
00996 
00997   if (SelectDirectAddr(Op1, Addr)) {
00998     switch (N->getOpcode()) {
00999     default:
01000       return NULL;
01001     case NVPTXISD::LDGV2:
01002       switch (EltVT.getSimpleVT().SimpleTy) {
01003       default:
01004         return NULL;
01005       case MVT::i8:
01006         Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
01007         break;
01008       case MVT::i16:
01009         Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
01010         break;
01011       case MVT::i32:
01012         Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
01013         break;
01014       case MVT::i64:
01015         Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
01016         break;
01017       case MVT::f32:
01018         Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
01019         break;
01020       case MVT::f64:
01021         Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
01022         break;
01023       }
01024       break;
01025     case NVPTXISD::LDUV2:
01026       switch (EltVT.getSimpleVT().SimpleTy) {
01027       default:
01028         return NULL;
01029       case MVT::i8:
01030         Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
01031         break;
01032       case MVT::i16:
01033         Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
01034         break;
01035       case MVT::i32:
01036         Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
01037         break;
01038       case MVT::i64:
01039         Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
01040         break;
01041       case MVT::f32:
01042         Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
01043         break;
01044       case MVT::f64:
01045         Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
01046         break;
01047       }
01048       break;
01049     case NVPTXISD::LDGV4:
01050       switch (EltVT.getSimpleVT().SimpleTy) {
01051       default:
01052         return NULL;
01053       case MVT::i8:
01054         Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
01055         break;
01056       case MVT::i16:
01057         Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
01058         break;
01059       case MVT::i32:
01060         Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
01061         break;
01062       case MVT::f32:
01063         Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
01064         break;
01065       }
01066       break;
01067     case NVPTXISD::LDUV4:
01068       switch (EltVT.getSimpleVT().SimpleTy) {
01069       default:
01070         return NULL;
01071       case MVT::i8:
01072         Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
01073         break;
01074       case MVT::i16:
01075         Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
01076         break;
01077       case MVT::i32:
01078         Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
01079         break;
01080       case MVT::f32:
01081         Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
01082         break;
01083       }
01084       break;
01085     }
01086 
01087     SDValue Ops[] = { Addr, Chain };
01088     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(),
01089                                 ArrayRef<SDValue>(Ops, 2));
01090   } else if (Subtarget.is64Bit()
01091                  ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
01092                  : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
01093     if (Subtarget.is64Bit()) {
01094       switch (N->getOpcode()) {
01095       default:
01096         return NULL;
01097       case NVPTXISD::LDGV2:
01098         switch (EltVT.getSimpleVT().SimpleTy) {
01099         default:
01100           return NULL;
01101         case MVT::i8:
01102           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
01103           break;
01104         case MVT::i16:
01105           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
01106           break;
01107         case MVT::i32:
01108           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
01109           break;
01110         case MVT::i64:
01111           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
01112           break;
01113         case MVT::f32:
01114           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
01115           break;
01116         case MVT::f64:
01117           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
01118           break;
01119         }
01120         break;
01121       case NVPTXISD::LDUV2:
01122         switch (EltVT.getSimpleVT().SimpleTy) {
01123         default:
01124           return NULL;
01125         case MVT::i8:
01126           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
01127           break;
01128         case MVT::i16:
01129           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
01130           break;
01131         case MVT::i32:
01132           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
01133           break;
01134         case MVT::i64:
01135           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
01136           break;
01137         case MVT::f32:
01138           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
01139           break;
01140         case MVT::f64:
01141           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
01142           break;
01143         }
01144         break;
01145       case NVPTXISD::LDGV4:
01146         switch (EltVT.getSimpleVT().SimpleTy) {
01147         default:
01148           return NULL;
01149         case MVT::i8:
01150           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
01151           break;
01152         case MVT::i16:
01153           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
01154           break;
01155         case MVT::i32:
01156           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
01157           break;
01158         case MVT::f32:
01159           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
01160           break;
01161         }
01162         break;
01163       case NVPTXISD::LDUV4:
01164         switch (EltVT.getSimpleVT().SimpleTy) {
01165         default:
01166           return NULL;
01167         case MVT::i8:
01168           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
01169           break;
01170         case MVT::i16:
01171           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
01172           break;
01173         case MVT::i32:
01174           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
01175           break;
01176         case MVT::f32:
01177           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
01178           break;
01179         }
01180         break;
01181       }
01182     } else {
01183       switch (N->getOpcode()) {
01184       default:
01185         return NULL;
01186       case NVPTXISD::LDGV2:
01187         switch (EltVT.getSimpleVT().SimpleTy) {
01188         default:
01189           return NULL;
01190         case MVT::i8:
01191           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
01192           break;
01193         case MVT::i16:
01194           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
01195           break;
01196         case MVT::i32:
01197           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
01198           break;
01199         case MVT::i64:
01200           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
01201           break;
01202         case MVT::f32:
01203           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
01204           break;
01205         case MVT::f64:
01206           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
01207           break;
01208         }
01209         break;
01210       case NVPTXISD::LDUV2:
01211         switch (EltVT.getSimpleVT().SimpleTy) {
01212         default:
01213           return NULL;
01214         case MVT::i8:
01215           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
01216           break;
01217         case MVT::i16:
01218           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
01219           break;
01220         case MVT::i32:
01221           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
01222           break;
01223         case MVT::i64:
01224           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
01225           break;
01226         case MVT::f32:
01227           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
01228           break;
01229         case MVT::f64:
01230           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
01231           break;
01232         }
01233         break;
01234       case NVPTXISD::LDGV4:
01235         switch (EltVT.getSimpleVT().SimpleTy) {
01236         default:
01237           return NULL;
01238         case MVT::i8:
01239           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
01240           break;
01241         case MVT::i16:
01242           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
01243           break;
01244         case MVT::i32:
01245           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
01246           break;
01247         case MVT::f32:
01248           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
01249           break;
01250         }
01251         break;
01252       case NVPTXISD::LDUV4:
01253         switch (EltVT.getSimpleVT().SimpleTy) {
01254         default:
01255           return NULL;
01256         case MVT::i8:
01257           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
01258           break;
01259         case MVT::i16:
01260           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
01261           break;
01262         case MVT::i32:
01263           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
01264           break;
01265         case MVT::f32:
01266           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
01267           break;
01268         }
01269         break;
01270       }
01271     }
01272 
01273     SDValue Ops[] = { Base, Offset, Chain };
01274 
01275     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(),
01276                                 ArrayRef<SDValue>(Ops, 3));
01277   } else {
01278     if (Subtarget.is64Bit()) {
01279       switch (N->getOpcode()) {
01280       default:
01281         return NULL;
01282       case NVPTXISD::LDGV2:
01283         switch (EltVT.getSimpleVT().SimpleTy) {
01284         default:
01285           return NULL;
01286         case MVT::i8:
01287           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
01288           break;
01289         case MVT::i16:
01290           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
01291           break;
01292         case MVT::i32:
01293           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
01294           break;
01295         case MVT::i64:
01296           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
01297           break;
01298         case MVT::f32:
01299           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
01300           break;
01301         case MVT::f64:
01302           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
01303           break;
01304         }
01305         break;
01306       case NVPTXISD::LDUV2:
01307         switch (EltVT.getSimpleVT().SimpleTy) {
01308         default:
01309           return NULL;
01310         case MVT::i8:
01311           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
01312           break;
01313         case MVT::i16:
01314           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
01315           break;
01316         case MVT::i32:
01317           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
01318           break;
01319         case MVT::i64:
01320           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
01321           break;
01322         case MVT::f32:
01323           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
01324           break;
01325         case MVT::f64:
01326           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
01327           break;
01328         }
01329         break;
01330       case NVPTXISD::LDGV4:
01331         switch (EltVT.getSimpleVT().SimpleTy) {
01332         default:
01333           return NULL;
01334         case MVT::i8:
01335           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
01336           break;
01337         case MVT::i16:
01338           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
01339           break;
01340         case MVT::i32:
01341           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
01342           break;
01343         case MVT::f32:
01344           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
01345           break;
01346         }
01347         break;
01348       case NVPTXISD::LDUV4:
01349         switch (EltVT.getSimpleVT().SimpleTy) {
01350         default:
01351           return NULL;
01352         case MVT::i8:
01353           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
01354           break;
01355         case MVT::i16:
01356           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
01357           break;
01358         case MVT::i32:
01359           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
01360           break;
01361         case MVT::f32:
01362           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
01363           break;
01364         }
01365         break;
01366       }
01367     } else {
01368       switch (N->getOpcode()) {
01369       default:
01370         return NULL;
01371       case NVPTXISD::LDGV2:
01372         switch (EltVT.getSimpleVT().SimpleTy) {
01373         default:
01374           return NULL;
01375         case MVT::i8:
01376           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
01377           break;
01378         case MVT::i16:
01379           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
01380           break;
01381         case MVT::i32:
01382           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
01383           break;
01384         case MVT::i64:
01385           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
01386           break;
01387         case MVT::f32:
01388           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
01389           break;
01390         case MVT::f64:
01391           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
01392           break;
01393         }
01394         break;
01395       case NVPTXISD::LDUV2:
01396         switch (EltVT.getSimpleVT().SimpleTy) {
01397         default:
01398           return NULL;
01399         case MVT::i8:
01400           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
01401           break;
01402         case MVT::i16:
01403           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
01404           break;
01405         case MVT::i32:
01406           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
01407           break;
01408         case MVT::i64:
01409           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
01410           break;
01411         case MVT::f32:
01412           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
01413           break;
01414         case MVT::f64:
01415           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
01416           break;
01417         }
01418         break;
01419       case NVPTXISD::LDGV4:
01420         switch (EltVT.getSimpleVT().SimpleTy) {
01421         default:
01422           return NULL;
01423         case MVT::i8:
01424           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
01425           break;
01426         case MVT::i16:
01427           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
01428           break;
01429         case MVT::i32:
01430           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
01431           break;
01432         case MVT::f32:
01433           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
01434           break;
01435         }
01436         break;
01437       case NVPTXISD::LDUV4:
01438         switch (EltVT.getSimpleVT().SimpleTy) {
01439         default:
01440           return NULL;
01441         case MVT::i8:
01442           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
01443           break;
01444         case MVT::i16:
01445           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
01446           break;
01447         case MVT::i32:
01448           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
01449           break;
01450         case MVT::f32:
01451           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
01452           break;
01453         }
01454         break;
01455       }
01456     }
01457 
01458     SDValue Ops[] = { Op1, Chain };
01459     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(),
01460                                 ArrayRef<SDValue>(Ops, 2));
01461   }
01462 
01463   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01464   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
01465   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
01466 
01467   return LD;
01468 }
01469 
01470 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
01471   SDLoc dl(N);
01472   StoreSDNode *ST = cast<StoreSDNode>(N);
01473   EVT StoreVT = ST->getMemoryVT();
01474   SDNode *NVPTXST = NULL;
01475 
01476   // do not support pre/post inc/dec
01477   if (ST->isIndexed())
01478     return NULL;
01479 
01480   if (!StoreVT.isSimple())
01481     return NULL;
01482 
01483   // Address Space Setting
01484   unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget);
01485 
01486   // Volatile Setting
01487   // - .volatile is only availalble for .global and .shared
01488   bool isVolatile = ST->isVolatile();
01489   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
01490       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
01491       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
01492     isVolatile = false;
01493 
01494   // Vector Setting
01495   MVT SimpleVT = StoreVT.getSimpleVT();
01496   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
01497   if (SimpleVT.isVector()) {
01498     unsigned num = SimpleVT.getVectorNumElements();
01499     if (num == 2)
01500       vecType = NVPTX::PTXLdStInstCode::V2;
01501     else if (num == 4)
01502       vecType = NVPTX::PTXLdStInstCode::V4;
01503     else
01504       return NULL;
01505   }
01506 
01507   // Type Setting: toType + toTypeWidth
01508   // - for integer type, always use 'u'
01509   //
01510   MVT ScalarVT = SimpleVT.getScalarType();
01511   unsigned toTypeWidth = ScalarVT.getSizeInBits();
01512   unsigned int toType;
01513   if (ScalarVT.isFloatingPoint())
01514     toType = NVPTX::PTXLdStInstCode::Float;
01515   else
01516     toType = NVPTX::PTXLdStInstCode::Unsigned;
01517 
01518   // Create the machine instruction DAG
01519   SDValue Chain = N->getOperand(0);
01520   SDValue N1 = N->getOperand(1);
01521   SDValue N2 = N->getOperand(2);
01522   SDValue Addr;
01523   SDValue Offset, Base;
01524   unsigned Opcode;
01525   MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
01526 
01527   if (SelectDirectAddr(N2, Addr)) {
01528     switch (SourceVT) {
01529     case MVT::i8:
01530       Opcode = NVPTX::ST_i8_avar;
01531       break;
01532     case MVT::i16:
01533       Opcode = NVPTX::ST_i16_avar;
01534       break;
01535     case MVT::i32:
01536       Opcode = NVPTX::ST_i32_avar;
01537       break;
01538     case MVT::i64:
01539       Opcode = NVPTX::ST_i64_avar;
01540       break;
01541     case MVT::f32:
01542       Opcode = NVPTX::ST_f32_avar;
01543       break;
01544     case MVT::f64:
01545       Opcode = NVPTX::ST_f64_avar;
01546       break;
01547     default:
01548       return NULL;
01549     }
01550     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
01551                       getI32Imm(vecType), getI32Imm(toType),
01552                       getI32Imm(toTypeWidth), Addr, Chain };
01553     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
01554   } else if (Subtarget.is64Bit()
01555                  ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
01556                  : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
01557     switch (SourceVT) {
01558     case MVT::i8:
01559       Opcode = NVPTX::ST_i8_asi;
01560       break;
01561     case MVT::i16:
01562       Opcode = NVPTX::ST_i16_asi;
01563       break;
01564     case MVT::i32:
01565       Opcode = NVPTX::ST_i32_asi;
01566       break;
01567     case MVT::i64:
01568       Opcode = NVPTX::ST_i64_asi;
01569       break;
01570     case MVT::f32:
01571       Opcode = NVPTX::ST_f32_asi;
01572       break;
01573     case MVT::f64:
01574       Opcode = NVPTX::ST_f64_asi;
01575       break;
01576     default:
01577       return NULL;
01578     }
01579     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
01580                       getI32Imm(vecType), getI32Imm(toType),
01581                       getI32Imm(toTypeWidth), Base, Offset, Chain };
01582     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
01583   } else if (Subtarget.is64Bit()
01584                  ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
01585                  : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
01586     if (Subtarget.is64Bit()) {
01587       switch (SourceVT) {
01588       case MVT::i8:
01589         Opcode = NVPTX::ST_i8_ari_64;
01590         break;
01591       case MVT::i16:
01592         Opcode = NVPTX::ST_i16_ari_64;
01593         break;
01594       case MVT::i32:
01595         Opcode = NVPTX::ST_i32_ari_64;
01596         break;
01597       case MVT::i64:
01598         Opcode = NVPTX::ST_i64_ari_64;
01599         break;
01600       case MVT::f32:
01601         Opcode = NVPTX::ST_f32_ari_64;
01602         break;
01603       case MVT::f64:
01604         Opcode = NVPTX::ST_f64_ari_64;
01605         break;
01606       default:
01607         return NULL;
01608       }
01609     } else {
01610       switch (SourceVT) {
01611       case MVT::i8:
01612         Opcode = NVPTX::ST_i8_ari;
01613         break;
01614       case MVT::i16:
01615         Opcode = NVPTX::ST_i16_ari;
01616         break;
01617       case MVT::i32:
01618         Opcode = NVPTX::ST_i32_ari;
01619         break;
01620       case MVT::i64:
01621         Opcode = NVPTX::ST_i64_ari;
01622         break;
01623       case MVT::f32:
01624         Opcode = NVPTX::ST_f32_ari;
01625         break;
01626       case MVT::f64:
01627         Opcode = NVPTX::ST_f64_ari;
01628         break;
01629       default:
01630         return NULL;
01631       }
01632     }
01633     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
01634                       getI32Imm(vecType), getI32Imm(toType),
01635                       getI32Imm(toTypeWidth), Base, Offset, Chain };
01636     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
01637   } else {
01638     if (Subtarget.is64Bit()) {
01639       switch (SourceVT) {
01640       case MVT::i8:
01641         Opcode = NVPTX::ST_i8_areg_64;
01642         break;
01643       case MVT::i16:
01644         Opcode = NVPTX::ST_i16_areg_64;
01645         break;
01646       case MVT::i32:
01647         Opcode = NVPTX::ST_i32_areg_64;
01648         break;
01649       case MVT::i64:
01650         Opcode = NVPTX::ST_i64_areg_64;
01651         break;
01652       case MVT::f32:
01653         Opcode = NVPTX::ST_f32_areg_64;
01654         break;
01655       case MVT::f64:
01656         Opcode = NVPTX::ST_f64_areg_64;
01657         break;
01658       default:
01659         return NULL;
01660       }
01661     } else {
01662       switch (SourceVT) {
01663       case MVT::i8:
01664         Opcode = NVPTX::ST_i8_areg;
01665         break;
01666       case MVT::i16:
01667         Opcode = NVPTX::ST_i16_areg;
01668         break;
01669       case MVT::i32:
01670         Opcode = NVPTX::ST_i32_areg;
01671         break;
01672       case MVT::i64:
01673         Opcode = NVPTX::ST_i64_areg;
01674         break;
01675       case MVT::f32:
01676         Opcode = NVPTX::ST_f32_areg;
01677         break;
01678       case MVT::f64:
01679         Opcode = NVPTX::ST_f64_areg;
01680         break;
01681       default:
01682         return NULL;
01683       }
01684     }
01685     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
01686                       getI32Imm(vecType), getI32Imm(toType),
01687                       getI32Imm(toTypeWidth), N2, Chain };
01688     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
01689   }
01690 
01691   if (NVPTXST != NULL) {
01692     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01693     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
01694     cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
01695   }
01696 
01697   return NVPTXST;
01698 }
01699 
01700 SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
01701   SDValue Chain = N->getOperand(0);
01702   SDValue Op1 = N->getOperand(1);
01703   SDValue Addr, Offset, Base;
01704   unsigned Opcode;
01705   SDLoc DL(N);
01706   SDNode *ST;
01707   EVT EltVT = Op1.getValueType();
01708   MemSDNode *MemSD = cast<MemSDNode>(N);
01709   EVT StoreVT = MemSD->getMemoryVT();
01710 
01711   // Address Space Setting
01712   unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
01713 
01714   if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
01715     report_fatal_error("Cannot store to pointer that points to constant "
01716                        "memory space");
01717   }
01718 
01719   // Volatile Setting
01720   // - .volatile is only availalble for .global and .shared
01721   bool IsVolatile = MemSD->isVolatile();
01722   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
01723       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
01724       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
01725     IsVolatile = false;
01726 
01727   // Type Setting: toType + toTypeWidth
01728   // - for integer type, always use 'u'
01729   assert(StoreVT.isSimple() && "Store value is not simple");
01730   MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
01731   unsigned ToTypeWidth = ScalarVT.getSizeInBits();
01732   unsigned ToType;
01733   if (ScalarVT.isFloatingPoint())
01734     ToType = NVPTX::PTXLdStInstCode::Float;
01735   else
01736     ToType = NVPTX::PTXLdStInstCode::Unsigned;
01737 
01738   SmallVector<SDValue, 12> StOps;
01739   SDValue N2;
01740   unsigned VecType;
01741 
01742   switch (N->getOpcode()) {
01743   case NVPTXISD::StoreV2:
01744     VecType = NVPTX::PTXLdStInstCode::V2;
01745     StOps.push_back(N->getOperand(1));
01746     StOps.push_back(N->getOperand(2));
01747     N2 = N->getOperand(3);
01748     break;
01749   case NVPTXISD::StoreV4:
01750     VecType = NVPTX::PTXLdStInstCode::V4;
01751     StOps.push_back(N->getOperand(1));
01752     StOps.push_back(N->getOperand(2));
01753     StOps.push_back(N->getOperand(3));
01754     StOps.push_back(N->getOperand(4));
01755     N2 = N->getOperand(5);
01756     break;
01757   default:
01758     return NULL;
01759   }
01760 
01761   StOps.push_back(getI32Imm(IsVolatile));
01762   StOps.push_back(getI32Imm(CodeAddrSpace));
01763   StOps.push_back(getI32Imm(VecType));
01764   StOps.push_back(getI32Imm(ToType));
01765   StOps.push_back(getI32Imm(ToTypeWidth));
01766 
01767   if (SelectDirectAddr(N2, Addr)) {
01768     switch (N->getOpcode()) {
01769     default:
01770       return NULL;
01771     case NVPTXISD::StoreV2:
01772       switch (EltVT.getSimpleVT().SimpleTy) {
01773       default:
01774         return NULL;
01775       case MVT::i8:
01776         Opcode = NVPTX::STV_i8_v2_avar;
01777         break;
01778       case MVT::i16:
01779         Opcode = NVPTX::STV_i16_v2_avar;
01780         break;
01781       case MVT::i32:
01782         Opcode = NVPTX::STV_i32_v2_avar;
01783         break;
01784       case MVT::i64:
01785         Opcode = NVPTX::STV_i64_v2_avar;
01786         break;
01787       case MVT::f32:
01788         Opcode = NVPTX::STV_f32_v2_avar;
01789         break;
01790       case MVT::f64:
01791         Opcode = NVPTX::STV_f64_v2_avar;
01792         break;
01793       }
01794       break;
01795     case NVPTXISD::StoreV4:
01796       switch (EltVT.getSimpleVT().SimpleTy) {
01797       default:
01798         return NULL;
01799       case MVT::i8:
01800         Opcode = NVPTX::STV_i8_v4_avar;
01801         break;
01802       case MVT::i16:
01803         Opcode = NVPTX::STV_i16_v4_avar;
01804         break;
01805       case MVT::i32:
01806         Opcode = NVPTX::STV_i32_v4_avar;
01807         break;
01808       case MVT::f32:
01809         Opcode = NVPTX::STV_f32_v4_avar;
01810         break;
01811       }
01812       break;
01813     }
01814     StOps.push_back(Addr);
01815   } else if (Subtarget.is64Bit()
01816                  ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
01817                  : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
01818     switch (N->getOpcode()) {
01819     default:
01820       return NULL;
01821     case NVPTXISD::StoreV2:
01822       switch (EltVT.getSimpleVT().SimpleTy) {
01823       default:
01824         return NULL;
01825       case MVT::i8:
01826         Opcode = NVPTX::STV_i8_v2_asi;
01827         break;
01828       case MVT::i16:
01829         Opcode = NVPTX::STV_i16_v2_asi;
01830         break;
01831       case MVT::i32:
01832         Opcode = NVPTX::STV_i32_v2_asi;
01833         break;
01834       case MVT::i64:
01835         Opcode = NVPTX::STV_i64_v2_asi;
01836         break;
01837       case MVT::f32:
01838         Opcode = NVPTX::STV_f32_v2_asi;
01839         break;
01840       case MVT::f64:
01841         Opcode = NVPTX::STV_f64_v2_asi;
01842         break;
01843       }
01844       break;
01845     case NVPTXISD::StoreV4:
01846       switch (EltVT.getSimpleVT().SimpleTy) {
01847       default:
01848         return NULL;
01849       case MVT::i8:
01850         Opcode = NVPTX::STV_i8_v4_asi;
01851         break;
01852       case MVT::i16:
01853         Opcode = NVPTX::STV_i16_v4_asi;
01854         break;
01855       case MVT::i32:
01856         Opcode = NVPTX::STV_i32_v4_asi;
01857         break;
01858       case MVT::f32:
01859         Opcode = NVPTX::STV_f32_v4_asi;
01860         break;
01861       }
01862       break;
01863     }
01864     StOps.push_back(Base);
01865     StOps.push_back(Offset);
01866   } else if (Subtarget.is64Bit()
01867                  ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
01868                  : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
01869     if (Subtarget.is64Bit()) {
01870       switch (N->getOpcode()) {
01871       default:
01872         return NULL;
01873       case NVPTXISD::StoreV2:
01874         switch (EltVT.getSimpleVT().SimpleTy) {
01875         default:
01876           return NULL;
01877         case MVT::i8:
01878           Opcode = NVPTX::STV_i8_v2_ari_64;
01879           break;
01880         case MVT::i16:
01881           Opcode = NVPTX::STV_i16_v2_ari_64;
01882           break;
01883         case MVT::i32:
01884           Opcode = NVPTX::STV_i32_v2_ari_64;
01885           break;
01886         case MVT::i64:
01887           Opcode = NVPTX::STV_i64_v2_ari_64;
01888           break;
01889         case MVT::f32:
01890           Opcode = NVPTX::STV_f32_v2_ari_64;
01891           break;
01892         case MVT::f64:
01893           Opcode = NVPTX::STV_f64_v2_ari_64;
01894           break;
01895         }
01896         break;
01897       case NVPTXISD::StoreV4:
01898         switch (EltVT.getSimpleVT().SimpleTy) {
01899         default:
01900           return NULL;
01901         case MVT::i8:
01902           Opcode = NVPTX::STV_i8_v4_ari_64;
01903           break;
01904         case MVT::i16:
01905           Opcode = NVPTX::STV_i16_v4_ari_64;
01906           break;
01907         case MVT::i32:
01908           Opcode = NVPTX::STV_i32_v4_ari_64;
01909           break;
01910         case MVT::f32:
01911           Opcode = NVPTX::STV_f32_v4_ari_64;
01912           break;
01913         }
01914         break;
01915       }
01916     } else {
01917       switch (N->getOpcode()) {
01918       default:
01919         return NULL;
01920       case NVPTXISD::StoreV2:
01921         switch (EltVT.getSimpleVT().SimpleTy) {
01922         default:
01923           return NULL;
01924         case MVT::i8:
01925           Opcode = NVPTX::STV_i8_v2_ari;
01926           break;
01927         case MVT::i16:
01928           Opcode = NVPTX::STV_i16_v2_ari;
01929           break;
01930         case MVT::i32:
01931           Opcode = NVPTX::STV_i32_v2_ari;
01932           break;
01933         case MVT::i64:
01934           Opcode = NVPTX::STV_i64_v2_ari;
01935           break;
01936         case MVT::f32:
01937           Opcode = NVPTX::STV_f32_v2_ari;
01938           break;
01939         case MVT::f64:
01940           Opcode = NVPTX::STV_f64_v2_ari;
01941           break;
01942         }
01943         break;
01944       case NVPTXISD::StoreV4:
01945         switch (EltVT.getSimpleVT().SimpleTy) {
01946         default:
01947           return NULL;
01948         case MVT::i8:
01949           Opcode = NVPTX::STV_i8_v4_ari;
01950           break;
01951         case MVT::i16:
01952           Opcode = NVPTX::STV_i16_v4_ari;
01953           break;
01954         case MVT::i32:
01955           Opcode = NVPTX::STV_i32_v4_ari;
01956           break;
01957         case MVT::f32:
01958           Opcode = NVPTX::STV_f32_v4_ari;
01959           break;
01960         }
01961         break;
01962       }
01963     }
01964     StOps.push_back(Base);
01965     StOps.push_back(Offset);
01966   } else {
01967     if (Subtarget.is64Bit()) {
01968       switch (N->getOpcode()) {
01969       default:
01970         return NULL;
01971       case NVPTXISD::StoreV2:
01972         switch (EltVT.getSimpleVT().SimpleTy) {
01973         default:
01974           return NULL;
01975         case MVT::i8:
01976           Opcode = NVPTX::STV_i8_v2_areg_64;
01977           break;
01978         case MVT::i16:
01979           Opcode = NVPTX::STV_i16_v2_areg_64;
01980           break;
01981         case MVT::i32:
01982           Opcode = NVPTX::STV_i32_v2_areg_64;
01983           break;
01984         case MVT::i64:
01985           Opcode = NVPTX::STV_i64_v2_areg_64;
01986           break;
01987         case MVT::f32:
01988           Opcode = NVPTX::STV_f32_v2_areg_64;
01989           break;
01990         case MVT::f64:
01991           Opcode = NVPTX::STV_f64_v2_areg_64;
01992           break;
01993         }
01994         break;
01995       case NVPTXISD::StoreV4:
01996         switch (EltVT.getSimpleVT().SimpleTy) {
01997         default:
01998           return NULL;
01999         case MVT::i8:
02000           Opcode = NVPTX::STV_i8_v4_areg_64;
02001           break;
02002         case MVT::i16:
02003           Opcode = NVPTX::STV_i16_v4_areg_64;
02004           break;
02005         case MVT::i32:
02006           Opcode = NVPTX::STV_i32_v4_areg_64;
02007           break;
02008         case MVT::f32:
02009           Opcode = NVPTX::STV_f32_v4_areg_64;
02010           break;
02011         }
02012         break;
02013       }
02014     } else {
02015       switch (N->getOpcode()) {
02016       default:
02017         return NULL;
02018       case NVPTXISD::StoreV2:
02019         switch (EltVT.getSimpleVT().SimpleTy) {
02020         default:
02021           return NULL;
02022         case MVT::i8:
02023           Opcode = NVPTX::STV_i8_v2_areg;
02024           break;
02025         case MVT::i16:
02026           Opcode = NVPTX::STV_i16_v2_areg;
02027           break;
02028         case MVT::i32:
02029           Opcode = NVPTX::STV_i32_v2_areg;
02030           break;
02031         case MVT::i64:
02032           Opcode = NVPTX::STV_i64_v2_areg;
02033           break;
02034         case MVT::f32:
02035           Opcode = NVPTX::STV_f32_v2_areg;
02036           break;
02037         case MVT::f64:
02038           Opcode = NVPTX::STV_f64_v2_areg;
02039           break;
02040         }
02041         break;
02042       case NVPTXISD::StoreV4:
02043         switch (EltVT.getSimpleVT().SimpleTy) {
02044         default:
02045           return NULL;
02046         case MVT::i8:
02047           Opcode = NVPTX::STV_i8_v4_areg;
02048           break;
02049         case MVT::i16:
02050           Opcode = NVPTX::STV_i16_v4_areg;
02051           break;
02052         case MVT::i32:
02053           Opcode = NVPTX::STV_i32_v4_areg;
02054           break;
02055         case MVT::f32:
02056           Opcode = NVPTX::STV_f32_v4_areg;
02057           break;
02058         }
02059         break;
02060       }
02061     }
02062     StOps.push_back(N2);
02063   }
02064 
02065   StOps.push_back(Chain);
02066 
02067   ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
02068 
02069   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02070   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02071   cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
02072 
02073   return ST;
02074 }
02075 
02076 SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
02077   SDValue Chain = Node->getOperand(0);
02078   SDValue Offset = Node->getOperand(2);
02079   SDValue Flag = Node->getOperand(3);
02080   SDLoc DL(Node);
02081   MemSDNode *Mem = cast<MemSDNode>(Node);
02082 
02083   unsigned VecSize;
02084   switch (Node->getOpcode()) {
02085   default:
02086     return NULL;
02087   case NVPTXISD::LoadParam:
02088     VecSize = 1;
02089     break;
02090   case NVPTXISD::LoadParamV2:
02091     VecSize = 2;
02092     break;
02093   case NVPTXISD::LoadParamV4:
02094     VecSize = 4;
02095     break;
02096   }
02097 
02098   EVT EltVT = Node->getValueType(0);
02099   EVT MemVT = Mem->getMemoryVT();
02100 
02101   unsigned Opc = 0;
02102 
02103   switch (VecSize) {
02104   default:
02105     return NULL;
02106   case 1:
02107     switch (MemVT.getSimpleVT().SimpleTy) {
02108     default:
02109       return NULL;
02110     case MVT::i1:
02111       Opc = NVPTX::LoadParamMemI8;
02112       break;
02113     case MVT::i8:
02114       Opc = NVPTX::LoadParamMemI8;
02115       break;
02116     case MVT::i16:
02117       Opc = NVPTX::LoadParamMemI16;
02118       break;
02119     case MVT::i32:
02120       Opc = NVPTX::LoadParamMemI32;
02121       break;
02122     case MVT::i64:
02123       Opc = NVPTX::LoadParamMemI64;
02124       break;
02125     case MVT::f32:
02126       Opc = NVPTX::LoadParamMemF32;
02127       break;
02128     case MVT::f64:
02129       Opc = NVPTX::LoadParamMemF64;
02130       break;
02131     }
02132     break;
02133   case 2:
02134     switch (MemVT.getSimpleVT().SimpleTy) {
02135     default:
02136       return NULL;
02137     case MVT::i1:
02138       Opc = NVPTX::LoadParamMemV2I8;
02139       break;
02140     case MVT::i8:
02141       Opc = NVPTX::LoadParamMemV2I8;
02142       break;
02143     case MVT::i16:
02144       Opc = NVPTX::LoadParamMemV2I16;
02145       break;
02146     case MVT::i32:
02147       Opc = NVPTX::LoadParamMemV2I32;
02148       break;
02149     case MVT::i64:
02150       Opc = NVPTX::LoadParamMemV2I64;
02151       break;
02152     case MVT::f32:
02153       Opc = NVPTX::LoadParamMemV2F32;
02154       break;
02155     case MVT::f64:
02156       Opc = NVPTX::LoadParamMemV2F64;
02157       break;
02158     }
02159     break;
02160   case 4:
02161     switch (MemVT.getSimpleVT().SimpleTy) {
02162     default:
02163       return NULL;
02164     case MVT::i1:
02165       Opc = NVPTX::LoadParamMemV4I8;
02166       break;
02167     case MVT::i8:
02168       Opc = NVPTX::LoadParamMemV4I8;
02169       break;
02170     case MVT::i16:
02171       Opc = NVPTX::LoadParamMemV4I16;
02172       break;
02173     case MVT::i32:
02174       Opc = NVPTX::LoadParamMemV4I32;
02175       break;
02176     case MVT::f32:
02177       Opc = NVPTX::LoadParamMemV4F32;
02178       break;
02179     }
02180     break;
02181   }
02182 
02183   SDVTList VTs;
02184   if (VecSize == 1) {
02185     VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
02186   } else if (VecSize == 2) {
02187     VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
02188   } else {
02189     EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
02190     VTs = CurDAG->getVTList(EVTs);
02191   }
02192 
02193   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02194 
02195   SmallVector<SDValue, 2> Ops;
02196   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
02197   Ops.push_back(Chain);
02198   Ops.push_back(Flag);
02199 
02200   SDNode *Ret =
02201       CurDAG->getMachineNode(Opc, DL, VTs, Ops);
02202   return Ret;
02203 }
02204 
02205 SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
02206   SDLoc DL(N);
02207   SDValue Chain = N->getOperand(0);
02208   SDValue Offset = N->getOperand(1);
02209   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02210   MemSDNode *Mem = cast<MemSDNode>(N);
02211 
02212   // How many elements do we have?
02213   unsigned NumElts = 1;
02214   switch (N->getOpcode()) {
02215   default:
02216     return NULL;
02217   case NVPTXISD::StoreRetval:
02218     NumElts = 1;
02219     break;
02220   case NVPTXISD::StoreRetvalV2:
02221     NumElts = 2;
02222     break;
02223   case NVPTXISD::StoreRetvalV4:
02224     NumElts = 4;
02225     break;
02226   }
02227 
02228   // Build vector of operands
02229   SmallVector<SDValue, 6> Ops;
02230   for (unsigned i = 0; i < NumElts; ++i)
02231     Ops.push_back(N->getOperand(i + 2));
02232   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
02233   Ops.push_back(Chain);
02234 
02235   // Determine target opcode
02236   // If we have an i1, use an 8-bit store. The lowering code in
02237   // NVPTXISelLowering will have already emitted an upcast.
02238   unsigned Opcode = 0;
02239   switch (NumElts) {
02240   default:
02241     return NULL;
02242   case 1:
02243     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02244     default:
02245       return NULL;
02246     case MVT::i1:
02247       Opcode = NVPTX::StoreRetvalI8;
02248       break;
02249     case MVT::i8:
02250       Opcode = NVPTX::StoreRetvalI8;
02251       break;
02252     case MVT::i16:
02253       Opcode = NVPTX::StoreRetvalI16;
02254       break;
02255     case MVT::i32:
02256       Opcode = NVPTX::StoreRetvalI32;
02257       break;
02258     case MVT::i64:
02259       Opcode = NVPTX::StoreRetvalI64;
02260       break;
02261     case MVT::f32:
02262       Opcode = NVPTX::StoreRetvalF32;
02263       break;
02264     case MVT::f64:
02265       Opcode = NVPTX::StoreRetvalF64;
02266       break;
02267     }
02268     break;
02269   case 2:
02270     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02271     default:
02272       return NULL;
02273     case MVT::i1:
02274       Opcode = NVPTX::StoreRetvalV2I8;
02275       break;
02276     case MVT::i8:
02277       Opcode = NVPTX::StoreRetvalV2I8;
02278       break;
02279     case MVT::i16:
02280       Opcode = NVPTX::StoreRetvalV2I16;
02281       break;
02282     case MVT::i32:
02283       Opcode = NVPTX::StoreRetvalV2I32;
02284       break;
02285     case MVT::i64:
02286       Opcode = NVPTX::StoreRetvalV2I64;
02287       break;
02288     case MVT::f32:
02289       Opcode = NVPTX::StoreRetvalV2F32;
02290       break;
02291     case MVT::f64:
02292       Opcode = NVPTX::StoreRetvalV2F64;
02293       break;
02294     }
02295     break;
02296   case 4:
02297     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02298     default:
02299       return NULL;
02300     case MVT::i1:
02301       Opcode = NVPTX::StoreRetvalV4I8;
02302       break;
02303     case MVT::i8:
02304       Opcode = NVPTX::StoreRetvalV4I8;
02305       break;
02306     case MVT::i16:
02307       Opcode = NVPTX::StoreRetvalV4I16;
02308       break;
02309     case MVT::i32:
02310       Opcode = NVPTX::StoreRetvalV4I32;
02311       break;
02312     case MVT::f32:
02313       Opcode = NVPTX::StoreRetvalV4F32;
02314       break;
02315     }
02316     break;
02317   }
02318 
02319   SDNode *Ret =
02320       CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
02321   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02322   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02323   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
02324 
02325   return Ret;
02326 }
02327 
02328 SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
02329   SDLoc DL(N);
02330   SDValue Chain = N->getOperand(0);
02331   SDValue Param = N->getOperand(1);
02332   unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
02333   SDValue Offset = N->getOperand(2);
02334   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02335   MemSDNode *Mem = cast<MemSDNode>(N);
02336   SDValue Flag = N->getOperand(N->getNumOperands() - 1);
02337 
02338   // How many elements do we have?
02339   unsigned NumElts = 1;
02340   switch (N->getOpcode()) {
02341   default:
02342     return NULL;
02343   case NVPTXISD::StoreParamU32:
02344   case NVPTXISD::StoreParamS32:
02345   case NVPTXISD::StoreParam:
02346     NumElts = 1;
02347     break;
02348   case NVPTXISD::StoreParamV2:
02349     NumElts = 2;
02350     break;
02351   case NVPTXISD::StoreParamV4:
02352     NumElts = 4;
02353     break;
02354   }
02355 
02356   // Build vector of operands
02357   SmallVector<SDValue, 8> Ops;
02358   for (unsigned i = 0; i < NumElts; ++i)
02359     Ops.push_back(N->getOperand(i + 3));
02360   Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
02361   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
02362   Ops.push_back(Chain);
02363   Ops.push_back(Flag);
02364 
02365   // Determine target opcode
02366   // If we have an i1, use an 8-bit store. The lowering code in
02367   // NVPTXISelLowering will have already emitted an upcast.
02368   unsigned Opcode = 0;
02369   switch (N->getOpcode()) {
02370   default:
02371     switch (NumElts) {
02372     default:
02373       return NULL;
02374     case 1:
02375       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02376       default:
02377         return NULL;
02378       case MVT::i1:
02379         Opcode = NVPTX::StoreParamI8;
02380         break;
02381       case MVT::i8:
02382         Opcode = NVPTX::StoreParamI8;
02383         break;
02384       case MVT::i16:
02385         Opcode = NVPTX::StoreParamI16;
02386         break;
02387       case MVT::i32:
02388         Opcode = NVPTX::StoreParamI32;
02389         break;
02390       case MVT::i64:
02391         Opcode = NVPTX::StoreParamI64;
02392         break;
02393       case MVT::f32:
02394         Opcode = NVPTX::StoreParamF32;
02395         break;
02396       case MVT::f64:
02397         Opcode = NVPTX::StoreParamF64;
02398         break;
02399       }
02400       break;
02401     case 2:
02402       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02403       default:
02404         return NULL;
02405       case MVT::i1:
02406         Opcode = NVPTX::StoreParamV2I8;
02407         break;
02408       case MVT::i8:
02409         Opcode = NVPTX::StoreParamV2I8;
02410         break;
02411       case MVT::i16:
02412         Opcode = NVPTX::StoreParamV2I16;
02413         break;
02414       case MVT::i32:
02415         Opcode = NVPTX::StoreParamV2I32;
02416         break;
02417       case MVT::i64:
02418         Opcode = NVPTX::StoreParamV2I64;
02419         break;
02420       case MVT::f32:
02421         Opcode = NVPTX::StoreParamV2F32;
02422         break;
02423       case MVT::f64:
02424         Opcode = NVPTX::StoreParamV2F64;
02425         break;
02426       }
02427       break;
02428     case 4:
02429       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02430       default:
02431         return NULL;
02432       case MVT::i1:
02433         Opcode = NVPTX::StoreParamV4I8;
02434         break;
02435       case MVT::i8:
02436         Opcode = NVPTX::StoreParamV4I8;
02437         break;
02438       case MVT::i16:
02439         Opcode = NVPTX::StoreParamV4I16;
02440         break;
02441       case MVT::i32:
02442         Opcode = NVPTX::StoreParamV4I32;
02443         break;
02444       case MVT::f32:
02445         Opcode = NVPTX::StoreParamV4F32;
02446         break;
02447       }
02448       break;
02449     }
02450     break;
02451   // Special case: if we have a sign-extend/zero-extend node, insert the
02452   // conversion instruction first, and use that as the value operand to
02453   // the selected StoreParam node.
02454   case NVPTXISD::StoreParamU32: {
02455     Opcode = NVPTX::StoreParamI32;
02456     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
02457                                                 MVT::i32);
02458     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
02459                                          MVT::i32, Ops[0], CvtNone);
02460     Ops[0] = SDValue(Cvt, 0);
02461     break;
02462   }
02463   case NVPTXISD::StoreParamS32: {
02464     Opcode = NVPTX::StoreParamI32;
02465     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
02466                                                 MVT::i32);
02467     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
02468                                          MVT::i32, Ops[0], CvtNone);
02469     Ops[0] = SDValue(Cvt, 0);
02470     break;
02471   }
02472   }
02473 
02474   SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
02475   SDNode *Ret =
02476       CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
02477   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02478   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02479   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
02480 
02481   return Ret;
02482 }
02483 
02484 SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
02485   SDValue Chain = N->getOperand(0);
02486   SDValue TexRef = N->getOperand(1);
02487   SDValue SampRef = N->getOperand(2);
02488   SDNode *Ret = NULL;
02489   unsigned Opc = 0;
02490   SmallVector<SDValue, 8> Ops;
02491 
02492   switch (N->getOpcode()) {
02493   default: return NULL;
02494   case NVPTXISD::Tex1DFloatI32:
02495     Opc = NVPTX::TEX_1D_F32_I32;
02496     break;
02497   case NVPTXISD::Tex1DFloatFloat:
02498     Opc = NVPTX::TEX_1D_F32_F32;
02499     break;
02500   case NVPTXISD::Tex1DFloatFloatLevel:
02501     Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
02502     break;
02503   case NVPTXISD::Tex1DFloatFloatGrad:
02504     Opc = NVPTX::TEX_1D_F32_F32_GRAD;
02505     break;
02506   case NVPTXISD::Tex1DI32I32:
02507     Opc = NVPTX::TEX_1D_I32_I32;
02508     break;
02509   case NVPTXISD::Tex1DI32Float:
02510     Opc = NVPTX::TEX_1D_I32_F32;
02511     break;
02512   case NVPTXISD::Tex1DI32FloatLevel:
02513     Opc = NVPTX::TEX_1D_I32_F32_LEVEL;
02514     break;
02515   case NVPTXISD::Tex1DI32FloatGrad:
02516     Opc = NVPTX::TEX_1D_I32_F32_GRAD;
02517     break;
02518   case NVPTXISD::Tex1DArrayFloatI32:
02519     Opc = NVPTX::TEX_1D_ARRAY_F32_I32;
02520     break;
02521   case NVPTXISD::Tex1DArrayFloatFloat:
02522     Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
02523     break;
02524   case NVPTXISD::Tex1DArrayFloatFloatLevel:
02525     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
02526     break;
02527   case NVPTXISD::Tex1DArrayFloatFloatGrad:
02528     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
02529     break;
02530   case NVPTXISD::Tex1DArrayI32I32:
02531     Opc = NVPTX::TEX_1D_ARRAY_I32_I32;
02532     break;
02533   case NVPTXISD::Tex1DArrayI32Float:
02534     Opc = NVPTX::TEX_1D_ARRAY_I32_F32;
02535     break;
02536   case NVPTXISD::Tex1DArrayI32FloatLevel:
02537     Opc = NVPTX::TEX_1D_ARRAY_I32_F32_LEVEL;
02538     break;
02539   case NVPTXISD::Tex1DArrayI32FloatGrad:
02540     Opc = NVPTX::TEX_1D_ARRAY_I32_F32_GRAD;
02541     break;
02542   case NVPTXISD::Tex2DFloatI32:
02543     Opc = NVPTX::TEX_2D_F32_I32;
02544     break;
02545   case NVPTXISD::Tex2DFloatFloat:
02546     Opc = NVPTX::TEX_2D_F32_F32;
02547     break;
02548   case NVPTXISD::Tex2DFloatFloatLevel:
02549     Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
02550     break;
02551   case NVPTXISD::Tex2DFloatFloatGrad:
02552     Opc = NVPTX::TEX_2D_F32_F32_GRAD;
02553     break;
02554   case NVPTXISD::Tex2DI32I32:
02555     Opc = NVPTX::TEX_2D_I32_I32;
02556     break;
02557   case NVPTXISD::Tex2DI32Float:
02558     Opc = NVPTX::TEX_2D_I32_F32;
02559     break;
02560   case NVPTXISD::Tex2DI32FloatLevel:
02561     Opc = NVPTX::TEX_2D_I32_F32_LEVEL;
02562     break;
02563   case NVPTXISD::Tex2DI32FloatGrad:
02564     Opc = NVPTX::TEX_2D_I32_F32_GRAD;
02565     break;
02566   case NVPTXISD::Tex2DArrayFloatI32:
02567     Opc = NVPTX::TEX_2D_ARRAY_F32_I32;
02568     break;
02569   case NVPTXISD::Tex2DArrayFloatFloat:
02570     Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
02571     break;
02572   case NVPTXISD::Tex2DArrayFloatFloatLevel:
02573     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
02574     break;
02575   case NVPTXISD::Tex2DArrayFloatFloatGrad:
02576     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
02577     break;
02578   case NVPTXISD::Tex2DArrayI32I32:
02579     Opc = NVPTX::TEX_2D_ARRAY_I32_I32;
02580     break;
02581   case NVPTXISD::Tex2DArrayI32Float:
02582     Opc = NVPTX::TEX_2D_ARRAY_I32_F32;
02583     break;
02584   case NVPTXISD::Tex2DArrayI32FloatLevel:
02585     Opc = NVPTX::TEX_2D_ARRAY_I32_F32_LEVEL;
02586     break;
02587   case NVPTXISD::Tex2DArrayI32FloatGrad:
02588     Opc = NVPTX::TEX_2D_ARRAY_I32_F32_GRAD;
02589     break;
02590   case NVPTXISD::Tex3DFloatI32:
02591     Opc = NVPTX::TEX_3D_F32_I32;
02592     break;
02593   case NVPTXISD::Tex3DFloatFloat:
02594     Opc = NVPTX::TEX_3D_F32_F32;
02595     break;
02596   case NVPTXISD::Tex3DFloatFloatLevel:
02597     Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
02598     break;
02599   case NVPTXISD::Tex3DFloatFloatGrad:
02600     Opc = NVPTX::TEX_3D_F32_F32_GRAD;
02601     break;
02602   case NVPTXISD::Tex3DI32I32:
02603     Opc = NVPTX::TEX_3D_I32_I32;
02604     break;
02605   case NVPTXISD::Tex3DI32Float:
02606     Opc = NVPTX::TEX_3D_I32_F32;
02607     break;
02608   case NVPTXISD::Tex3DI32FloatLevel:
02609     Opc = NVPTX::TEX_3D_I32_F32_LEVEL;
02610     break;
02611   case NVPTXISD::Tex3DI32FloatGrad:
02612     Opc = NVPTX::TEX_3D_I32_F32_GRAD;
02613     break;
02614   }
02615 
02616   Ops.push_back(TexRef);
02617   Ops.push_back(SampRef);
02618 
02619   // Copy over indices
02620   for (unsigned i = 3; i < N->getNumOperands(); ++i) {
02621     Ops.push_back(N->getOperand(i));
02622   }
02623 
02624   Ops.push_back(Chain);
02625   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
02626   return Ret;
02627 }
02628 
02629 SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
02630   SDValue Chain = N->getOperand(0);
02631   SDValue TexHandle = N->getOperand(1);
02632   SDNode *Ret = NULL;
02633   unsigned Opc = 0;
02634   SmallVector<SDValue, 8> Ops;
02635   switch (N->getOpcode()) {
02636   default: return NULL;
02637   case NVPTXISD::Suld1DI8Trap:
02638     Opc = NVPTX::SULD_1D_I8_TRAP;
02639     Ops.push_back(TexHandle);
02640     Ops.push_back(N->getOperand(2));
02641     Ops.push_back(Chain);
02642     break;
02643   case NVPTXISD::Suld1DI16Trap:
02644     Opc = NVPTX::SULD_1D_I16_TRAP;
02645     Ops.push_back(TexHandle);
02646     Ops.push_back(N->getOperand(2));
02647     Ops.push_back(Chain);
02648     break;
02649   case NVPTXISD::Suld1DI32Trap:
02650     Opc = NVPTX::SULD_1D_I32_TRAP;
02651     Ops.push_back(TexHandle);
02652     Ops.push_back(N->getOperand(2));
02653     Ops.push_back(Chain);
02654     break;
02655   case NVPTXISD::Suld1DV2I8Trap:
02656     Opc = NVPTX::SULD_1D_V2I8_TRAP;
02657     Ops.push_back(TexHandle);
02658     Ops.push_back(N->getOperand(2));
02659     Ops.push_back(Chain);
02660     break;
02661   case NVPTXISD::Suld1DV2I16Trap:
02662     Opc = NVPTX::SULD_1D_V2I16_TRAP;
02663     Ops.push_back(TexHandle);
02664     Ops.push_back(N->getOperand(2));
02665     Ops.push_back(Chain);
02666     break;
02667   case NVPTXISD::Suld1DV2I32Trap:
02668     Opc = NVPTX::SULD_1D_V2I32_TRAP;
02669     Ops.push_back(TexHandle);
02670     Ops.push_back(N->getOperand(2));
02671     Ops.push_back(Chain);
02672     break;
02673   case NVPTXISD::Suld1DV4I8Trap:
02674     Opc = NVPTX::SULD_1D_V4I8_TRAP;
02675     Ops.push_back(TexHandle);
02676     Ops.push_back(N->getOperand(2));
02677     Ops.push_back(Chain);
02678     break;
02679   case NVPTXISD::Suld1DV4I16Trap:
02680     Opc = NVPTX::SULD_1D_V4I16_TRAP;
02681     Ops.push_back(TexHandle);
02682     Ops.push_back(N->getOperand(2));
02683     Ops.push_back(Chain);
02684     break;
02685   case NVPTXISD::Suld1DV4I32Trap:
02686     Opc = NVPTX::SULD_1D_V4I32_TRAP;
02687     Ops.push_back(TexHandle);
02688     Ops.push_back(N->getOperand(2));
02689     Ops.push_back(Chain);
02690     break;
02691   case NVPTXISD::Suld1DArrayI8Trap:
02692     Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
02693     Ops.push_back(TexHandle);
02694     Ops.push_back(N->getOperand(2));
02695     Ops.push_back(N->getOperand(3));
02696     Ops.push_back(Chain);
02697     break;
02698   case NVPTXISD::Suld1DArrayI16Trap:
02699     Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
02700     Ops.push_back(TexHandle);
02701     Ops.push_back(N->getOperand(2));
02702     Ops.push_back(N->getOperand(3));
02703     Ops.push_back(Chain);
02704     break;
02705   case NVPTXISD::Suld1DArrayI32Trap:
02706     Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
02707     Ops.push_back(TexHandle);
02708     Ops.push_back(N->getOperand(2));
02709     Ops.push_back(N->getOperand(3));
02710     Ops.push_back(Chain);
02711     break;
02712   case NVPTXISD::Suld1DArrayV2I8Trap:
02713     Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
02714     Ops.push_back(TexHandle);
02715     Ops.push_back(N->getOperand(2));
02716     Ops.push_back(N->getOperand(3));
02717     Ops.push_back(Chain);
02718     break;
02719   case NVPTXISD::Suld1DArrayV2I16Trap:
02720     Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
02721     Ops.push_back(TexHandle);
02722     Ops.push_back(N->getOperand(2));
02723     Ops.push_back(N->getOperand(3));
02724     Ops.push_back(Chain);
02725     break;
02726   case NVPTXISD::Suld1DArrayV2I32Trap:
02727     Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
02728     Ops.push_back(TexHandle);
02729     Ops.push_back(N->getOperand(2));
02730     Ops.push_back(N->getOperand(3));
02731     Ops.push_back(Chain);
02732     break;
02733   case NVPTXISD::Suld1DArrayV4I8Trap:
02734     Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
02735     Ops.push_back(TexHandle);
02736     Ops.push_back(N->getOperand(2));
02737     Ops.push_back(N->getOperand(3));
02738     Ops.push_back(Chain);
02739     break;
02740   case NVPTXISD::Suld1DArrayV4I16Trap:
02741     Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
02742     Ops.push_back(TexHandle);
02743     Ops.push_back(N->getOperand(2));
02744     Ops.push_back(N->getOperand(3));
02745     Ops.push_back(Chain);
02746     break;
02747   case NVPTXISD::Suld1DArrayV4I32Trap:
02748     Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
02749     Ops.push_back(TexHandle);
02750     Ops.push_back(N->getOperand(2));
02751     Ops.push_back(N->getOperand(3));
02752     Ops.push_back(Chain);
02753     break;
02754   case NVPTXISD::Suld2DI8Trap:
02755     Opc = NVPTX::SULD_2D_I8_TRAP;
02756     Ops.push_back(TexHandle);
02757     Ops.push_back(N->getOperand(2));
02758     Ops.push_back(N->getOperand(3));
02759     Ops.push_back(Chain);
02760     break;
02761   case NVPTXISD::Suld2DI16Trap:
02762     Opc = NVPTX::SULD_2D_I16_TRAP;
02763     Ops.push_back(TexHandle);
02764     Ops.push_back(N->getOperand(2));
02765     Ops.push_back(N->getOperand(3));
02766     Ops.push_back(Chain);
02767     break;
02768   case NVPTXISD::Suld2DI32Trap:
02769     Opc = NVPTX::SULD_2D_I32_TRAP;
02770     Ops.push_back(TexHandle);
02771     Ops.push_back(N->getOperand(2));
02772     Ops.push_back(N->getOperand(3));
02773     Ops.push_back(Chain);
02774     break;
02775   case NVPTXISD::Suld2DV2I8Trap:
02776     Opc = NVPTX::SULD_2D_V2I8_TRAP;
02777     Ops.push_back(TexHandle);
02778     Ops.push_back(N->getOperand(2));
02779     Ops.push_back(N->getOperand(3));
02780     Ops.push_back(Chain);
02781     break;
02782   case NVPTXISD::Suld2DV2I16Trap:
02783     Opc = NVPTX::SULD_2D_V2I16_TRAP;
02784     Ops.push_back(TexHandle);
02785     Ops.push_back(N->getOperand(2));
02786     Ops.push_back(N->getOperand(3));
02787     Ops.push_back(Chain);
02788     break;
02789   case NVPTXISD::Suld2DV2I32Trap:
02790     Opc = NVPTX::SULD_2D_V2I32_TRAP;
02791     Ops.push_back(TexHandle);
02792     Ops.push_back(N->getOperand(2));
02793     Ops.push_back(N->getOperand(3));
02794     Ops.push_back(Chain);
02795     break;
02796   case NVPTXISD::Suld2DV4I8Trap:
02797     Opc = NVPTX::SULD_2D_V4I8_TRAP;
02798     Ops.push_back(TexHandle);
02799     Ops.push_back(N->getOperand(2));
02800     Ops.push_back(N->getOperand(3));
02801     Ops.push_back(Chain);
02802     break;
02803   case NVPTXISD::Suld2DV4I16Trap:
02804     Opc = NVPTX::SULD_2D_V4I16_TRAP;
02805     Ops.push_back(TexHandle);
02806     Ops.push_back(N->getOperand(2));
02807     Ops.push_back(N->getOperand(3));
02808     Ops.push_back(Chain);
02809     break;
02810   case NVPTXISD::Suld2DV4I32Trap:
02811     Opc = NVPTX::SULD_2D_V4I32_TRAP;
02812     Ops.push_back(TexHandle);
02813     Ops.push_back(N->getOperand(2));
02814     Ops.push_back(N->getOperand(3));
02815     Ops.push_back(Chain);
02816     break;
02817   case NVPTXISD::Suld2DArrayI8Trap:
02818     Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
02819     Ops.push_back(TexHandle);
02820     Ops.push_back(N->getOperand(2));
02821     Ops.push_back(N->getOperand(3));
02822     Ops.push_back(N->getOperand(4));
02823     Ops.push_back(Chain);
02824     break;
02825   case NVPTXISD::Suld2DArrayI16Trap:
02826     Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
02827     Ops.push_back(TexHandle);
02828     Ops.push_back(N->getOperand(2));
02829     Ops.push_back(N->getOperand(3));
02830     Ops.push_back(N->getOperand(4));
02831     Ops.push_back(Chain);
02832     break;
02833   case NVPTXISD::Suld2DArrayI32Trap:
02834     Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
02835     Ops.push_back(TexHandle);
02836     Ops.push_back(N->getOperand(2));
02837     Ops.push_back(N->getOperand(3));
02838     Ops.push_back(N->getOperand(4));
02839     Ops.push_back(Chain);
02840     break;
02841   case NVPTXISD::Suld2DArrayV2I8Trap:
02842     Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
02843     Ops.push_back(TexHandle);
02844     Ops.push_back(N->getOperand(2));
02845     Ops.push_back(N->getOperand(3));
02846     Ops.push_back(N->getOperand(4));
02847     Ops.push_back(Chain);
02848     break;
02849   case NVPTXISD::Suld2DArrayV2I16Trap:
02850     Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
02851     Ops.push_back(TexHandle);
02852     Ops.push_back(N->getOperand(2));
02853     Ops.push_back(N->getOperand(3));
02854     Ops.push_back(N->getOperand(4));
02855     Ops.push_back(Chain);
02856     break;
02857   case NVPTXISD::Suld2DArrayV2I32Trap:
02858     Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
02859     Ops.push_back(TexHandle);
02860     Ops.push_back(N->getOperand(2));
02861     Ops.push_back(N->getOperand(3));
02862     Ops.push_back(N->getOperand(4));
02863     Ops.push_back(Chain);
02864     break;
02865   case NVPTXISD::Suld2DArrayV4I8Trap:
02866     Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
02867     Ops.push_back(TexHandle);
02868     Ops.push_back(N->getOperand(2));
02869     Ops.push_back(N->getOperand(3));
02870     Ops.push_back(N->getOperand(4));
02871     Ops.push_back(Chain);
02872     break;
02873   case NVPTXISD::Suld2DArrayV4I16Trap:
02874     Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
02875     Ops.push_back(TexHandle);
02876     Ops.push_back(N->getOperand(2));
02877     Ops.push_back(N->getOperand(3));
02878     Ops.push_back(N->getOperand(4));
02879     Ops.push_back(Chain);
02880     break;
02881   case NVPTXISD::Suld2DArrayV4I32Trap:
02882     Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
02883     Ops.push_back(TexHandle);
02884     Ops.push_back(N->getOperand(2));
02885     Ops.push_back(N->getOperand(3));
02886     Ops.push_back(N->getOperand(4));
02887     Ops.push_back(Chain);
02888     break;
02889   case NVPTXISD::Suld3DI8Trap:
02890     Opc = NVPTX::SULD_3D_I8_TRAP;
02891     Ops.push_back(TexHandle);
02892     Ops.push_back(N->getOperand(2));
02893     Ops.push_back(N->getOperand(3));
02894     Ops.push_back(N->getOperand(4));
02895     Ops.push_back(Chain);
02896     break;
02897   case NVPTXISD::Suld3DI16Trap:
02898     Opc = NVPTX::SULD_3D_I16_TRAP;
02899     Ops.push_back(TexHandle);
02900     Ops.push_back(N->getOperand(2));
02901     Ops.push_back(N->getOperand(3));
02902     Ops.push_back(N->getOperand(4));
02903     Ops.push_back(Chain);
02904     break;
02905   case NVPTXISD::Suld3DI32Trap:
02906     Opc = NVPTX::SULD_3D_I32_TRAP;
02907     Ops.push_back(TexHandle);
02908     Ops.push_back(N->getOperand(2));
02909     Ops.push_back(N->getOperand(3));
02910     Ops.push_back(N->getOperand(4));
02911     Ops.push_back(Chain);
02912     break;
02913   case NVPTXISD::Suld3DV2I8Trap:
02914     Opc = NVPTX::SULD_3D_V2I8_TRAP;
02915     Ops.push_back(TexHandle);
02916     Ops.push_back(N->getOperand(2));
02917     Ops.push_back(N->getOperand(3));
02918     Ops.push_back(N->getOperand(4));
02919     Ops.push_back(Chain);
02920     break;
02921   case NVPTXISD::Suld3DV2I16Trap:
02922     Opc = NVPTX::SULD_3D_V2I16_TRAP;
02923     Ops.push_back(TexHandle);
02924     Ops.push_back(N->getOperand(2));
02925     Ops.push_back(N->getOperand(3));
02926     Ops.push_back(N->getOperand(4));
02927     Ops.push_back(Chain);
02928     break;
02929   case NVPTXISD::Suld3DV2I32Trap:
02930     Opc = NVPTX::SULD_3D_V2I32_TRAP;
02931     Ops.push_back(TexHandle);
02932     Ops.push_back(N->getOperand(2));
02933     Ops.push_back(N->getOperand(3));
02934     Ops.push_back(N->getOperand(4));
02935     Ops.push_back(Chain);
02936     break;
02937   case NVPTXISD::Suld3DV4I8Trap:
02938     Opc = NVPTX::SULD_3D_V4I8_TRAP;
02939     Ops.push_back(TexHandle);
02940     Ops.push_back(N->getOperand(2));
02941     Ops.push_back(N->getOperand(3));
02942     Ops.push_back(N->getOperand(4));
02943     Ops.push_back(Chain);
02944     break;
02945   case NVPTXISD::Suld3DV4I16Trap:
02946     Opc = NVPTX::SULD_3D_V4I16_TRAP;
02947     Ops.push_back(TexHandle);
02948     Ops.push_back(N->getOperand(2));
02949     Ops.push_back(N->getOperand(3));
02950     Ops.push_back(N->getOperand(4));
02951     Ops.push_back(Chain);
02952     break;
02953   case NVPTXISD::Suld3DV4I32Trap:
02954     Opc = NVPTX::SULD_3D_V4I32_TRAP;
02955     Ops.push_back(TexHandle);
02956     Ops.push_back(N->getOperand(2));
02957     Ops.push_back(N->getOperand(3));
02958     Ops.push_back(N->getOperand(4));
02959     Ops.push_back(Chain);
02960     break;
02961   }
02962   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
02963   return Ret;
02964 }
02965 
02966 // SelectDirectAddr - Match a direct address for DAG.
02967 // A direct address could be a globaladdress or externalsymbol.
02968 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
02969   // Return true if TGA or ES.
02970   if (N.getOpcode() == ISD::TargetGlobalAddress ||
02971       N.getOpcode() == ISD::TargetExternalSymbol) {
02972     Address = N;
02973     return true;
02974   }
02975   if (N.getOpcode() == NVPTXISD::Wrapper) {
02976     Address = N.getOperand(0);
02977     return true;
02978   }
02979   if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
02980     unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
02981     if (IID == Intrinsic::nvvm_ptr_gen_to_param)
02982       if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
02983         return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
02984   }
02985   return false;
02986 }
02987 
02988 // symbol+offset
02989 bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
02990     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
02991   if (Addr.getOpcode() == ISD::ADD) {
02992     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
02993       SDValue base = Addr.getOperand(0);
02994       if (SelectDirectAddr(base, Base)) {
02995         Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
02996         return true;
02997       }
02998     }
02999   }
03000   return false;
03001 }
03002 
03003 // symbol+offset
03004 bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
03005                                      SDValue &Base, SDValue &Offset) {
03006   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
03007 }
03008 
03009 // symbol+offset
03010 bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
03011                                        SDValue &Base, SDValue &Offset) {
03012   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
03013 }
03014 
03015 // register+offset
03016 bool NVPTXDAGToDAGISel::SelectADDRri_imp(
03017     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
03018   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
03019     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
03020     Offset = CurDAG->getTargetConstant(0, mvt);
03021     return true;
03022   }
03023   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
03024       Addr.getOpcode() == ISD::TargetGlobalAddress)
03025     return false; // direct calls.
03026 
03027   if (Addr.getOpcode() == ISD::ADD) {
03028     if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
03029       return false;
03030     }
03031     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
03032       if (FrameIndexSDNode *FIN =
03033               dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
03034         // Constant offset from frame ref.
03035         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
03036       else
03037         Base = Addr.getOperand(0);
03038       Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
03039       return true;
03040     }
03041   }
03042   return false;
03043 }
03044 
03045 // register+offset
03046 bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
03047                                      SDValue &Base, SDValue &Offset) {
03048   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
03049 }
03050 
03051 // register+offset
03052 bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
03053                                        SDValue &Base, SDValue &Offset) {
03054   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
03055 }
03056 
03057 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
03058                                                  unsigned int spN) const {
03059   const Value *Src = NULL;
03060   // Even though MemIntrinsicSDNode is a subclas of MemSDNode,
03061   // the classof() for MemSDNode does not include MemIntrinsicSDNode
03062   // (See SelectionDAGNodes.h). So we need to check for both.
03063   if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
03064     if (spN == 0 && mN->getMemOperand()->getPseudoValue())
03065       return true;
03066     Src = mN->getMemOperand()->getValue();
03067   } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) {
03068     if (spN == 0 && mN->getMemOperand()->getPseudoValue())
03069       return true;
03070     Src = mN->getMemOperand()->getValue();
03071   }
03072   if (!Src)
03073     return false;
03074   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
03075     return (PT->getAddressSpace() == spN);
03076   return false;
03077 }
03078 
03079 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
03080 /// inline asm expressions.
03081 bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
03082     const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) {
03083   SDValue Op0, Op1;
03084   switch (ConstraintCode) {
03085   default:
03086     return true;
03087   case 'm': // memory
03088     if (SelectDirectAddr(Op, Op0)) {
03089       OutOps.push_back(Op0);
03090       OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
03091       return false;
03092     }
03093     if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
03094       OutOps.push_back(Op0);
03095       OutOps.push_back(Op1);
03096       return false;
03097     }
03098     break;
03099   }
03100   return true;
03101 }