LLVM  mainline
NVPTXISelDAGToDAG.cpp
Go to the documentation of this file.
00001 //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file defines an instruction selector for the NVPTX target.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "NVPTXISelDAGToDAG.h"
00015 #include "llvm/IR/GlobalValue.h"
00016 #include "llvm/IR/Instructions.h"
00017 #include "llvm/Support/CommandLine.h"
00018 #include "llvm/Support/Debug.h"
00019 #include "llvm/Support/ErrorHandling.h"
00020 #include "llvm/Support/raw_ostream.h"
00021 #include "llvm/Target/TargetIntrinsicInfo.h"
00022 
00023 using namespace llvm;
00024 
00025 #define DEBUG_TYPE "nvptx-isel"
00026 
00027 static cl::opt<int> UsePrecDivF32(
00028     "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
00029     cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
00030              " IEEE Compliant F32 div.rnd if available."),
00031     cl::init(2));
00032 
00033 static cl::opt<bool>
00034 UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
00035           cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
00036           cl::init(true));
00037 
00038 static cl::opt<bool>
00039 FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
00040            cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
00041            cl::init(false));
00042 
00043 
00044 /// createNVPTXISelDag - This pass converts a legalized DAG into a
00045 /// NVPTX-specific DAG, ready for instruction scheduling.
00046 FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
00047                                        llvm::CodeGenOpt::Level OptLevel) {
00048   return new NVPTXDAGToDAGISel(TM, OptLevel);
00049 }
00050 
00051 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
00052                                      CodeGenOpt::Level OptLevel)
00053     : SelectionDAGISel(tm, OptLevel), TM(tm) {
00054   doMulWide = (OptLevel > 0);
00055 }
00056 
00057 bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
00058     Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget());
00059     return SelectionDAGISel::runOnMachineFunction(MF);
00060 }
00061 
00062 int NVPTXDAGToDAGISel::getDivF32Level() const {
00063   if (UsePrecDivF32.getNumOccurrences() > 0) {
00064     // If nvptx-prec-div32=N is used on the command-line, always honor it
00065     return UsePrecDivF32;
00066   } else {
00067     // Otherwise, use div.approx if fast math is enabled
00068     if (TM.Options.UnsafeFPMath)
00069       return 0;
00070     else
00071       return 2;
00072   }
00073 }
00074 
00075 bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
00076   if (UsePrecSqrtF32.getNumOccurrences() > 0) {
00077     // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
00078     return UsePrecSqrtF32;
00079   } else {
00080     // Otherwise, use sqrt.approx if fast math is enabled
00081     return !TM.Options.UnsafeFPMath;
00082   }
00083 }
00084 
00085 bool NVPTXDAGToDAGISel::useF32FTZ() const {
00086   if (FtzEnabled.getNumOccurrences() > 0) {
00087     // If nvptx-f32ftz is used on the command-line, always honor it
00088     return FtzEnabled;
00089   } else {
00090     const Function *F = MF->getFunction();
00091     // Otherwise, check for an nvptx-f32ftz attribute on the function
00092     if (F->hasFnAttribute("nvptx-f32ftz"))
00093       return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true";
00094     else
00095       return false;
00096   }
00097 }
00098 
00099 bool NVPTXDAGToDAGISel::allowFMA() const {
00100   const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
00101   return TL->allowFMA(*MF, OptLevel);
00102 }
00103 
00104 /// Select - Select instructions not customized! Used for
00105 /// expanded, promoted and normal instructions.
00106 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
00107 
00108   if (N->isMachineOpcode()) {
00109     N->setNodeId(-1);
00110     return nullptr; // Already selected.
00111   }
00112 
00113   SDNode *ResNode = nullptr;
00114   switch (N->getOpcode()) {
00115   case ISD::LOAD:
00116     ResNode = SelectLoad(N);
00117     break;
00118   case ISD::STORE:
00119     ResNode = SelectStore(N);
00120     break;
00121   case NVPTXISD::LoadV2:
00122   case NVPTXISD::LoadV4:
00123     ResNode = SelectLoadVector(N);
00124     break;
00125   case NVPTXISD::LDGV2:
00126   case NVPTXISD::LDGV4:
00127   case NVPTXISD::LDUV2:
00128   case NVPTXISD::LDUV4:
00129     ResNode = SelectLDGLDU(N);
00130     break;
00131   case NVPTXISD::StoreV2:
00132   case NVPTXISD::StoreV4:
00133     ResNode = SelectStoreVector(N);
00134     break;
00135   case NVPTXISD::LoadParam:
00136   case NVPTXISD::LoadParamV2:
00137   case NVPTXISD::LoadParamV4:
00138     ResNode = SelectLoadParam(N);
00139     break;
00140   case NVPTXISD::StoreRetval:
00141   case NVPTXISD::StoreRetvalV2:
00142   case NVPTXISD::StoreRetvalV4:
00143     ResNode = SelectStoreRetval(N);
00144     break;
00145   case NVPTXISD::StoreParam:
00146   case NVPTXISD::StoreParamV2:
00147   case NVPTXISD::StoreParamV4:
00148   case NVPTXISD::StoreParamS32:
00149   case NVPTXISD::StoreParamU32:
00150     ResNode = SelectStoreParam(N);
00151     break;
00152   case ISD::INTRINSIC_WO_CHAIN:
00153     ResNode = SelectIntrinsicNoChain(N);
00154     break;
00155   case ISD::INTRINSIC_W_CHAIN:
00156     ResNode = SelectIntrinsicChain(N);
00157     break;
00158   case NVPTXISD::Tex1DFloatS32:
00159   case NVPTXISD::Tex1DFloatFloat:
00160   case NVPTXISD::Tex1DFloatFloatLevel:
00161   case NVPTXISD::Tex1DFloatFloatGrad:
00162   case NVPTXISD::Tex1DS32S32:
00163   case NVPTXISD::Tex1DS32Float:
00164   case NVPTXISD::Tex1DS32FloatLevel:
00165   case NVPTXISD::Tex1DS32FloatGrad:
00166   case NVPTXISD::Tex1DU32S32:
00167   case NVPTXISD::Tex1DU32Float:
00168   case NVPTXISD::Tex1DU32FloatLevel:
00169   case NVPTXISD::Tex1DU32FloatGrad:
00170   case NVPTXISD::Tex1DArrayFloatS32:
00171   case NVPTXISD::Tex1DArrayFloatFloat:
00172   case NVPTXISD::Tex1DArrayFloatFloatLevel:
00173   case NVPTXISD::Tex1DArrayFloatFloatGrad:
00174   case NVPTXISD::Tex1DArrayS32S32:
00175   case NVPTXISD::Tex1DArrayS32Float:
00176   case NVPTXISD::Tex1DArrayS32FloatLevel:
00177   case NVPTXISD::Tex1DArrayS32FloatGrad:
00178   case NVPTXISD::Tex1DArrayU32S32:
00179   case NVPTXISD::Tex1DArrayU32Float:
00180   case NVPTXISD::Tex1DArrayU32FloatLevel:
00181   case NVPTXISD::Tex1DArrayU32FloatGrad:
00182   case NVPTXISD::Tex2DFloatS32:
00183   case NVPTXISD::Tex2DFloatFloat:
00184   case NVPTXISD::Tex2DFloatFloatLevel:
00185   case NVPTXISD::Tex2DFloatFloatGrad:
00186   case NVPTXISD::Tex2DS32S32:
00187   case NVPTXISD::Tex2DS32Float:
00188   case NVPTXISD::Tex2DS32FloatLevel:
00189   case NVPTXISD::Tex2DS32FloatGrad:
00190   case NVPTXISD::Tex2DU32S32:
00191   case NVPTXISD::Tex2DU32Float:
00192   case NVPTXISD::Tex2DU32FloatLevel:
00193   case NVPTXISD::Tex2DU32FloatGrad:
00194   case NVPTXISD::Tex2DArrayFloatS32:
00195   case NVPTXISD::Tex2DArrayFloatFloat:
00196   case NVPTXISD::Tex2DArrayFloatFloatLevel:
00197   case NVPTXISD::Tex2DArrayFloatFloatGrad:
00198   case NVPTXISD::Tex2DArrayS32S32:
00199   case NVPTXISD::Tex2DArrayS32Float:
00200   case NVPTXISD::Tex2DArrayS32FloatLevel:
00201   case NVPTXISD::Tex2DArrayS32FloatGrad:
00202   case NVPTXISD::Tex2DArrayU32S32:
00203   case NVPTXISD::Tex2DArrayU32Float:
00204   case NVPTXISD::Tex2DArrayU32FloatLevel:
00205   case NVPTXISD::Tex2DArrayU32FloatGrad:
00206   case NVPTXISD::Tex3DFloatS32:
00207   case NVPTXISD::Tex3DFloatFloat:
00208   case NVPTXISD::Tex3DFloatFloatLevel:
00209   case NVPTXISD::Tex3DFloatFloatGrad:
00210   case NVPTXISD::Tex3DS32S32:
00211   case NVPTXISD::Tex3DS32Float:
00212   case NVPTXISD::Tex3DS32FloatLevel:
00213   case NVPTXISD::Tex3DS32FloatGrad:
00214   case NVPTXISD::Tex3DU32S32:
00215   case NVPTXISD::Tex3DU32Float:
00216   case NVPTXISD::Tex3DU32FloatLevel:
00217   case NVPTXISD::Tex3DU32FloatGrad:
00218   case NVPTXISD::TexCubeFloatFloat:
00219   case NVPTXISD::TexCubeFloatFloatLevel:
00220   case NVPTXISD::TexCubeS32Float:
00221   case NVPTXISD::TexCubeS32FloatLevel:
00222   case NVPTXISD::TexCubeU32Float:
00223   case NVPTXISD::TexCubeU32FloatLevel:
00224   case NVPTXISD::TexCubeArrayFloatFloat:
00225   case NVPTXISD::TexCubeArrayFloatFloatLevel:
00226   case NVPTXISD::TexCubeArrayS32Float:
00227   case NVPTXISD::TexCubeArrayS32FloatLevel:
00228   case NVPTXISD::TexCubeArrayU32Float:
00229   case NVPTXISD::TexCubeArrayU32FloatLevel:
00230   case NVPTXISD::Tld4R2DFloatFloat:
00231   case NVPTXISD::Tld4G2DFloatFloat:
00232   case NVPTXISD::Tld4B2DFloatFloat:
00233   case NVPTXISD::Tld4A2DFloatFloat:
00234   case NVPTXISD::Tld4R2DS64Float:
00235   case NVPTXISD::Tld4G2DS64Float:
00236   case NVPTXISD::Tld4B2DS64Float:
00237   case NVPTXISD::Tld4A2DS64Float:
00238   case NVPTXISD::Tld4R2DU64Float:
00239   case NVPTXISD::Tld4G2DU64Float:
00240   case NVPTXISD::Tld4B2DU64Float:
00241   case NVPTXISD::Tld4A2DU64Float:
00242   case NVPTXISD::TexUnified1DFloatS32:
00243   case NVPTXISD::TexUnified1DFloatFloat:
00244   case NVPTXISD::TexUnified1DFloatFloatLevel:
00245   case NVPTXISD::TexUnified1DFloatFloatGrad:
00246   case NVPTXISD::TexUnified1DS32S32:
00247   case NVPTXISD::TexUnified1DS32Float:
00248   case NVPTXISD::TexUnified1DS32FloatLevel:
00249   case NVPTXISD::TexUnified1DS32FloatGrad:
00250   case NVPTXISD::TexUnified1DU32S32:
00251   case NVPTXISD::TexUnified1DU32Float:
00252   case NVPTXISD::TexUnified1DU32FloatLevel:
00253   case NVPTXISD::TexUnified1DU32FloatGrad:
00254   case NVPTXISD::TexUnified1DArrayFloatS32:
00255   case NVPTXISD::TexUnified1DArrayFloatFloat:
00256   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
00257   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
00258   case NVPTXISD::TexUnified1DArrayS32S32:
00259   case NVPTXISD::TexUnified1DArrayS32Float:
00260   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
00261   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
00262   case NVPTXISD::TexUnified1DArrayU32S32:
00263   case NVPTXISD::TexUnified1DArrayU32Float:
00264   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
00265   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
00266   case NVPTXISD::TexUnified2DFloatS32:
00267   case NVPTXISD::TexUnified2DFloatFloat:
00268   case NVPTXISD::TexUnified2DFloatFloatLevel:
00269   case NVPTXISD::TexUnified2DFloatFloatGrad:
00270   case NVPTXISD::TexUnified2DS32S32:
00271   case NVPTXISD::TexUnified2DS32Float:
00272   case NVPTXISD::TexUnified2DS32FloatLevel:
00273   case NVPTXISD::TexUnified2DS32FloatGrad:
00274   case NVPTXISD::TexUnified2DU32S32:
00275   case NVPTXISD::TexUnified2DU32Float:
00276   case NVPTXISD::TexUnified2DU32FloatLevel:
00277   case NVPTXISD::TexUnified2DU32FloatGrad:
00278   case NVPTXISD::TexUnified2DArrayFloatS32:
00279   case NVPTXISD::TexUnified2DArrayFloatFloat:
00280   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
00281   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
00282   case NVPTXISD::TexUnified2DArrayS32S32:
00283   case NVPTXISD::TexUnified2DArrayS32Float:
00284   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
00285   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
00286   case NVPTXISD::TexUnified2DArrayU32S32:
00287   case NVPTXISD::TexUnified2DArrayU32Float:
00288   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
00289   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
00290   case NVPTXISD::TexUnified3DFloatS32:
00291   case NVPTXISD::TexUnified3DFloatFloat:
00292   case NVPTXISD::TexUnified3DFloatFloatLevel:
00293   case NVPTXISD::TexUnified3DFloatFloatGrad:
00294   case NVPTXISD::TexUnified3DS32S32:
00295   case NVPTXISD::TexUnified3DS32Float:
00296   case NVPTXISD::TexUnified3DS32FloatLevel:
00297   case NVPTXISD::TexUnified3DS32FloatGrad:
00298   case NVPTXISD::TexUnified3DU32S32:
00299   case NVPTXISD::TexUnified3DU32Float:
00300   case NVPTXISD::TexUnified3DU32FloatLevel:
00301   case NVPTXISD::TexUnified3DU32FloatGrad:
00302   case NVPTXISD::TexUnifiedCubeFloatFloat:
00303   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
00304   case NVPTXISD::TexUnifiedCubeS32Float:
00305   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
00306   case NVPTXISD::TexUnifiedCubeU32Float:
00307   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
00308   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
00309   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
00310   case NVPTXISD::TexUnifiedCubeArrayS32Float:
00311   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
00312   case NVPTXISD::TexUnifiedCubeArrayU32Float:
00313   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
00314   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
00315   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
00316   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
00317   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
00318   case NVPTXISD::Tld4UnifiedR2DS64Float:
00319   case NVPTXISD::Tld4UnifiedG2DS64Float:
00320   case NVPTXISD::Tld4UnifiedB2DS64Float:
00321   case NVPTXISD::Tld4UnifiedA2DS64Float:
00322   case NVPTXISD::Tld4UnifiedR2DU64Float:
00323   case NVPTXISD::Tld4UnifiedG2DU64Float:
00324   case NVPTXISD::Tld4UnifiedB2DU64Float:
00325   case NVPTXISD::Tld4UnifiedA2DU64Float:
00326     ResNode = SelectTextureIntrinsic(N);
00327     break;
00328   case NVPTXISD::Suld1DI8Clamp:
00329   case NVPTXISD::Suld1DI16Clamp:
00330   case NVPTXISD::Suld1DI32Clamp:
00331   case NVPTXISD::Suld1DI64Clamp:
00332   case NVPTXISD::Suld1DV2I8Clamp:
00333   case NVPTXISD::Suld1DV2I16Clamp:
00334   case NVPTXISD::Suld1DV2I32Clamp:
00335   case NVPTXISD::Suld1DV2I64Clamp:
00336   case NVPTXISD::Suld1DV4I8Clamp:
00337   case NVPTXISD::Suld1DV4I16Clamp:
00338   case NVPTXISD::Suld1DV4I32Clamp:
00339   case NVPTXISD::Suld1DArrayI8Clamp:
00340   case NVPTXISD::Suld1DArrayI16Clamp:
00341   case NVPTXISD::Suld1DArrayI32Clamp:
00342   case NVPTXISD::Suld1DArrayI64Clamp:
00343   case NVPTXISD::Suld1DArrayV2I8Clamp:
00344   case NVPTXISD::Suld1DArrayV2I16Clamp:
00345   case NVPTXISD::Suld1DArrayV2I32Clamp:
00346   case NVPTXISD::Suld1DArrayV2I64Clamp:
00347   case NVPTXISD::Suld1DArrayV4I8Clamp:
00348   case NVPTXISD::Suld1DArrayV4I16Clamp:
00349   case NVPTXISD::Suld1DArrayV4I32Clamp:
00350   case NVPTXISD::Suld2DI8Clamp:
00351   case NVPTXISD::Suld2DI16Clamp:
00352   case NVPTXISD::Suld2DI32Clamp:
00353   case NVPTXISD::Suld2DI64Clamp:
00354   case NVPTXISD::Suld2DV2I8Clamp:
00355   case NVPTXISD::Suld2DV2I16Clamp:
00356   case NVPTXISD::Suld2DV2I32Clamp:
00357   case NVPTXISD::Suld2DV2I64Clamp:
00358   case NVPTXISD::Suld2DV4I8Clamp:
00359   case NVPTXISD::Suld2DV4I16Clamp:
00360   case NVPTXISD::Suld2DV4I32Clamp:
00361   case NVPTXISD::Suld2DArrayI8Clamp:
00362   case NVPTXISD::Suld2DArrayI16Clamp:
00363   case NVPTXISD::Suld2DArrayI32Clamp:
00364   case NVPTXISD::Suld2DArrayI64Clamp:
00365   case NVPTXISD::Suld2DArrayV2I8Clamp:
00366   case NVPTXISD::Suld2DArrayV2I16Clamp:
00367   case NVPTXISD::Suld2DArrayV2I32Clamp:
00368   case NVPTXISD::Suld2DArrayV2I64Clamp:
00369   case NVPTXISD::Suld2DArrayV4I8Clamp:
00370   case NVPTXISD::Suld2DArrayV4I16Clamp:
00371   case NVPTXISD::Suld2DArrayV4I32Clamp:
00372   case NVPTXISD::Suld3DI8Clamp:
00373   case NVPTXISD::Suld3DI16Clamp:
00374   case NVPTXISD::Suld3DI32Clamp:
00375   case NVPTXISD::Suld3DI64Clamp:
00376   case NVPTXISD::Suld3DV2I8Clamp:
00377   case NVPTXISD::Suld3DV2I16Clamp:
00378   case NVPTXISD::Suld3DV2I32Clamp:
00379   case NVPTXISD::Suld3DV2I64Clamp:
00380   case NVPTXISD::Suld3DV4I8Clamp:
00381   case NVPTXISD::Suld3DV4I16Clamp:
00382   case NVPTXISD::Suld3DV4I32Clamp:
00383   case NVPTXISD::Suld1DI8Trap:
00384   case NVPTXISD::Suld1DI16Trap:
00385   case NVPTXISD::Suld1DI32Trap:
00386   case NVPTXISD::Suld1DI64Trap:
00387   case NVPTXISD::Suld1DV2I8Trap:
00388   case NVPTXISD::Suld1DV2I16Trap:
00389   case NVPTXISD::Suld1DV2I32Trap:
00390   case NVPTXISD::Suld1DV2I64Trap:
00391   case NVPTXISD::Suld1DV4I8Trap:
00392   case NVPTXISD::Suld1DV4I16Trap:
00393   case NVPTXISD::Suld1DV4I32Trap:
00394   case NVPTXISD::Suld1DArrayI8Trap:
00395   case NVPTXISD::Suld1DArrayI16Trap:
00396   case NVPTXISD::Suld1DArrayI32Trap:
00397   case NVPTXISD::Suld1DArrayI64Trap:
00398   case NVPTXISD::Suld1DArrayV2I8Trap:
00399   case NVPTXISD::Suld1DArrayV2I16Trap:
00400   case NVPTXISD::Suld1DArrayV2I32Trap:
00401   case NVPTXISD::Suld1DArrayV2I64Trap:
00402   case NVPTXISD::Suld1DArrayV4I8Trap:
00403   case NVPTXISD::Suld1DArrayV4I16Trap:
00404   case NVPTXISD::Suld1DArrayV4I32Trap:
00405   case NVPTXISD::Suld2DI8Trap:
00406   case NVPTXISD::Suld2DI16Trap:
00407   case NVPTXISD::Suld2DI32Trap:
00408   case NVPTXISD::Suld2DI64Trap:
00409   case NVPTXISD::Suld2DV2I8Trap:
00410   case NVPTXISD::Suld2DV2I16Trap:
00411   case NVPTXISD::Suld2DV2I32Trap:
00412   case NVPTXISD::Suld2DV2I64Trap:
00413   case NVPTXISD::Suld2DV4I8Trap:
00414   case NVPTXISD::Suld2DV4I16Trap:
00415   case NVPTXISD::Suld2DV4I32Trap:
00416   case NVPTXISD::Suld2DArrayI8Trap:
00417   case NVPTXISD::Suld2DArrayI16Trap:
00418   case NVPTXISD::Suld2DArrayI32Trap:
00419   case NVPTXISD::Suld2DArrayI64Trap:
00420   case NVPTXISD::Suld2DArrayV2I8Trap:
00421   case NVPTXISD::Suld2DArrayV2I16Trap:
00422   case NVPTXISD::Suld2DArrayV2I32Trap:
00423   case NVPTXISD::Suld2DArrayV2I64Trap:
00424   case NVPTXISD::Suld2DArrayV4I8Trap:
00425   case NVPTXISD::Suld2DArrayV4I16Trap:
00426   case NVPTXISD::Suld2DArrayV4I32Trap:
00427   case NVPTXISD::Suld3DI8Trap:
00428   case NVPTXISD::Suld3DI16Trap:
00429   case NVPTXISD::Suld3DI32Trap:
00430   case NVPTXISD::Suld3DI64Trap:
00431   case NVPTXISD::Suld3DV2I8Trap:
00432   case NVPTXISD::Suld3DV2I16Trap:
00433   case NVPTXISD::Suld3DV2I32Trap:
00434   case NVPTXISD::Suld3DV2I64Trap:
00435   case NVPTXISD::Suld3DV4I8Trap:
00436   case NVPTXISD::Suld3DV4I16Trap:
00437   case NVPTXISD::Suld3DV4I32Trap:
00438   case NVPTXISD::Suld1DI8Zero:
00439   case NVPTXISD::Suld1DI16Zero:
00440   case NVPTXISD::Suld1DI32Zero:
00441   case NVPTXISD::Suld1DI64Zero:
00442   case NVPTXISD::Suld1DV2I8Zero:
00443   case NVPTXISD::Suld1DV2I16Zero:
00444   case NVPTXISD::Suld1DV2I32Zero:
00445   case NVPTXISD::Suld1DV2I64Zero:
00446   case NVPTXISD::Suld1DV4I8Zero:
00447   case NVPTXISD::Suld1DV4I16Zero:
00448   case NVPTXISD::Suld1DV4I32Zero:
00449   case NVPTXISD::Suld1DArrayI8Zero:
00450   case NVPTXISD::Suld1DArrayI16Zero:
00451   case NVPTXISD::Suld1DArrayI32Zero:
00452   case NVPTXISD::Suld1DArrayI64Zero:
00453   case NVPTXISD::Suld1DArrayV2I8Zero:
00454   case NVPTXISD::Suld1DArrayV2I16Zero:
00455   case NVPTXISD::Suld1DArrayV2I32Zero:
00456   case NVPTXISD::Suld1DArrayV2I64Zero:
00457   case NVPTXISD::Suld1DArrayV4I8Zero:
00458   case NVPTXISD::Suld1DArrayV4I16Zero:
00459   case NVPTXISD::Suld1DArrayV4I32Zero:
00460   case NVPTXISD::Suld2DI8Zero:
00461   case NVPTXISD::Suld2DI16Zero:
00462   case NVPTXISD::Suld2DI32Zero:
00463   case NVPTXISD::Suld2DI64Zero:
00464   case NVPTXISD::Suld2DV2I8Zero:
00465   case NVPTXISD::Suld2DV2I16Zero:
00466   case NVPTXISD::Suld2DV2I32Zero:
00467   case NVPTXISD::Suld2DV2I64Zero:
00468   case NVPTXISD::Suld2DV4I8Zero:
00469   case NVPTXISD::Suld2DV4I16Zero:
00470   case NVPTXISD::Suld2DV4I32Zero:
00471   case NVPTXISD::Suld2DArrayI8Zero:
00472   case NVPTXISD::Suld2DArrayI16Zero:
00473   case NVPTXISD::Suld2DArrayI32Zero:
00474   case NVPTXISD::Suld2DArrayI64Zero:
00475   case NVPTXISD::Suld2DArrayV2I8Zero:
00476   case NVPTXISD::Suld2DArrayV2I16Zero:
00477   case NVPTXISD::Suld2DArrayV2I32Zero:
00478   case NVPTXISD::Suld2DArrayV2I64Zero:
00479   case NVPTXISD::Suld2DArrayV4I8Zero:
00480   case NVPTXISD::Suld2DArrayV4I16Zero:
00481   case NVPTXISD::Suld2DArrayV4I32Zero:
00482   case NVPTXISD::Suld3DI8Zero:
00483   case NVPTXISD::Suld3DI16Zero:
00484   case NVPTXISD::Suld3DI32Zero:
00485   case NVPTXISD::Suld3DI64Zero:
00486   case NVPTXISD::Suld3DV2I8Zero:
00487   case NVPTXISD::Suld3DV2I16Zero:
00488   case NVPTXISD::Suld3DV2I32Zero:
00489   case NVPTXISD::Suld3DV2I64Zero:
00490   case NVPTXISD::Suld3DV4I8Zero:
00491   case NVPTXISD::Suld3DV4I16Zero:
00492   case NVPTXISD::Suld3DV4I32Zero:
00493     ResNode = SelectSurfaceIntrinsic(N);
00494     break;
00495   case ISD::AND:
00496   case ISD::SRA:
00497   case ISD::SRL:
00498     // Try to select BFE
00499     ResNode = SelectBFE(N);
00500     break;
00501   case ISD::ADDRSPACECAST:
00502     ResNode = SelectAddrSpaceCast(N);
00503     break;
00504   default:
00505     break;
00506   }
00507   if (ResNode)
00508     return ResNode;
00509   return SelectCode(N);
00510 }
00511 
00512 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
00513   unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
00514   switch (IID) {
00515   default:
00516     return NULL;
00517   case Intrinsic::nvvm_ldg_global_f:
00518   case Intrinsic::nvvm_ldg_global_i:
00519   case Intrinsic::nvvm_ldg_global_p:
00520   case Intrinsic::nvvm_ldu_global_f:
00521   case Intrinsic::nvvm_ldu_global_i:
00522   case Intrinsic::nvvm_ldu_global_p:
00523     return SelectLDGLDU(N);
00524   }
00525 }
00526 
00527 static unsigned int getCodeAddrSpace(MemSDNode *N) {
00528   const Value *Src = N->getMemOperand()->getValue();
00529 
00530   if (!Src)
00531     return NVPTX::PTXLdStInstCode::GENERIC;
00532 
00533   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
00534     switch (PT->getAddressSpace()) {
00535     case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
00536     case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
00537     case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
00538     case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
00539     case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
00540     case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
00541     default: break;
00542     }
00543   }
00544   return NVPTX::PTXLdStInstCode::GENERIC;
00545 }
00546 
00547 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
00548   unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
00549   switch (IID) {
00550   default:
00551     return nullptr;
00552   case Intrinsic::nvvm_texsurf_handle_internal:
00553     return SelectTexSurfHandle(N);
00554   }
00555 }
00556 
00557 SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
00558   // Op 0 is the intrinsic ID
00559   SDValue Wrapper = N->getOperand(1);
00560   SDValue GlobalVal = Wrapper.getOperand(0);
00561   return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
00562                                 GlobalVal);
00563 }
00564 
00565 SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
00566   SDValue Src = N->getOperand(0);
00567   AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
00568   unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
00569   unsigned DstAddrSpace = CastN->getDestAddressSpace();
00570 
00571   assert(SrcAddrSpace != DstAddrSpace &&
00572          "addrspacecast must be between different address spaces");
00573 
00574   if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
00575     // Specific to generic
00576     unsigned Opc;
00577     switch (SrcAddrSpace) {
00578     default: report_fatal_error("Bad address space in addrspacecast");
00579     case ADDRESS_SPACE_GLOBAL:
00580       Opc = TM.is64Bit() ? NVPTX::cvta_global_yes_64 : NVPTX::cvta_global_yes;
00581       break;
00582     case ADDRESS_SPACE_SHARED:
00583       Opc = TM.is64Bit() ? NVPTX::cvta_shared_yes_64 : NVPTX::cvta_shared_yes;
00584       break;
00585     case ADDRESS_SPACE_CONST:
00586       Opc = TM.is64Bit() ? NVPTX::cvta_const_yes_64 : NVPTX::cvta_const_yes;
00587       break;
00588     case ADDRESS_SPACE_LOCAL:
00589       Opc = TM.is64Bit() ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes;
00590       break;
00591     }
00592     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00593   } else {
00594     // Generic to specific
00595     if (SrcAddrSpace != 0)
00596       report_fatal_error("Cannot cast between two non-generic address spaces");
00597     unsigned Opc;
00598     switch (DstAddrSpace) {
00599     default: report_fatal_error("Bad address space in addrspacecast");
00600     case ADDRESS_SPACE_GLOBAL:
00601       Opc = TM.is64Bit() ? NVPTX::cvta_to_global_yes_64
00602                          : NVPTX::cvta_to_global_yes;
00603       break;
00604     case ADDRESS_SPACE_SHARED:
00605       Opc = TM.is64Bit() ? NVPTX::cvta_to_shared_yes_64
00606                          : NVPTX::cvta_to_shared_yes;
00607       break;
00608     case ADDRESS_SPACE_CONST:
00609       Opc =
00610           TM.is64Bit() ? NVPTX::cvta_to_const_yes_64 : NVPTX::cvta_to_const_yes;
00611       break;
00612     case ADDRESS_SPACE_LOCAL:
00613       Opc =
00614           TM.is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes;
00615       break;
00616     case ADDRESS_SPACE_PARAM:
00617       Opc = TM.is64Bit() ? NVPTX::nvvm_ptr_gen_to_param_64
00618                          : NVPTX::nvvm_ptr_gen_to_param;
00619       break;
00620     }
00621     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00622   }
00623 }
00624 
00625 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
00626   SDLoc dl(N);
00627   LoadSDNode *LD = cast<LoadSDNode>(N);
00628   EVT LoadedVT = LD->getMemoryVT();
00629   SDNode *NVPTXLD = nullptr;
00630 
00631   // do not support pre/post inc/dec
00632   if (LD->isIndexed())
00633     return nullptr;
00634 
00635   if (!LoadedVT.isSimple())
00636     return nullptr;
00637 
00638   // Address Space Setting
00639   unsigned int codeAddrSpace = getCodeAddrSpace(LD);
00640 
00641   // Volatile Setting
00642   // - .volatile is only availalble for .global and .shared
00643   bool isVolatile = LD->isVolatile();
00644   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00645       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00646       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00647     isVolatile = false;
00648 
00649   // Vector Setting
00650   MVT SimpleVT = LoadedVT.getSimpleVT();
00651   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
00652   if (SimpleVT.isVector()) {
00653     unsigned num = SimpleVT.getVectorNumElements();
00654     if (num == 2)
00655       vecType = NVPTX::PTXLdStInstCode::V2;
00656     else if (num == 4)
00657       vecType = NVPTX::PTXLdStInstCode::V4;
00658     else
00659       return nullptr;
00660   }
00661 
00662   // Type Setting: fromType + fromTypeWidth
00663   //
00664   // Sign   : ISD::SEXTLOAD
00665   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00666   //          type is integer
00667   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00668   MVT ScalarVT = SimpleVT.getScalarType();
00669   // Read at least 8 bits (predicates are stored as 8-bit values)
00670   unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00671   unsigned int fromType;
00672   if ((LD->getExtensionType() == ISD::SEXTLOAD))
00673     fromType = NVPTX::PTXLdStInstCode::Signed;
00674   else if (ScalarVT.isFloatingPoint())
00675     fromType = NVPTX::PTXLdStInstCode::Float;
00676   else
00677     fromType = NVPTX::PTXLdStInstCode::Unsigned;
00678 
00679   // Create the machine instruction DAG
00680   SDValue Chain = N->getOperand(0);
00681   SDValue N1 = N->getOperand(1);
00682   SDValue Addr;
00683   SDValue Offset, Base;
00684   unsigned Opcode;
00685   MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
00686 
00687   if (SelectDirectAddr(N1, Addr)) {
00688     switch (TargetVT) {
00689     case MVT::i8:
00690       Opcode = NVPTX::LD_i8_avar;
00691       break;
00692     case MVT::i16:
00693       Opcode = NVPTX::LD_i16_avar;
00694       break;
00695     case MVT::i32:
00696       Opcode = NVPTX::LD_i32_avar;
00697       break;
00698     case MVT::i64:
00699       Opcode = NVPTX::LD_i64_avar;
00700       break;
00701     case MVT::f32:
00702       Opcode = NVPTX::LD_f32_avar;
00703       break;
00704     case MVT::f64:
00705       Opcode = NVPTX::LD_f64_avar;
00706       break;
00707     default:
00708       return nullptr;
00709     }
00710     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
00711                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
00712                       getI32Imm(fromTypeWidth, dl), Addr, Chain };
00713     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00714   } else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
00715                           : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
00716     switch (TargetVT) {
00717     case MVT::i8:
00718       Opcode = NVPTX::LD_i8_asi;
00719       break;
00720     case MVT::i16:
00721       Opcode = NVPTX::LD_i16_asi;
00722       break;
00723     case MVT::i32:
00724       Opcode = NVPTX::LD_i32_asi;
00725       break;
00726     case MVT::i64:
00727       Opcode = NVPTX::LD_i64_asi;
00728       break;
00729     case MVT::f32:
00730       Opcode = NVPTX::LD_f32_asi;
00731       break;
00732     case MVT::f64:
00733       Opcode = NVPTX::LD_f64_asi;
00734       break;
00735     default:
00736       return nullptr;
00737     }
00738     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
00739                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
00740                       getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
00741     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00742   } else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
00743                           : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
00744     if (TM.is64Bit()) {
00745       switch (TargetVT) {
00746       case MVT::i8:
00747         Opcode = NVPTX::LD_i8_ari_64;
00748         break;
00749       case MVT::i16:
00750         Opcode = NVPTX::LD_i16_ari_64;
00751         break;
00752       case MVT::i32:
00753         Opcode = NVPTX::LD_i32_ari_64;
00754         break;
00755       case MVT::i64:
00756         Opcode = NVPTX::LD_i64_ari_64;
00757         break;
00758       case MVT::f32:
00759         Opcode = NVPTX::LD_f32_ari_64;
00760         break;
00761       case MVT::f64:
00762         Opcode = NVPTX::LD_f64_ari_64;
00763         break;
00764       default:
00765         return nullptr;
00766       }
00767     } else {
00768       switch (TargetVT) {
00769       case MVT::i8:
00770         Opcode = NVPTX::LD_i8_ari;
00771         break;
00772       case MVT::i16:
00773         Opcode = NVPTX::LD_i16_ari;
00774         break;
00775       case MVT::i32:
00776         Opcode = NVPTX::LD_i32_ari;
00777         break;
00778       case MVT::i64:
00779         Opcode = NVPTX::LD_i64_ari;
00780         break;
00781       case MVT::f32:
00782         Opcode = NVPTX::LD_f32_ari;
00783         break;
00784       case MVT::f64:
00785         Opcode = NVPTX::LD_f64_ari;
00786         break;
00787       default:
00788         return nullptr;
00789       }
00790     }
00791     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
00792                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
00793                       getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
00794     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00795   } else {
00796     if (TM.is64Bit()) {
00797       switch (TargetVT) {
00798       case MVT::i8:
00799         Opcode = NVPTX::LD_i8_areg_64;
00800         break;
00801       case MVT::i16:
00802         Opcode = NVPTX::LD_i16_areg_64;
00803         break;
00804       case MVT::i32:
00805         Opcode = NVPTX::LD_i32_areg_64;
00806         break;
00807       case MVT::i64:
00808         Opcode = NVPTX::LD_i64_areg_64;
00809         break;
00810       case MVT::f32:
00811         Opcode = NVPTX::LD_f32_areg_64;
00812         break;
00813       case MVT::f64:
00814         Opcode = NVPTX::LD_f64_areg_64;
00815         break;
00816       default:
00817         return nullptr;
00818       }
00819     } else {
00820       switch (TargetVT) {
00821       case MVT::i8:
00822         Opcode = NVPTX::LD_i8_areg;
00823         break;
00824       case MVT::i16:
00825         Opcode = NVPTX::LD_i16_areg;
00826         break;
00827       case MVT::i32:
00828         Opcode = NVPTX::LD_i32_areg;
00829         break;
00830       case MVT::i64:
00831         Opcode = NVPTX::LD_i64_areg;
00832         break;
00833       case MVT::f32:
00834         Opcode = NVPTX::LD_f32_areg;
00835         break;
00836       case MVT::f64:
00837         Opcode = NVPTX::LD_f64_areg;
00838         break;
00839       default:
00840         return nullptr;
00841       }
00842     }
00843     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
00844                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
00845                       getI32Imm(fromTypeWidth, dl), N1, Chain };
00846     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00847   }
00848 
00849   if (NVPTXLD) {
00850     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
00851     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
00852     cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
00853   }
00854 
00855   return NVPTXLD;
00856 }
00857 
00858 SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
00859 
00860   SDValue Chain = N->getOperand(0);
00861   SDValue Op1 = N->getOperand(1);
00862   SDValue Addr, Offset, Base;
00863   unsigned Opcode;
00864   SDLoc DL(N);
00865   SDNode *LD;
00866   MemSDNode *MemSD = cast<MemSDNode>(N);
00867   EVT LoadedVT = MemSD->getMemoryVT();
00868 
00869   if (!LoadedVT.isSimple())
00870     return nullptr;
00871 
00872   // Address Space Setting
00873   unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
00874 
00875   // Volatile Setting
00876   // - .volatile is only availalble for .global and .shared
00877   bool IsVolatile = MemSD->isVolatile();
00878   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00879       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00880       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00881     IsVolatile = false;
00882 
00883   // Vector Setting
00884   MVT SimpleVT = LoadedVT.getSimpleVT();
00885 
00886   // Type Setting: fromType + fromTypeWidth
00887   //
00888   // Sign   : ISD::SEXTLOAD
00889   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00890   //          type is integer
00891   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00892   MVT ScalarVT = SimpleVT.getScalarType();
00893   // Read at least 8 bits (predicates are stored as 8-bit values)
00894   unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00895   unsigned int FromType;
00896   // The last operand holds the original LoadSDNode::getExtensionType() value
00897   unsigned ExtensionType = cast<ConstantSDNode>(
00898       N->getOperand(N->getNumOperands() - 1))->getZExtValue();
00899   if (ExtensionType == ISD::SEXTLOAD)
00900     FromType = NVPTX::PTXLdStInstCode::Signed;
00901   else if (ScalarVT.isFloatingPoint())
00902     FromType = NVPTX::PTXLdStInstCode::Float;
00903   else
00904     FromType = NVPTX::PTXLdStInstCode::Unsigned;
00905 
00906   unsigned VecType;
00907 
00908   switch (N->getOpcode()) {
00909   case NVPTXISD::LoadV2:
00910     VecType = NVPTX::PTXLdStInstCode::V2;
00911     break;
00912   case NVPTXISD::LoadV4:
00913     VecType = NVPTX::PTXLdStInstCode::V4;
00914     break;
00915   default:
00916     return nullptr;
00917   }
00918 
00919   EVT EltVT = N->getValueType(0);
00920 
00921   if (SelectDirectAddr(Op1, Addr)) {
00922     switch (N->getOpcode()) {
00923     default:
00924       return nullptr;
00925     case NVPTXISD::LoadV2:
00926       switch (EltVT.getSimpleVT().SimpleTy) {
00927       default:
00928         return nullptr;
00929       case MVT::i8:
00930         Opcode = NVPTX::LDV_i8_v2_avar;
00931         break;
00932       case MVT::i16:
00933         Opcode = NVPTX::LDV_i16_v2_avar;
00934         break;
00935       case MVT::i32:
00936         Opcode = NVPTX::LDV_i32_v2_avar;
00937         break;
00938       case MVT::i64:
00939         Opcode = NVPTX::LDV_i64_v2_avar;
00940         break;
00941       case MVT::f32:
00942         Opcode = NVPTX::LDV_f32_v2_avar;
00943         break;
00944       case MVT::f64:
00945         Opcode = NVPTX::LDV_f64_v2_avar;
00946         break;
00947       }
00948       break;
00949     case NVPTXISD::LoadV4:
00950       switch (EltVT.getSimpleVT().SimpleTy) {
00951       default:
00952         return nullptr;
00953       case MVT::i8:
00954         Opcode = NVPTX::LDV_i8_v4_avar;
00955         break;
00956       case MVT::i16:
00957         Opcode = NVPTX::LDV_i16_v4_avar;
00958         break;
00959       case MVT::i32:
00960         Opcode = NVPTX::LDV_i32_v4_avar;
00961         break;
00962       case MVT::f32:
00963         Opcode = NVPTX::LDV_f32_v4_avar;
00964         break;
00965       }
00966       break;
00967     }
00968 
00969     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
00970                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
00971                       getI32Imm(FromTypeWidth, DL), Addr, Chain };
00972     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
00973   } else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
00974                           : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
00975     switch (N->getOpcode()) {
00976     default:
00977       return nullptr;
00978     case NVPTXISD::LoadV2:
00979       switch (EltVT.getSimpleVT().SimpleTy) {
00980       default:
00981         return nullptr;
00982       case MVT::i8:
00983         Opcode = NVPTX::LDV_i8_v2_asi;
00984         break;
00985       case MVT::i16:
00986         Opcode = NVPTX::LDV_i16_v2_asi;
00987         break;
00988       case MVT::i32:
00989         Opcode = NVPTX::LDV_i32_v2_asi;
00990         break;
00991       case MVT::i64:
00992         Opcode = NVPTX::LDV_i64_v2_asi;
00993         break;
00994       case MVT::f32:
00995         Opcode = NVPTX::LDV_f32_v2_asi;
00996         break;
00997       case MVT::f64:
00998         Opcode = NVPTX::LDV_f64_v2_asi;
00999         break;
01000       }
01001       break;
01002     case NVPTXISD::LoadV4:
01003       switch (EltVT.getSimpleVT().SimpleTy) {
01004       default:
01005         return nullptr;
01006       case MVT::i8:
01007         Opcode = NVPTX::LDV_i8_v4_asi;
01008         break;
01009       case MVT::i16:
01010         Opcode = NVPTX::LDV_i16_v4_asi;
01011         break;
01012       case MVT::i32:
01013         Opcode = NVPTX::LDV_i32_v4_asi;
01014         break;
01015       case MVT::f32:
01016         Opcode = NVPTX::LDV_f32_v4_asi;
01017         break;
01018       }
01019       break;
01020     }
01021 
01022     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
01023                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
01024                       getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
01025     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01026   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
01027                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
01028     if (TM.is64Bit()) {
01029       switch (N->getOpcode()) {
01030       default:
01031         return nullptr;
01032       case NVPTXISD::LoadV2:
01033         switch (EltVT.getSimpleVT().SimpleTy) {
01034         default:
01035           return nullptr;
01036         case MVT::i8:
01037           Opcode = NVPTX::LDV_i8_v2_ari_64;
01038           break;
01039         case MVT::i16:
01040           Opcode = NVPTX::LDV_i16_v2_ari_64;
01041           break;
01042         case MVT::i32:
01043           Opcode = NVPTX::LDV_i32_v2_ari_64;
01044           break;
01045         case MVT::i64:
01046           Opcode = NVPTX::LDV_i64_v2_ari_64;
01047           break;
01048         case MVT::f32:
01049           Opcode = NVPTX::LDV_f32_v2_ari_64;
01050           break;
01051         case MVT::f64:
01052           Opcode = NVPTX::LDV_f64_v2_ari_64;
01053           break;
01054         }
01055         break;
01056       case NVPTXISD::LoadV4:
01057         switch (EltVT.getSimpleVT().SimpleTy) {
01058         default:
01059           return nullptr;
01060         case MVT::i8:
01061           Opcode = NVPTX::LDV_i8_v4_ari_64;
01062           break;
01063         case MVT::i16:
01064           Opcode = NVPTX::LDV_i16_v4_ari_64;
01065           break;
01066         case MVT::i32:
01067           Opcode = NVPTX::LDV_i32_v4_ari_64;
01068           break;
01069         case MVT::f32:
01070           Opcode = NVPTX::LDV_f32_v4_ari_64;
01071           break;
01072         }
01073         break;
01074       }
01075     } else {
01076       switch (N->getOpcode()) {
01077       default:
01078         return nullptr;
01079       case NVPTXISD::LoadV2:
01080         switch (EltVT.getSimpleVT().SimpleTy) {
01081         default:
01082           return nullptr;
01083         case MVT::i8:
01084           Opcode = NVPTX::LDV_i8_v2_ari;
01085           break;
01086         case MVT::i16:
01087           Opcode = NVPTX::LDV_i16_v2_ari;
01088           break;
01089         case MVT::i32:
01090           Opcode = NVPTX::LDV_i32_v2_ari;
01091           break;
01092         case MVT::i64:
01093           Opcode = NVPTX::LDV_i64_v2_ari;
01094           break;
01095         case MVT::f32:
01096           Opcode = NVPTX::LDV_f32_v2_ari;
01097           break;
01098         case MVT::f64:
01099           Opcode = NVPTX::LDV_f64_v2_ari;
01100           break;
01101         }
01102         break;
01103       case NVPTXISD::LoadV4:
01104         switch (EltVT.getSimpleVT().SimpleTy) {
01105         default:
01106           return nullptr;
01107         case MVT::i8:
01108           Opcode = NVPTX::LDV_i8_v4_ari;
01109           break;
01110         case MVT::i16:
01111           Opcode = NVPTX::LDV_i16_v4_ari;
01112           break;
01113         case MVT::i32:
01114           Opcode = NVPTX::LDV_i32_v4_ari;
01115           break;
01116         case MVT::f32:
01117           Opcode = NVPTX::LDV_f32_v4_ari;
01118           break;
01119         }
01120         break;
01121       }
01122     }
01123 
01124     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
01125                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
01126                       getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
01127 
01128     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01129   } else {
01130     if (TM.is64Bit()) {
01131       switch (N->getOpcode()) {
01132       default:
01133         return nullptr;
01134       case NVPTXISD::LoadV2:
01135         switch (EltVT.getSimpleVT().SimpleTy) {
01136         default:
01137           return nullptr;
01138         case MVT::i8:
01139           Opcode = NVPTX::LDV_i8_v2_areg_64;
01140           break;
01141         case MVT::i16:
01142           Opcode = NVPTX::LDV_i16_v2_areg_64;
01143           break;
01144         case MVT::i32:
01145           Opcode = NVPTX::LDV_i32_v2_areg_64;
01146           break;
01147         case MVT::i64:
01148           Opcode = NVPTX::LDV_i64_v2_areg_64;
01149           break;
01150         case MVT::f32:
01151           Opcode = NVPTX::LDV_f32_v2_areg_64;
01152           break;
01153         case MVT::f64:
01154           Opcode = NVPTX::LDV_f64_v2_areg_64;
01155           break;
01156         }
01157         break;
01158       case NVPTXISD::LoadV4:
01159         switch (EltVT.getSimpleVT().SimpleTy) {
01160         default:
01161           return nullptr;
01162         case MVT::i8:
01163           Opcode = NVPTX::LDV_i8_v4_areg_64;
01164           break;
01165         case MVT::i16:
01166           Opcode = NVPTX::LDV_i16_v4_areg_64;
01167           break;
01168         case MVT::i32:
01169           Opcode = NVPTX::LDV_i32_v4_areg_64;
01170           break;
01171         case MVT::f32:
01172           Opcode = NVPTX::LDV_f32_v4_areg_64;
01173           break;
01174         }
01175         break;
01176       }
01177     } else {
01178       switch (N->getOpcode()) {
01179       default:
01180         return nullptr;
01181       case NVPTXISD::LoadV2:
01182         switch (EltVT.getSimpleVT().SimpleTy) {
01183         default:
01184           return nullptr;
01185         case MVT::i8:
01186           Opcode = NVPTX::LDV_i8_v2_areg;
01187           break;
01188         case MVT::i16:
01189           Opcode = NVPTX::LDV_i16_v2_areg;
01190           break;
01191         case MVT::i32:
01192           Opcode = NVPTX::LDV_i32_v2_areg;
01193           break;
01194         case MVT::i64:
01195           Opcode = NVPTX::LDV_i64_v2_areg;
01196           break;
01197         case MVT::f32:
01198           Opcode = NVPTX::LDV_f32_v2_areg;
01199           break;
01200         case MVT::f64:
01201           Opcode = NVPTX::LDV_f64_v2_areg;
01202           break;
01203         }
01204         break;
01205       case NVPTXISD::LoadV4:
01206         switch (EltVT.getSimpleVT().SimpleTy) {
01207         default:
01208           return nullptr;
01209         case MVT::i8:
01210           Opcode = NVPTX::LDV_i8_v4_areg;
01211           break;
01212         case MVT::i16:
01213           Opcode = NVPTX::LDV_i16_v4_areg;
01214           break;
01215         case MVT::i32:
01216           Opcode = NVPTX::LDV_i32_v4_areg;
01217           break;
01218         case MVT::f32:
01219           Opcode = NVPTX::LDV_f32_v4_areg;
01220           break;
01221         }
01222         break;
01223       }
01224     }
01225 
01226     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
01227                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
01228                       getI32Imm(FromTypeWidth, DL), Op1, Chain };
01229     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01230   }
01231 
01232   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01233   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
01234   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
01235 
01236   return LD;
01237 }
01238 
01239 SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
01240 
01241   SDValue Chain = N->getOperand(0);
01242   SDValue Op1;
01243   MemSDNode *Mem;
01244   bool IsLDG = true;
01245 
01246   // If this is an LDG intrinsic, the address is the third operand. Its its an
01247   // LDG/LDU SD node (from custom vector handling), then its the second operand
01248   if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
01249     Op1 = N->getOperand(2);
01250     Mem = cast<MemIntrinsicSDNode>(N);
01251     unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
01252     switch (IID) {
01253     default:
01254       return NULL;
01255     case Intrinsic::nvvm_ldg_global_f:
01256     case Intrinsic::nvvm_ldg_global_i:
01257     case Intrinsic::nvvm_ldg_global_p:
01258       IsLDG = true;
01259       break;
01260     case Intrinsic::nvvm_ldu_global_f:
01261     case Intrinsic::nvvm_ldu_global_i:
01262     case Intrinsic::nvvm_ldu_global_p:
01263       IsLDG = false;
01264       break;
01265     }
01266   } else {
01267     Op1 = N->getOperand(1);
01268     Mem = cast<MemSDNode>(N);
01269   }
01270 
01271   unsigned Opcode;
01272   SDLoc DL(N);
01273   SDNode *LD;
01274   SDValue Base, Offset, Addr;
01275 
01276   EVT EltVT = Mem->getMemoryVT();
01277   if (EltVT.isVector()) {
01278     EltVT = EltVT.getVectorElementType();
01279   }
01280 
01281   if (SelectDirectAddr(Op1, Addr)) {
01282     switch (N->getOpcode()) {
01283     default:
01284       return nullptr;
01285     case ISD::INTRINSIC_W_CHAIN:
01286       if (IsLDG) {
01287         switch (EltVT.getSimpleVT().SimpleTy) {
01288         default:
01289           return nullptr;
01290         case MVT::i8:
01291           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
01292           break;
01293         case MVT::i16:
01294           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
01295           break;
01296         case MVT::i32:
01297           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
01298           break;
01299         case MVT::i64:
01300           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
01301           break;
01302         case MVT::f32:
01303           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
01304           break;
01305         case MVT::f64:
01306           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
01307           break;
01308         }
01309       } else {
01310         switch (EltVT.getSimpleVT().SimpleTy) {
01311         default:
01312           return nullptr;
01313         case MVT::i8:
01314           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
01315           break;
01316         case MVT::i16:
01317           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
01318           break;
01319         case MVT::i32:
01320           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
01321           break;
01322         case MVT::i64:
01323           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
01324           break;
01325         case MVT::f32:
01326           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
01327           break;
01328         case MVT::f64:
01329           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
01330           break;
01331         }
01332       }
01333       break;
01334     case NVPTXISD::LDGV2:
01335       switch (EltVT.getSimpleVT().SimpleTy) {
01336       default:
01337         return nullptr;
01338       case MVT::i8:
01339         Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
01340         break;
01341       case MVT::i16:
01342         Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
01343         break;
01344       case MVT::i32:
01345         Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
01346         break;
01347       case MVT::i64:
01348         Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
01349         break;
01350       case MVT::f32:
01351         Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
01352         break;
01353       case MVT::f64:
01354         Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
01355         break;
01356       }
01357       break;
01358     case NVPTXISD::LDUV2:
01359       switch (EltVT.getSimpleVT().SimpleTy) {
01360       default:
01361         return nullptr;
01362       case MVT::i8:
01363         Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
01364         break;
01365       case MVT::i16:
01366         Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
01367         break;
01368       case MVT::i32:
01369         Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
01370         break;
01371       case MVT::i64:
01372         Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
01373         break;
01374       case MVT::f32:
01375         Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
01376         break;
01377       case MVT::f64:
01378         Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
01379         break;
01380       }
01381       break;
01382     case NVPTXISD::LDGV4:
01383       switch (EltVT.getSimpleVT().SimpleTy) {
01384       default:
01385         return nullptr;
01386       case MVT::i8:
01387         Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
01388         break;
01389       case MVT::i16:
01390         Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
01391         break;
01392       case MVT::i32:
01393         Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
01394         break;
01395       case MVT::f32:
01396         Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
01397         break;
01398       }
01399       break;
01400     case NVPTXISD::LDUV4:
01401       switch (EltVT.getSimpleVT().SimpleTy) {
01402       default:
01403         return nullptr;
01404       case MVT::i8:
01405         Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
01406         break;
01407       case MVT::i16:
01408         Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
01409         break;
01410       case MVT::i32:
01411         Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
01412         break;
01413       case MVT::f32:
01414         Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
01415         break;
01416       }
01417       break;
01418     }
01419 
01420     SDValue Ops[] = { Addr, Chain };
01421     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01422   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
01423                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
01424     if (TM.is64Bit()) {
01425       switch (N->getOpcode()) {
01426       default:
01427         return nullptr;
01428       case ISD::INTRINSIC_W_CHAIN:
01429         if (IsLDG) {
01430           switch (EltVT.getSimpleVT().SimpleTy) {
01431           default:
01432             return nullptr;
01433           case MVT::i8:
01434             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
01435             break;
01436           case MVT::i16:
01437             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
01438             break;
01439           case MVT::i32:
01440             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
01441             break;
01442           case MVT::i64:
01443             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
01444             break;
01445           case MVT::f32:
01446             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
01447             break;
01448           case MVT::f64:
01449             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
01450             break;
01451           }
01452         } else {
01453           switch (EltVT.getSimpleVT().SimpleTy) {
01454           default:
01455             return nullptr;
01456           case MVT::i8:
01457             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
01458             break;
01459           case MVT::i16:
01460             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
01461             break;
01462           case MVT::i32:
01463             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
01464             break;
01465           case MVT::i64:
01466             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
01467             break;
01468           case MVT::f32:
01469             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
01470             break;
01471           case MVT::f64:
01472             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
01473             break;
01474           }
01475         }
01476         break;
01477       case NVPTXISD::LDGV2:
01478         switch (EltVT.getSimpleVT().SimpleTy) {
01479         default:
01480           return nullptr;
01481         case MVT::i8:
01482           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
01483           break;
01484         case MVT::i16:
01485           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
01486           break;
01487         case MVT::i32:
01488           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
01489           break;
01490         case MVT::i64:
01491           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
01492           break;
01493         case MVT::f32:
01494           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
01495           break;
01496         case MVT::f64:
01497           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
01498           break;
01499         }
01500         break;
01501       case NVPTXISD::LDUV2:
01502         switch (EltVT.getSimpleVT().SimpleTy) {
01503         default:
01504           return nullptr;
01505         case MVT::i8:
01506           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
01507           break;
01508         case MVT::i16:
01509           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
01510           break;
01511         case MVT::i32:
01512           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
01513           break;
01514         case MVT::i64:
01515           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
01516           break;
01517         case MVT::f32:
01518           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
01519           break;
01520         case MVT::f64:
01521           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
01522           break;
01523         }
01524         break;
01525       case NVPTXISD::LDGV4:
01526         switch (EltVT.getSimpleVT().SimpleTy) {
01527         default:
01528           return nullptr;
01529         case MVT::i8:
01530           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
01531           break;
01532         case MVT::i16:
01533           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
01534           break;
01535         case MVT::i32:
01536           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
01537           break;
01538         case MVT::f32:
01539           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
01540           break;
01541         }
01542         break;
01543       case NVPTXISD::LDUV4:
01544         switch (EltVT.getSimpleVT().SimpleTy) {
01545         default:
01546           return nullptr;
01547         case MVT::i8:
01548           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
01549           break;
01550         case MVT::i16:
01551           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
01552           break;
01553         case MVT::i32:
01554           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
01555           break;
01556         case MVT::f32:
01557           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
01558           break;
01559         }
01560         break;
01561       }
01562     } else {
01563       switch (N->getOpcode()) {
01564       default:
01565         return nullptr;
01566       case ISD::INTRINSIC_W_CHAIN:
01567         if (IsLDG) {
01568           switch (EltVT.getSimpleVT().SimpleTy) {
01569           default:
01570             return nullptr;
01571           case MVT::i8:
01572             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
01573             break;
01574           case MVT::i16:
01575             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
01576             break;
01577           case MVT::i32:
01578             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
01579             break;
01580           case MVT::i64:
01581             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
01582             break;
01583           case MVT::f32:
01584             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
01585             break;
01586           case MVT::f64:
01587             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
01588             break;
01589           }
01590         } else {
01591           switch (EltVT.getSimpleVT().SimpleTy) {
01592           default:
01593             return nullptr;
01594           case MVT::i8:
01595             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
01596             break;
01597           case MVT::i16:
01598             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
01599             break;
01600           case MVT::i32:
01601             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
01602             break;
01603           case MVT::i64:
01604             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
01605             break;
01606           case MVT::f32:
01607             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
01608             break;
01609           case MVT::f64:
01610             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
01611             break;
01612           }
01613         }
01614         break;
01615       case NVPTXISD::LDGV2:
01616         switch (EltVT.getSimpleVT().SimpleTy) {
01617         default:
01618           return nullptr;
01619         case MVT::i8:
01620           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
01621           break;
01622         case MVT::i16:
01623           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
01624           break;
01625         case MVT::i32:
01626           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
01627           break;
01628         case MVT::i64:
01629           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
01630           break;
01631         case MVT::f32:
01632           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
01633           break;
01634         case MVT::f64:
01635           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
01636           break;
01637         }
01638         break;
01639       case NVPTXISD::LDUV2:
01640         switch (EltVT.getSimpleVT().SimpleTy) {
01641         default:
01642           return nullptr;
01643         case MVT::i8:
01644           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
01645           break;
01646         case MVT::i16:
01647           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
01648           break;
01649         case MVT::i32:
01650           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
01651           break;
01652         case MVT::i64:
01653           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
01654           break;
01655         case MVT::f32:
01656           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
01657           break;
01658         case MVT::f64:
01659           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
01660           break;
01661         }
01662         break;
01663       case NVPTXISD::LDGV4:
01664         switch (EltVT.getSimpleVT().SimpleTy) {
01665         default:
01666           return nullptr;
01667         case MVT::i8:
01668           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
01669           break;
01670         case MVT::i16:
01671           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
01672           break;
01673         case MVT::i32:
01674           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
01675           break;
01676         case MVT::f32:
01677           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
01678           break;
01679         }
01680         break;
01681       case NVPTXISD::LDUV4:
01682         switch (EltVT.getSimpleVT().SimpleTy) {
01683         default:
01684           return nullptr;
01685         case MVT::i8:
01686           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
01687           break;
01688         case MVT::i16:
01689           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
01690           break;
01691         case MVT::i32:
01692           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
01693           break;
01694         case MVT::f32:
01695           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
01696           break;
01697         }
01698         break;
01699       }
01700     }
01701 
01702     SDValue Ops[] = { Base, Offset, Chain };
01703 
01704     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01705   } else {
01706     if (TM.is64Bit()) {
01707       switch (N->getOpcode()) {
01708       default:
01709         return nullptr;
01710       case ISD::INTRINSIC_W_CHAIN:
01711         if (IsLDG) {
01712           switch (EltVT.getSimpleVT().SimpleTy) {
01713           default:
01714             return nullptr;
01715           case MVT::i8:
01716             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
01717             break;
01718           case MVT::i16:
01719             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
01720             break;
01721           case MVT::i32:
01722             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
01723             break;
01724           case MVT::i64:
01725             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
01726             break;
01727           case MVT::f32:
01728             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
01729             break;
01730           case MVT::f64:
01731             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
01732             break;
01733           }
01734         } else {
01735           switch (EltVT.getSimpleVT().SimpleTy) {
01736           default:
01737             return nullptr;
01738           case MVT::i8:
01739             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
01740             break;
01741           case MVT::i16:
01742             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
01743             break;
01744           case MVT::i32:
01745             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
01746             break;
01747           case MVT::i64:
01748             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
01749             break;
01750           case MVT::f32:
01751             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
01752             break;
01753           case MVT::f64:
01754             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
01755             break;
01756           }
01757         }
01758         break;
01759       case NVPTXISD::LDGV2:
01760         switch (EltVT.getSimpleVT().SimpleTy) {
01761         default:
01762           return nullptr;
01763         case MVT::i8:
01764           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
01765           break;
01766         case MVT::i16:
01767           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
01768           break;
01769         case MVT::i32:
01770           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
01771           break;
01772         case MVT::i64:
01773           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
01774           break;
01775         case MVT::f32:
01776           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
01777           break;
01778         case MVT::f64:
01779           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
01780           break;
01781         }
01782         break;
01783       case NVPTXISD::LDUV2:
01784         switch (EltVT.getSimpleVT().SimpleTy) {
01785         default:
01786           return nullptr;
01787         case MVT::i8:
01788           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
01789           break;
01790         case MVT::i16:
01791           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
01792           break;
01793         case MVT::i32:
01794           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
01795           break;
01796         case MVT::i64:
01797           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
01798           break;
01799         case MVT::f32:
01800           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
01801           break;
01802         case MVT::f64:
01803           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
01804           break;
01805         }
01806         break;
01807       case NVPTXISD::LDGV4:
01808         switch (EltVT.getSimpleVT().SimpleTy) {
01809         default:
01810           return nullptr;
01811         case MVT::i8:
01812           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
01813           break;
01814         case MVT::i16:
01815           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
01816           break;
01817         case MVT::i32:
01818           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
01819           break;
01820         case MVT::f32:
01821           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
01822           break;
01823         }
01824         break;
01825       case NVPTXISD::LDUV4:
01826         switch (EltVT.getSimpleVT().SimpleTy) {
01827         default:
01828           return nullptr;
01829         case MVT::i8:
01830           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
01831           break;
01832         case MVT::i16:
01833           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
01834           break;
01835         case MVT::i32:
01836           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
01837           break;
01838         case MVT::f32:
01839           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
01840           break;
01841         }
01842         break;
01843       }
01844     } else {
01845       switch (N->getOpcode()) {
01846       default:
01847         return nullptr;
01848       case ISD::INTRINSIC_W_CHAIN:
01849         if (IsLDG) {
01850           switch (EltVT.getSimpleVT().SimpleTy) {
01851           default:
01852             return nullptr;
01853           case MVT::i8:
01854             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
01855             break;
01856           case MVT::i16:
01857             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
01858             break;
01859           case MVT::i32:
01860             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
01861             break;
01862           case MVT::i64:
01863             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
01864             break;
01865           case MVT::f32:
01866             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
01867             break;
01868           case MVT::f64:
01869             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
01870             break;
01871           }
01872         } else {
01873           switch (EltVT.getSimpleVT().SimpleTy) {
01874           default:
01875             return nullptr;
01876           case MVT::i8:
01877             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
01878             break;
01879           case MVT::i16:
01880             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
01881             break;
01882           case MVT::i32:
01883             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
01884             break;
01885           case MVT::i64:
01886             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
01887             break;
01888           case MVT::f32:
01889             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
01890             break;
01891           case MVT::f64:
01892             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
01893             break;
01894           }
01895         }
01896         break;
01897       case NVPTXISD::LDGV2:
01898         switch (EltVT.getSimpleVT().SimpleTy) {
01899         default:
01900           return nullptr;
01901         case MVT::i8:
01902           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
01903           break;
01904         case MVT::i16:
01905           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
01906           break;
01907         case MVT::i32:
01908           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
01909           break;
01910         case MVT::i64:
01911           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
01912           break;
01913         case MVT::f32:
01914           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
01915           break;
01916         case MVT::f64:
01917           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
01918           break;
01919         }
01920         break;
01921       case NVPTXISD::LDUV2:
01922         switch (EltVT.getSimpleVT().SimpleTy) {
01923         default:
01924           return nullptr;
01925         case MVT::i8:
01926           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
01927           break;
01928         case MVT::i16:
01929           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
01930           break;
01931         case MVT::i32:
01932           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
01933           break;
01934         case MVT::i64:
01935           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
01936           break;
01937         case MVT::f32:
01938           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
01939           break;
01940         case MVT::f64:
01941           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
01942           break;
01943         }
01944         break;
01945       case NVPTXISD::LDGV4:
01946         switch (EltVT.getSimpleVT().SimpleTy) {
01947         default:
01948           return nullptr;
01949         case MVT::i8:
01950           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
01951           break;
01952         case MVT::i16:
01953           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
01954           break;
01955         case MVT::i32:
01956           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
01957           break;
01958         case MVT::f32:
01959           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
01960           break;
01961         }
01962         break;
01963       case NVPTXISD::LDUV4:
01964         switch (EltVT.getSimpleVT().SimpleTy) {
01965         default:
01966           return nullptr;
01967         case MVT::i8:
01968           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
01969           break;
01970         case MVT::i16:
01971           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
01972           break;
01973         case MVT::i32:
01974           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
01975           break;
01976         case MVT::f32:
01977           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
01978           break;
01979         }
01980         break;
01981       }
01982     }
01983 
01984     SDValue Ops[] = { Op1, Chain };
01985     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01986   }
01987 
01988   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01989   MemRefs0[0] = Mem->getMemOperand();
01990   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
01991 
01992   return LD;
01993 }
01994 
01995 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
01996   SDLoc dl(N);
01997   StoreSDNode *ST = cast<StoreSDNode>(N);
01998   EVT StoreVT = ST->getMemoryVT();
01999   SDNode *NVPTXST = nullptr;
02000 
02001   // do not support pre/post inc/dec
02002   if (ST->isIndexed())
02003     return nullptr;
02004 
02005   if (!StoreVT.isSimple())
02006     return nullptr;
02007 
02008   // Address Space Setting
02009   unsigned int codeAddrSpace = getCodeAddrSpace(ST);
02010 
02011   // Volatile Setting
02012   // - .volatile is only availalble for .global and .shared
02013   bool isVolatile = ST->isVolatile();
02014   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
02015       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
02016       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
02017     isVolatile = false;
02018 
02019   // Vector Setting
02020   MVT SimpleVT = StoreVT.getSimpleVT();
02021   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
02022   if (SimpleVT.isVector()) {
02023     unsigned num = SimpleVT.getVectorNumElements();
02024     if (num == 2)
02025       vecType = NVPTX::PTXLdStInstCode::V2;
02026     else if (num == 4)
02027       vecType = NVPTX::PTXLdStInstCode::V4;
02028     else
02029       return nullptr;
02030   }
02031 
02032   // Type Setting: toType + toTypeWidth
02033   // - for integer type, always use 'u'
02034   //
02035   MVT ScalarVT = SimpleVT.getScalarType();
02036   unsigned toTypeWidth = ScalarVT.getSizeInBits();
02037   unsigned int toType;
02038   if (ScalarVT.isFloatingPoint())
02039     toType = NVPTX::PTXLdStInstCode::Float;
02040   else
02041     toType = NVPTX::PTXLdStInstCode::Unsigned;
02042 
02043   // Create the machine instruction DAG
02044   SDValue Chain = N->getOperand(0);
02045   SDValue N1 = N->getOperand(1);
02046   SDValue N2 = N->getOperand(2);
02047   SDValue Addr;
02048   SDValue Offset, Base;
02049   unsigned Opcode;
02050   MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
02051 
02052   if (SelectDirectAddr(N2, Addr)) {
02053     switch (SourceVT) {
02054     case MVT::i8:
02055       Opcode = NVPTX::ST_i8_avar;
02056       break;
02057     case MVT::i16:
02058       Opcode = NVPTX::ST_i16_avar;
02059       break;
02060     case MVT::i32:
02061       Opcode = NVPTX::ST_i32_avar;
02062       break;
02063     case MVT::i64:
02064       Opcode = NVPTX::ST_i64_avar;
02065       break;
02066     case MVT::f32:
02067       Opcode = NVPTX::ST_f32_avar;
02068       break;
02069     case MVT::f64:
02070       Opcode = NVPTX::ST_f64_avar;
02071       break;
02072     default:
02073       return nullptr;
02074     }
02075     SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
02076                       getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
02077                       getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Addr,
02078                       Chain };
02079     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02080   } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
02081                           : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
02082     switch (SourceVT) {
02083     case MVT::i8:
02084       Opcode = NVPTX::ST_i8_asi;
02085       break;
02086     case MVT::i16:
02087       Opcode = NVPTX::ST_i16_asi;
02088       break;
02089     case MVT::i32:
02090       Opcode = NVPTX::ST_i32_asi;
02091       break;
02092     case MVT::i64:
02093       Opcode = NVPTX::ST_i64_asi;
02094       break;
02095     case MVT::f32:
02096       Opcode = NVPTX::ST_f32_asi;
02097       break;
02098     case MVT::f64:
02099       Opcode = NVPTX::ST_f64_asi;
02100       break;
02101     default:
02102       return nullptr;
02103     }
02104     SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
02105                       getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
02106                       getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
02107                       Offset, Chain };
02108     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02109   } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
02110                           : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
02111     if (TM.is64Bit()) {
02112       switch (SourceVT) {
02113       case MVT::i8:
02114         Opcode = NVPTX::ST_i8_ari_64;
02115         break;
02116       case MVT::i16:
02117         Opcode = NVPTX::ST_i16_ari_64;
02118         break;
02119       case MVT::i32:
02120         Opcode = NVPTX::ST_i32_ari_64;
02121         break;
02122       case MVT::i64:
02123         Opcode = NVPTX::ST_i64_ari_64;
02124         break;
02125       case MVT::f32:
02126         Opcode = NVPTX::ST_f32_ari_64;
02127         break;
02128       case MVT::f64:
02129         Opcode = NVPTX::ST_f64_ari_64;
02130         break;
02131       default:
02132         return nullptr;
02133       }
02134     } else {
02135       switch (SourceVT) {
02136       case MVT::i8:
02137         Opcode = NVPTX::ST_i8_ari;
02138         break;
02139       case MVT::i16:
02140         Opcode = NVPTX::ST_i16_ari;
02141         break;
02142       case MVT::i32:
02143         Opcode = NVPTX::ST_i32_ari;
02144         break;
02145       case MVT::i64:
02146         Opcode = NVPTX::ST_i64_ari;
02147         break;
02148       case MVT::f32:
02149         Opcode = NVPTX::ST_f32_ari;
02150         break;
02151       case MVT::f64:
02152         Opcode = NVPTX::ST_f64_ari;
02153         break;
02154       default:
02155         return nullptr;
02156       }
02157     }
02158     SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
02159                       getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
02160                       getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
02161                       Offset, Chain };
02162     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02163   } else {
02164     if (TM.is64Bit()) {
02165       switch (SourceVT) {
02166       case MVT::i8:
02167         Opcode = NVPTX::ST_i8_areg_64;
02168         break;
02169       case MVT::i16:
02170         Opcode = NVPTX::ST_i16_areg_64;
02171         break;
02172       case MVT::i32:
02173         Opcode = NVPTX::ST_i32_areg_64;
02174         break;
02175       case MVT::i64:
02176         Opcode = NVPTX::ST_i64_areg_64;
02177         break;
02178       case MVT::f32:
02179         Opcode = NVPTX::ST_f32_areg_64;
02180         break;
02181       case MVT::f64:
02182         Opcode = NVPTX::ST_f64_areg_64;
02183         break;
02184       default:
02185         return nullptr;
02186       }
02187     } else {
02188       switch (SourceVT) {
02189       case MVT::i8:
02190         Opcode = NVPTX::ST_i8_areg;
02191         break;
02192       case MVT::i16:
02193         Opcode = NVPTX::ST_i16_areg;
02194         break;
02195       case MVT::i32:
02196         Opcode = NVPTX::ST_i32_areg;
02197         break;
02198       case MVT::i64:
02199         Opcode = NVPTX::ST_i64_areg;
02200         break;
02201       case MVT::f32:
02202         Opcode = NVPTX::ST_f32_areg;
02203         break;
02204       case MVT::f64:
02205         Opcode = NVPTX::ST_f64_areg;
02206         break;
02207       default:
02208         return nullptr;
02209       }
02210     }
02211     SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
02212                       getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
02213                       getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), N2,
02214                       Chain };
02215     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02216   }
02217 
02218   if (NVPTXST) {
02219     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02220     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02221     cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
02222   }
02223 
02224   return NVPTXST;
02225 }
02226 
02227 SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
02228   SDValue Chain = N->getOperand(0);
02229   SDValue Op1 = N->getOperand(1);
02230   SDValue Addr, Offset, Base;
02231   unsigned Opcode;
02232   SDLoc DL(N);
02233   SDNode *ST;
02234   EVT EltVT = Op1.getValueType();
02235   MemSDNode *MemSD = cast<MemSDNode>(N);
02236   EVT StoreVT = MemSD->getMemoryVT();
02237 
02238   // Address Space Setting
02239   unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
02240 
02241   if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
02242     report_fatal_error("Cannot store to pointer that points to constant "
02243                        "memory space");
02244   }
02245 
02246   // Volatile Setting
02247   // - .volatile is only availalble for .global and .shared
02248   bool IsVolatile = MemSD->isVolatile();
02249   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
02250       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
02251       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
02252     IsVolatile = false;
02253 
02254   // Type Setting: toType + toTypeWidth
02255   // - for integer type, always use 'u'
02256   assert(StoreVT.isSimple() && "Store value is not simple");
02257   MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
02258   unsigned ToTypeWidth = ScalarVT.getSizeInBits();
02259   unsigned ToType;
02260   if (ScalarVT.isFloatingPoint())
02261     ToType = NVPTX::PTXLdStInstCode::Float;
02262   else
02263     ToType = NVPTX::PTXLdStInstCode::Unsigned;
02264 
02265   SmallVector<SDValue, 12> StOps;
02266   SDValue N2;
02267   unsigned VecType;
02268 
02269   switch (N->getOpcode()) {
02270   case NVPTXISD::StoreV2:
02271     VecType = NVPTX::PTXLdStInstCode::V2;
02272     StOps.push_back(N->getOperand(1));
02273     StOps.push_back(N->getOperand(2));
02274     N2 = N->getOperand(3);
02275     break;
02276   case NVPTXISD::StoreV4:
02277     VecType = NVPTX::PTXLdStInstCode::V4;
02278     StOps.push_back(N->getOperand(1));
02279     StOps.push_back(N->getOperand(2));
02280     StOps.push_back(N->getOperand(3));
02281     StOps.push_back(N->getOperand(4));
02282     N2 = N->getOperand(5);
02283     break;
02284   default:
02285     return nullptr;
02286   }
02287 
02288   StOps.push_back(getI32Imm(IsVolatile, DL));
02289   StOps.push_back(getI32Imm(CodeAddrSpace, DL));
02290   StOps.push_back(getI32Imm(VecType, DL));
02291   StOps.push_back(getI32Imm(ToType, DL));
02292   StOps.push_back(getI32Imm(ToTypeWidth, DL));
02293 
02294   if (SelectDirectAddr(N2, Addr)) {
02295     switch (N->getOpcode()) {
02296     default:
02297       return nullptr;
02298     case NVPTXISD::StoreV2:
02299       switch (EltVT.getSimpleVT().SimpleTy) {
02300       default:
02301         return nullptr;
02302       case MVT::i8:
02303         Opcode = NVPTX::STV_i8_v2_avar;
02304         break;
02305       case MVT::i16:
02306         Opcode = NVPTX::STV_i16_v2_avar;
02307         break;
02308       case MVT::i32:
02309         Opcode = NVPTX::STV_i32_v2_avar;
02310         break;
02311       case MVT::i64:
02312         Opcode = NVPTX::STV_i64_v2_avar;
02313         break;
02314       case MVT::f32:
02315         Opcode = NVPTX::STV_f32_v2_avar;
02316         break;
02317       case MVT::f64:
02318         Opcode = NVPTX::STV_f64_v2_avar;
02319         break;
02320       }
02321       break;
02322     case NVPTXISD::StoreV4:
02323       switch (EltVT.getSimpleVT().SimpleTy) {
02324       default:
02325         return nullptr;
02326       case MVT::i8:
02327         Opcode = NVPTX::STV_i8_v4_avar;
02328         break;
02329       case MVT::i16:
02330         Opcode = NVPTX::STV_i16_v4_avar;
02331         break;
02332       case MVT::i32:
02333         Opcode = NVPTX::STV_i32_v4_avar;
02334         break;
02335       case MVT::f32:
02336         Opcode = NVPTX::STV_f32_v4_avar;
02337         break;
02338       }
02339       break;
02340     }
02341     StOps.push_back(Addr);
02342   } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
02343                           : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
02344     switch (N->getOpcode()) {
02345     default:
02346       return nullptr;
02347     case NVPTXISD::StoreV2:
02348       switch (EltVT.getSimpleVT().SimpleTy) {
02349       default:
02350         return nullptr;
02351       case MVT::i8:
02352         Opcode = NVPTX::STV_i8_v2_asi;
02353         break;
02354       case MVT::i16:
02355         Opcode = NVPTX::STV_i16_v2_asi;
02356         break;
02357       case MVT::i32:
02358         Opcode = NVPTX::STV_i32_v2_asi;
02359         break;
02360       case MVT::i64:
02361         Opcode = NVPTX::STV_i64_v2_asi;
02362         break;
02363       case MVT::f32:
02364         Opcode = NVPTX::STV_f32_v2_asi;
02365         break;
02366       case MVT::f64:
02367         Opcode = NVPTX::STV_f64_v2_asi;
02368         break;
02369       }
02370       break;
02371     case NVPTXISD::StoreV4:
02372       switch (EltVT.getSimpleVT().SimpleTy) {
02373       default:
02374         return nullptr;
02375       case MVT::i8:
02376         Opcode = NVPTX::STV_i8_v4_asi;
02377         break;
02378       case MVT::i16:
02379         Opcode = NVPTX::STV_i16_v4_asi;
02380         break;
02381       case MVT::i32:
02382         Opcode = NVPTX::STV_i32_v4_asi;
02383         break;
02384       case MVT::f32:
02385         Opcode = NVPTX::STV_f32_v4_asi;
02386         break;
02387       }
02388       break;
02389     }
02390     StOps.push_back(Base);
02391     StOps.push_back(Offset);
02392   } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
02393                           : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
02394     if (TM.is64Bit()) {
02395       switch (N->getOpcode()) {
02396       default:
02397         return nullptr;
02398       case NVPTXISD::StoreV2:
02399         switch (EltVT.getSimpleVT().SimpleTy) {
02400         default:
02401           return nullptr;
02402         case MVT::i8:
02403           Opcode = NVPTX::STV_i8_v2_ari_64;
02404           break;
02405         case MVT::i16:
02406           Opcode = NVPTX::STV_i16_v2_ari_64;
02407           break;
02408         case MVT::i32:
02409           Opcode = NVPTX::STV_i32_v2_ari_64;
02410           break;
02411         case MVT::i64:
02412           Opcode = NVPTX::STV_i64_v2_ari_64;
02413           break;
02414         case MVT::f32:
02415           Opcode = NVPTX::STV_f32_v2_ari_64;
02416           break;
02417         case MVT::f64:
02418           Opcode = NVPTX::STV_f64_v2_ari_64;
02419           break;
02420         }
02421         break;
02422       case NVPTXISD::StoreV4:
02423         switch (EltVT.getSimpleVT().SimpleTy) {
02424         default:
02425           return nullptr;
02426         case MVT::i8:
02427           Opcode = NVPTX::STV_i8_v4_ari_64;
02428           break;
02429         case MVT::i16:
02430           Opcode = NVPTX::STV_i16_v4_ari_64;
02431           break;
02432         case MVT::i32:
02433           Opcode = NVPTX::STV_i32_v4_ari_64;
02434           break;
02435         case MVT::f32:
02436           Opcode = NVPTX::STV_f32_v4_ari_64;
02437           break;
02438         }
02439         break;
02440       }
02441     } else {
02442       switch (N->getOpcode()) {
02443       default:
02444         return nullptr;
02445       case NVPTXISD::StoreV2:
02446         switch (EltVT.getSimpleVT().SimpleTy) {
02447         default:
02448           return nullptr;
02449         case MVT::i8:
02450           Opcode = NVPTX::STV_i8_v2_ari;
02451           break;
02452         case MVT::i16:
02453           Opcode = NVPTX::STV_i16_v2_ari;
02454           break;
02455         case MVT::i32:
02456           Opcode = NVPTX::STV_i32_v2_ari;
02457           break;
02458         case MVT::i64:
02459           Opcode = NVPTX::STV_i64_v2_ari;
02460           break;
02461         case MVT::f32:
02462           Opcode = NVPTX::STV_f32_v2_ari;
02463           break;
02464         case MVT::f64:
02465           Opcode = NVPTX::STV_f64_v2_ari;
02466           break;
02467         }
02468         break;
02469       case NVPTXISD::StoreV4:
02470         switch (EltVT.getSimpleVT().SimpleTy) {
02471         default:
02472           return nullptr;
02473         case MVT::i8:
02474           Opcode = NVPTX::STV_i8_v4_ari;
02475           break;
02476         case MVT::i16:
02477           Opcode = NVPTX::STV_i16_v4_ari;
02478           break;
02479         case MVT::i32:
02480           Opcode = NVPTX::STV_i32_v4_ari;
02481           break;
02482         case MVT::f32:
02483           Opcode = NVPTX::STV_f32_v4_ari;
02484           break;
02485         }
02486         break;
02487       }
02488     }
02489     StOps.push_back(Base);
02490     StOps.push_back(Offset);
02491   } else {
02492     if (TM.is64Bit()) {
02493       switch (N->getOpcode()) {
02494       default:
02495         return nullptr;
02496       case NVPTXISD::StoreV2:
02497         switch (EltVT.getSimpleVT().SimpleTy) {
02498         default:
02499           return nullptr;
02500         case MVT::i8:
02501           Opcode = NVPTX::STV_i8_v2_areg_64;
02502           break;
02503         case MVT::i16:
02504           Opcode = NVPTX::STV_i16_v2_areg_64;
02505           break;
02506         case MVT::i32:
02507           Opcode = NVPTX::STV_i32_v2_areg_64;
02508           break;
02509         case MVT::i64:
02510           Opcode = NVPTX::STV_i64_v2_areg_64;
02511           break;
02512         case MVT::f32:
02513           Opcode = NVPTX::STV_f32_v2_areg_64;
02514           break;
02515         case MVT::f64:
02516           Opcode = NVPTX::STV_f64_v2_areg_64;
02517           break;
02518         }
02519         break;
02520       case NVPTXISD::StoreV4:
02521         switch (EltVT.getSimpleVT().SimpleTy) {
02522         default:
02523           return nullptr;
02524         case MVT::i8:
02525           Opcode = NVPTX::STV_i8_v4_areg_64;
02526           break;
02527         case MVT::i16:
02528           Opcode = NVPTX::STV_i16_v4_areg_64;
02529           break;
02530         case MVT::i32:
02531           Opcode = NVPTX::STV_i32_v4_areg_64;
02532           break;
02533         case MVT::f32:
02534           Opcode = NVPTX::STV_f32_v4_areg_64;
02535           break;
02536         }
02537         break;
02538       }
02539     } else {
02540       switch (N->getOpcode()) {
02541       default:
02542         return nullptr;
02543       case NVPTXISD::StoreV2:
02544         switch (EltVT.getSimpleVT().SimpleTy) {
02545         default:
02546           return nullptr;
02547         case MVT::i8:
02548           Opcode = NVPTX::STV_i8_v2_areg;
02549           break;
02550         case MVT::i16:
02551           Opcode = NVPTX::STV_i16_v2_areg;
02552           break;
02553         case MVT::i32:
02554           Opcode = NVPTX::STV_i32_v2_areg;
02555           break;
02556         case MVT::i64:
02557           Opcode = NVPTX::STV_i64_v2_areg;
02558           break;
02559         case MVT::f32:
02560           Opcode = NVPTX::STV_f32_v2_areg;
02561           break;
02562         case MVT::f64:
02563           Opcode = NVPTX::STV_f64_v2_areg;
02564           break;
02565         }
02566         break;
02567       case NVPTXISD::StoreV4:
02568         switch (EltVT.getSimpleVT().SimpleTy) {
02569         default:
02570           return nullptr;
02571         case MVT::i8:
02572           Opcode = NVPTX::STV_i8_v4_areg;
02573           break;
02574         case MVT::i16:
02575           Opcode = NVPTX::STV_i16_v4_areg;
02576           break;
02577         case MVT::i32:
02578           Opcode = NVPTX::STV_i32_v4_areg;
02579           break;
02580         case MVT::f32:
02581           Opcode = NVPTX::STV_f32_v4_areg;
02582           break;
02583         }
02584         break;
02585       }
02586     }
02587     StOps.push_back(N2);
02588   }
02589 
02590   StOps.push_back(Chain);
02591 
02592   ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
02593 
02594   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02595   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02596   cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
02597 
02598   return ST;
02599 }
02600 
02601 SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
02602   SDValue Chain = Node->getOperand(0);
02603   SDValue Offset = Node->getOperand(2);
02604   SDValue Flag = Node->getOperand(3);
02605   SDLoc DL(Node);
02606   MemSDNode *Mem = cast<MemSDNode>(Node);
02607 
02608   unsigned VecSize;
02609   switch (Node->getOpcode()) {
02610   default:
02611     return nullptr;
02612   case NVPTXISD::LoadParam:
02613     VecSize = 1;
02614     break;
02615   case NVPTXISD::LoadParamV2:
02616     VecSize = 2;
02617     break;
02618   case NVPTXISD::LoadParamV4:
02619     VecSize = 4;
02620     break;
02621   }
02622 
02623   EVT EltVT = Node->getValueType(0);
02624   EVT MemVT = Mem->getMemoryVT();
02625 
02626   unsigned Opc = 0;
02627 
02628   switch (VecSize) {
02629   default:
02630     return nullptr;
02631   case 1:
02632     switch (MemVT.getSimpleVT().SimpleTy) {
02633     default:
02634       return nullptr;
02635     case MVT::i1:
02636       Opc = NVPTX::LoadParamMemI8;
02637       break;
02638     case MVT::i8:
02639       Opc = NVPTX::LoadParamMemI8;
02640       break;
02641     case MVT::i16:
02642       Opc = NVPTX::LoadParamMemI16;
02643       break;
02644     case MVT::i32:
02645       Opc = NVPTX::LoadParamMemI32;
02646       break;
02647     case MVT::i64:
02648       Opc = NVPTX::LoadParamMemI64;
02649       break;
02650     case MVT::f32:
02651       Opc = NVPTX::LoadParamMemF32;
02652       break;
02653     case MVT::f64:
02654       Opc = NVPTX::LoadParamMemF64;
02655       break;
02656     }
02657     break;
02658   case 2:
02659     switch (MemVT.getSimpleVT().SimpleTy) {
02660     default:
02661       return nullptr;
02662     case MVT::i1:
02663       Opc = NVPTX::LoadParamMemV2I8;
02664       break;
02665     case MVT::i8:
02666       Opc = NVPTX::LoadParamMemV2I8;
02667       break;
02668     case MVT::i16:
02669       Opc = NVPTX::LoadParamMemV2I16;
02670       break;
02671     case MVT::i32:
02672       Opc = NVPTX::LoadParamMemV2I32;
02673       break;
02674     case MVT::i64:
02675       Opc = NVPTX::LoadParamMemV2I64;
02676       break;
02677     case MVT::f32:
02678       Opc = NVPTX::LoadParamMemV2F32;
02679       break;
02680     case MVT::f64:
02681       Opc = NVPTX::LoadParamMemV2F64;
02682       break;
02683     }
02684     break;
02685   case 4:
02686     switch (MemVT.getSimpleVT().SimpleTy) {
02687     default:
02688       return nullptr;
02689     case MVT::i1:
02690       Opc = NVPTX::LoadParamMemV4I8;
02691       break;
02692     case MVT::i8:
02693       Opc = NVPTX::LoadParamMemV4I8;
02694       break;
02695     case MVT::i16:
02696       Opc = NVPTX::LoadParamMemV4I16;
02697       break;
02698     case MVT::i32:
02699       Opc = NVPTX::LoadParamMemV4I32;
02700       break;
02701     case MVT::f32:
02702       Opc = NVPTX::LoadParamMemV4F32;
02703       break;
02704     }
02705     break;
02706   }
02707 
02708   SDVTList VTs;
02709   if (VecSize == 1) {
02710     VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
02711   } else if (VecSize == 2) {
02712     VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
02713   } else {
02714     EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
02715     VTs = CurDAG->getVTList(EVTs);
02716   }
02717 
02718   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02719 
02720   SmallVector<SDValue, 2> Ops;
02721   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
02722   Ops.push_back(Chain);
02723   Ops.push_back(Flag);
02724 
02725   return CurDAG->getMachineNode(Opc, DL, VTs, Ops);
02726 }
02727 
02728 SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
02729   SDLoc DL(N);
02730   SDValue Chain = N->getOperand(0);
02731   SDValue Offset = N->getOperand(1);
02732   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02733   MemSDNode *Mem = cast<MemSDNode>(N);
02734 
02735   // How many elements do we have?
02736   unsigned NumElts = 1;
02737   switch (N->getOpcode()) {
02738   default:
02739     return nullptr;
02740   case NVPTXISD::StoreRetval:
02741     NumElts = 1;
02742     break;
02743   case NVPTXISD::StoreRetvalV2:
02744     NumElts = 2;
02745     break;
02746   case NVPTXISD::StoreRetvalV4:
02747     NumElts = 4;
02748     break;
02749   }
02750 
02751   // Build vector of operands
02752   SmallVector<SDValue, 6> Ops;
02753   for (unsigned i = 0; i < NumElts; ++i)
02754     Ops.push_back(N->getOperand(i + 2));
02755   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
02756   Ops.push_back(Chain);
02757 
02758   // Determine target opcode
02759   // If we have an i1, use an 8-bit store. The lowering code in
02760   // NVPTXISelLowering will have already emitted an upcast.
02761   unsigned Opcode = 0;
02762   switch (NumElts) {
02763   default:
02764     return nullptr;
02765   case 1:
02766     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02767     default:
02768       return nullptr;
02769     case MVT::i1:
02770       Opcode = NVPTX::StoreRetvalI8;
02771       break;
02772     case MVT::i8:
02773       Opcode = NVPTX::StoreRetvalI8;
02774       break;
02775     case MVT::i16:
02776       Opcode = NVPTX::StoreRetvalI16;
02777       break;
02778     case MVT::i32:
02779       Opcode = NVPTX::StoreRetvalI32;
02780       break;
02781     case MVT::i64:
02782       Opcode = NVPTX::StoreRetvalI64;
02783       break;
02784     case MVT::f32:
02785       Opcode = NVPTX::StoreRetvalF32;
02786       break;
02787     case MVT::f64:
02788       Opcode = NVPTX::StoreRetvalF64;
02789       break;
02790     }
02791     break;
02792   case 2:
02793     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02794     default:
02795       return nullptr;
02796     case MVT::i1:
02797       Opcode = NVPTX::StoreRetvalV2I8;
02798       break;
02799     case MVT::i8:
02800       Opcode = NVPTX::StoreRetvalV2I8;
02801       break;
02802     case MVT::i16:
02803       Opcode = NVPTX::StoreRetvalV2I16;
02804       break;
02805     case MVT::i32:
02806       Opcode = NVPTX::StoreRetvalV2I32;
02807       break;
02808     case MVT::i64:
02809       Opcode = NVPTX::StoreRetvalV2I64;
02810       break;
02811     case MVT::f32:
02812       Opcode = NVPTX::StoreRetvalV2F32;
02813       break;
02814     case MVT::f64:
02815       Opcode = NVPTX::StoreRetvalV2F64;
02816       break;
02817     }
02818     break;
02819   case 4:
02820     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02821     default:
02822       return nullptr;
02823     case MVT::i1:
02824       Opcode = NVPTX::StoreRetvalV4I8;
02825       break;
02826     case MVT::i8:
02827       Opcode = NVPTX::StoreRetvalV4I8;
02828       break;
02829     case MVT::i16:
02830       Opcode = NVPTX::StoreRetvalV4I16;
02831       break;
02832     case MVT::i32:
02833       Opcode = NVPTX::StoreRetvalV4I32;
02834       break;
02835     case MVT::f32:
02836       Opcode = NVPTX::StoreRetvalV4F32;
02837       break;
02838     }
02839     break;
02840   }
02841 
02842   SDNode *Ret =
02843       CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
02844   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02845   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02846   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
02847 
02848   return Ret;
02849 }
02850 
02851 SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
02852   SDLoc DL(N);
02853   SDValue Chain = N->getOperand(0);
02854   SDValue Param = N->getOperand(1);
02855   unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
02856   SDValue Offset = N->getOperand(2);
02857   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02858   MemSDNode *Mem = cast<MemSDNode>(N);
02859   SDValue Flag = N->getOperand(N->getNumOperands() - 1);
02860 
02861   // How many elements do we have?
02862   unsigned NumElts = 1;
02863   switch (N->getOpcode()) {
02864   default:
02865     return nullptr;
02866   case NVPTXISD::StoreParamU32:
02867   case NVPTXISD::StoreParamS32:
02868   case NVPTXISD::StoreParam:
02869     NumElts = 1;
02870     break;
02871   case NVPTXISD::StoreParamV2:
02872     NumElts = 2;
02873     break;
02874   case NVPTXISD::StoreParamV4:
02875     NumElts = 4;
02876     break;
02877   }
02878 
02879   // Build vector of operands
02880   SmallVector<SDValue, 8> Ops;
02881   for (unsigned i = 0; i < NumElts; ++i)
02882     Ops.push_back(N->getOperand(i + 3));
02883   Ops.push_back(CurDAG->getTargetConstant(ParamVal, DL, MVT::i32));
02884   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
02885   Ops.push_back(Chain);
02886   Ops.push_back(Flag);
02887 
02888   // Determine target opcode
02889   // If we have an i1, use an 8-bit store. The lowering code in
02890   // NVPTXISelLowering will have already emitted an upcast.
02891   unsigned Opcode = 0;
02892   switch (N->getOpcode()) {
02893   default:
02894     switch (NumElts) {
02895     default:
02896       return nullptr;
02897     case 1:
02898       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02899       default:
02900         return nullptr;
02901       case MVT::i1:
02902         Opcode = NVPTX::StoreParamI8;
02903         break;
02904       case MVT::i8:
02905         Opcode = NVPTX::StoreParamI8;
02906         break;
02907       case MVT::i16:
02908         Opcode = NVPTX::StoreParamI16;
02909         break;
02910       case MVT::i32:
02911         Opcode = NVPTX::StoreParamI32;
02912         break;
02913       case MVT::i64:
02914         Opcode = NVPTX::StoreParamI64;
02915         break;
02916       case MVT::f32:
02917         Opcode = NVPTX::StoreParamF32;
02918         break;
02919       case MVT::f64:
02920         Opcode = NVPTX::StoreParamF64;
02921         break;
02922       }
02923       break;
02924     case 2:
02925       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02926       default:
02927         return nullptr;
02928       case MVT::i1:
02929         Opcode = NVPTX::StoreParamV2I8;
02930         break;
02931       case MVT::i8:
02932         Opcode = NVPTX::StoreParamV2I8;
02933         break;
02934       case MVT::i16:
02935         Opcode = NVPTX::StoreParamV2I16;
02936         break;
02937       case MVT::i32:
02938         Opcode = NVPTX::StoreParamV2I32;
02939         break;
02940       case MVT::i64:
02941         Opcode = NVPTX::StoreParamV2I64;
02942         break;
02943       case MVT::f32:
02944         Opcode = NVPTX::StoreParamV2F32;
02945         break;
02946       case MVT::f64:
02947         Opcode = NVPTX::StoreParamV2F64;
02948         break;
02949       }
02950       break;
02951     case 4:
02952       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02953       default:
02954         return nullptr;
02955       case MVT::i1:
02956         Opcode = NVPTX::StoreParamV4I8;
02957         break;
02958       case MVT::i8:
02959         Opcode = NVPTX::StoreParamV4I8;
02960         break;
02961       case MVT::i16:
02962         Opcode = NVPTX::StoreParamV4I16;
02963         break;
02964       case MVT::i32:
02965         Opcode = NVPTX::StoreParamV4I32;
02966         break;
02967       case MVT::f32:
02968         Opcode = NVPTX::StoreParamV4F32;
02969         break;
02970       }
02971       break;
02972     }
02973     break;
02974   // Special case: if we have a sign-extend/zero-extend node, insert the
02975   // conversion instruction first, and use that as the value operand to
02976   // the selected StoreParam node.
02977   case NVPTXISD::StoreParamU32: {
02978     Opcode = NVPTX::StoreParamI32;
02979     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
02980                                                 MVT::i32);
02981     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
02982                                          MVT::i32, Ops[0], CvtNone);
02983     Ops[0] = SDValue(Cvt, 0);
02984     break;
02985   }
02986   case NVPTXISD::StoreParamS32: {
02987     Opcode = NVPTX::StoreParamI32;
02988     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
02989                                                 MVT::i32);
02990     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
02991                                          MVT::i32, Ops[0], CvtNone);
02992     Ops[0] = SDValue(Cvt, 0);
02993     break;
02994   }
02995   }
02996 
02997   SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
02998   SDNode *Ret =
02999       CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
03000   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
03001   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
03002   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
03003 
03004   return Ret;
03005 }
03006 
03007 SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
03008   SDValue Chain = N->getOperand(0);
03009   SDNode *Ret = nullptr;
03010   unsigned Opc = 0;
03011   SmallVector<SDValue, 8> Ops;
03012 
03013   switch (N->getOpcode()) {
03014   default: return nullptr;
03015   case NVPTXISD::Tex1DFloatS32:
03016     Opc = NVPTX::TEX_1D_F32_S32;
03017     break;
03018   case NVPTXISD::Tex1DFloatFloat:
03019     Opc = NVPTX::TEX_1D_F32_F32;
03020     break;
03021   case NVPTXISD::Tex1DFloatFloatLevel:
03022     Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
03023     break;
03024   case NVPTXISD::Tex1DFloatFloatGrad:
03025     Opc = NVPTX::TEX_1D_F32_F32_GRAD;
03026     break;
03027   case NVPTXISD::Tex1DS32S32:
03028     Opc = NVPTX::TEX_1D_S32_S32;
03029     break;
03030   case NVPTXISD::Tex1DS32Float:
03031     Opc = NVPTX::TEX_1D_S32_F32;
03032     break;
03033   case NVPTXISD::Tex1DS32FloatLevel:
03034     Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
03035     break;
03036   case NVPTXISD::Tex1DS32FloatGrad:
03037     Opc = NVPTX::TEX_1D_S32_F32_GRAD;
03038     break;
03039   case NVPTXISD::Tex1DU32S32:
03040     Opc = NVPTX::TEX_1D_U32_S32;
03041     break;
03042   case NVPTXISD::Tex1DU32Float:
03043     Opc = NVPTX::TEX_1D_U32_F32;
03044     break;
03045   case NVPTXISD::Tex1DU32FloatLevel:
03046     Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
03047     break;
03048   case NVPTXISD::Tex1DU32FloatGrad:
03049     Opc = NVPTX::TEX_1D_U32_F32_GRAD;
03050     break;
03051   case NVPTXISD::Tex1DArrayFloatS32:
03052     Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
03053     break;
03054   case NVPTXISD::Tex1DArrayFloatFloat:
03055     Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
03056     break;
03057   case NVPTXISD::Tex1DArrayFloatFloatLevel:
03058     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
03059     break;
03060   case NVPTXISD::Tex1DArrayFloatFloatGrad:
03061     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
03062     break;
03063   case NVPTXISD::Tex1DArrayS32S32:
03064     Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
03065     break;
03066   case NVPTXISD::Tex1DArrayS32Float:
03067     Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
03068     break;
03069   case NVPTXISD::Tex1DArrayS32FloatLevel:
03070     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
03071     break;
03072   case NVPTXISD::Tex1DArrayS32FloatGrad:
03073     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
03074     break;
03075   case NVPTXISD::Tex1DArrayU32S32:
03076     Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
03077     break;
03078   case NVPTXISD::Tex1DArrayU32Float:
03079     Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
03080     break;
03081   case NVPTXISD::Tex1DArrayU32FloatLevel:
03082     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
03083     break;
03084   case NVPTXISD::Tex1DArrayU32FloatGrad:
03085     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
03086     break;
03087   case NVPTXISD::Tex2DFloatS32:
03088     Opc = NVPTX::TEX_2D_F32_S32;
03089     break;
03090   case NVPTXISD::Tex2DFloatFloat:
03091     Opc = NVPTX::TEX_2D_F32_F32;
03092     break;
03093   case NVPTXISD::Tex2DFloatFloatLevel:
03094     Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
03095     break;
03096   case NVPTXISD::Tex2DFloatFloatGrad:
03097     Opc = NVPTX::TEX_2D_F32_F32_GRAD;
03098     break;
03099   case NVPTXISD::Tex2DS32S32:
03100     Opc = NVPTX::TEX_2D_S32_S32;
03101     break;
03102   case NVPTXISD::Tex2DS32Float:
03103     Opc = NVPTX::TEX_2D_S32_F32;
03104     break;
03105   case NVPTXISD::Tex2DS32FloatLevel:
03106     Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
03107     break;
03108   case NVPTXISD::Tex2DS32FloatGrad:
03109     Opc = NVPTX::TEX_2D_S32_F32_GRAD;
03110     break;
03111   case NVPTXISD::Tex2DU32S32:
03112     Opc = NVPTX::TEX_2D_U32_S32;
03113     break;
03114   case NVPTXISD::Tex2DU32Float:
03115     Opc = NVPTX::TEX_2D_U32_F32;
03116     break;
03117   case NVPTXISD::Tex2DU32FloatLevel:
03118     Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
03119     break;
03120   case NVPTXISD::Tex2DU32FloatGrad:
03121     Opc = NVPTX::TEX_2D_U32_F32_GRAD;
03122     break;
03123   case NVPTXISD::Tex2DArrayFloatS32:
03124     Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
03125     break;
03126   case NVPTXISD::Tex2DArrayFloatFloat:
03127     Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
03128     break;
03129   case NVPTXISD::Tex2DArrayFloatFloatLevel:
03130     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
03131     break;
03132   case NVPTXISD::Tex2DArrayFloatFloatGrad:
03133     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
03134     break;
03135   case NVPTXISD::Tex2DArrayS32S32:
03136     Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
03137     break;
03138   case NVPTXISD::Tex2DArrayS32Float:
03139     Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
03140     break;
03141   case NVPTXISD::Tex2DArrayS32FloatLevel:
03142     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
03143     break;
03144   case NVPTXISD::Tex2DArrayS32FloatGrad:
03145     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
03146     break;
03147   case NVPTXISD::Tex2DArrayU32S32:
03148     Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
03149     break;
03150   case NVPTXISD::Tex2DArrayU32Float:
03151     Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
03152     break;
03153   case NVPTXISD::Tex2DArrayU32FloatLevel:
03154     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
03155     break;
03156   case NVPTXISD::Tex2DArrayU32FloatGrad:
03157     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
03158     break;
03159   case NVPTXISD::Tex3DFloatS32:
03160     Opc = NVPTX::TEX_3D_F32_S32;
03161     break;
03162   case NVPTXISD::Tex3DFloatFloat:
03163     Opc = NVPTX::TEX_3D_F32_F32;
03164     break;
03165   case NVPTXISD::Tex3DFloatFloatLevel:
03166     Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
03167     break;
03168   case NVPTXISD::Tex3DFloatFloatGrad:
03169     Opc = NVPTX::TEX_3D_F32_F32_GRAD;
03170     break;
03171   case NVPTXISD::Tex3DS32S32:
03172     Opc = NVPTX::TEX_3D_S32_S32;
03173     break;
03174   case NVPTXISD::Tex3DS32Float:
03175     Opc = NVPTX::TEX_3D_S32_F32;
03176     break;
03177   case NVPTXISD::Tex3DS32FloatLevel:
03178     Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
03179     break;
03180   case NVPTXISD::Tex3DS32FloatGrad:
03181     Opc = NVPTX::TEX_3D_S32_F32_GRAD;
03182     break;
03183   case NVPTXISD::Tex3DU32S32:
03184     Opc = NVPTX::TEX_3D_U32_S32;
03185     break;
03186   case NVPTXISD::Tex3DU32Float:
03187     Opc = NVPTX::TEX_3D_U32_F32;
03188     break;
03189   case NVPTXISD::Tex3DU32FloatLevel:
03190     Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
03191     break;
03192   case NVPTXISD::Tex3DU32FloatGrad:
03193     Opc = NVPTX::TEX_3D_U32_F32_GRAD;
03194     break;
03195   case NVPTXISD::TexCubeFloatFloat:
03196     Opc = NVPTX::TEX_CUBE_F32_F32;
03197     break;
03198   case NVPTXISD::TexCubeFloatFloatLevel:
03199     Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
03200     break;
03201   case NVPTXISD::TexCubeS32Float:
03202     Opc = NVPTX::TEX_CUBE_S32_F32;
03203     break;
03204   case NVPTXISD::TexCubeS32FloatLevel:
03205     Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
03206     break;
03207   case NVPTXISD::TexCubeU32Float:
03208     Opc = NVPTX::TEX_CUBE_U32_F32;
03209     break;
03210   case NVPTXISD::TexCubeU32FloatLevel:
03211     Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
03212     break;
03213   case NVPTXISD::TexCubeArrayFloatFloat:
03214     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
03215     break;
03216   case NVPTXISD::TexCubeArrayFloatFloatLevel:
03217     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
03218     break;
03219   case NVPTXISD::TexCubeArrayS32Float:
03220     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
03221     break;
03222   case NVPTXISD::TexCubeArrayS32FloatLevel:
03223     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
03224     break;
03225   case NVPTXISD::TexCubeArrayU32Float:
03226     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
03227     break;
03228   case NVPTXISD::TexCubeArrayU32FloatLevel:
03229     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
03230     break;
03231   case NVPTXISD::Tld4R2DFloatFloat:
03232     Opc = NVPTX::TLD4_R_2D_F32_F32;
03233     break;
03234   case NVPTXISD::Tld4G2DFloatFloat:
03235     Opc = NVPTX::TLD4_G_2D_F32_F32;
03236     break;
03237   case NVPTXISD::Tld4B2DFloatFloat:
03238     Opc = NVPTX::TLD4_B_2D_F32_F32;
03239     break;
03240   case NVPTXISD::Tld4A2DFloatFloat:
03241     Opc = NVPTX::TLD4_A_2D_F32_F32;
03242     break;
03243   case NVPTXISD::Tld4R2DS64Float:
03244     Opc = NVPTX::TLD4_R_2D_S32_F32;
03245     break;
03246   case NVPTXISD::Tld4G2DS64Float:
03247     Opc = NVPTX::TLD4_G_2D_S32_F32;
03248     break;
03249   case NVPTXISD::Tld4B2DS64Float:
03250     Opc = NVPTX::TLD4_B_2D_S32_F32;
03251     break;
03252   case NVPTXISD::Tld4A2DS64Float:
03253     Opc = NVPTX::TLD4_A_2D_S32_F32;
03254     break;
03255   case NVPTXISD::Tld4R2DU64Float:
03256     Opc = NVPTX::TLD4_R_2D_U32_F32;
03257     break;
03258   case NVPTXISD::Tld4G2DU64Float:
03259     Opc = NVPTX::TLD4_G_2D_U32_F32;
03260     break;
03261   case NVPTXISD::Tld4B2DU64Float:
03262     Opc = NVPTX::TLD4_B_2D_U32_F32;
03263     break;
03264   case NVPTXISD::Tld4A2DU64Float:
03265     Opc = NVPTX::TLD4_A_2D_U32_F32;
03266     break;
03267   case NVPTXISD::TexUnified1DFloatS32:
03268     Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
03269     break;
03270   case NVPTXISD::TexUnified1DFloatFloat:
03271     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
03272     break;
03273   case NVPTXISD::TexUnified1DFloatFloatLevel:
03274     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
03275     break;
03276   case NVPTXISD::TexUnified1DFloatFloatGrad:
03277     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
03278     break;
03279   case NVPTXISD::TexUnified1DS32S32:
03280     Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
03281     break;
03282   case NVPTXISD::TexUnified1DS32Float:
03283     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
03284     break;
03285   case NVPTXISD::TexUnified1DS32FloatLevel:
03286     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
03287     break;
03288   case NVPTXISD::TexUnified1DS32FloatGrad:
03289     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
03290     break;
03291   case NVPTXISD::TexUnified1DU32S32:
03292     Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
03293     break;
03294   case NVPTXISD::TexUnified1DU32Float:
03295     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
03296     break;
03297   case NVPTXISD::TexUnified1DU32FloatLevel:
03298     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
03299     break;
03300   case NVPTXISD::TexUnified1DU32FloatGrad:
03301     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
03302     break;
03303   case NVPTXISD::TexUnified1DArrayFloatS32:
03304     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
03305     break;
03306   case NVPTXISD::TexUnified1DArrayFloatFloat:
03307     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
03308     break;
03309   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
03310     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
03311     break;
03312   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
03313     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
03314     break;
03315   case NVPTXISD::TexUnified1DArrayS32S32:
03316     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
03317     break;
03318   case NVPTXISD::TexUnified1DArrayS32Float:
03319     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
03320     break;
03321   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
03322     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
03323     break;
03324   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
03325     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
03326     break;
03327   case NVPTXISD::TexUnified1DArrayU32S32:
03328     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
03329     break;
03330   case NVPTXISD::TexUnified1DArrayU32Float:
03331     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
03332     break;
03333   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
03334     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
03335     break;
03336   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
03337     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
03338     break;
03339   case NVPTXISD::TexUnified2DFloatS32:
03340     Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
03341     break;
03342   case NVPTXISD::TexUnified2DFloatFloat:
03343     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
03344     break;
03345   case NVPTXISD::TexUnified2DFloatFloatLevel:
03346     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
03347     break;
03348   case NVPTXISD::TexUnified2DFloatFloatGrad:
03349     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
03350     break;
03351   case NVPTXISD::TexUnified2DS32S32:
03352     Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
03353     break;
03354   case NVPTXISD::TexUnified2DS32Float:
03355     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
03356     break;
03357   case NVPTXISD::TexUnified2DS32FloatLevel:
03358     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
03359     break;
03360   case NVPTXISD::TexUnified2DS32FloatGrad:
03361     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
03362     break;
03363   case NVPTXISD::TexUnified2DU32S32:
03364     Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
03365     break;
03366   case NVPTXISD::TexUnified2DU32Float:
03367     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
03368     break;
03369   case NVPTXISD::TexUnified2DU32FloatLevel:
03370     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
03371     break;
03372   case NVPTXISD::TexUnified2DU32FloatGrad:
03373     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
03374     break;
03375   case NVPTXISD::TexUnified2DArrayFloatS32:
03376     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
03377     break;
03378   case NVPTXISD::TexUnified2DArrayFloatFloat:
03379     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
03380     break;
03381   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
03382     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
03383     break;
03384   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
03385     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
03386     break;
03387   case NVPTXISD::TexUnified2DArrayS32S32:
03388     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
03389     break;
03390   case NVPTXISD::TexUnified2DArrayS32Float:
03391     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
03392     break;
03393   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
03394     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
03395     break;
03396   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
03397     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
03398     break;
03399   case NVPTXISD::TexUnified2DArrayU32S32:
03400     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
03401     break;
03402   case NVPTXISD::TexUnified2DArrayU32Float:
03403     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
03404     break;
03405   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
03406     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
03407     break;
03408   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
03409     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
03410     break;
03411   case NVPTXISD::TexUnified3DFloatS32:
03412     Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
03413     break;
03414   case NVPTXISD::TexUnified3DFloatFloat:
03415     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
03416     break;
03417   case NVPTXISD::TexUnified3DFloatFloatLevel:
03418     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
03419     break;
03420   case NVPTXISD::TexUnified3DFloatFloatGrad:
03421     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
03422     break;
03423   case NVPTXISD::TexUnified3DS32S32:
03424     Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
03425     break;
03426   case NVPTXISD::TexUnified3DS32Float:
03427     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
03428     break;
03429   case NVPTXISD::TexUnified3DS32FloatLevel:
03430     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
03431     break;
03432   case NVPTXISD::TexUnified3DS32FloatGrad:
03433     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
03434     break;
03435   case NVPTXISD::TexUnified3DU32S32:
03436     Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
03437     break;
03438   case NVPTXISD::TexUnified3DU32Float:
03439     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
03440     break;
03441   case NVPTXISD::TexUnified3DU32FloatLevel:
03442     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
03443     break;
03444   case NVPTXISD::TexUnified3DU32FloatGrad:
03445     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
03446     break;
03447   case NVPTXISD::TexUnifiedCubeFloatFloat:
03448     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
03449     break;
03450   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
03451     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
03452     break;
03453   case NVPTXISD::TexUnifiedCubeS32Float:
03454     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
03455     break;
03456   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
03457     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
03458     break;
03459   case NVPTXISD::TexUnifiedCubeU32Float:
03460     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
03461     break;
03462   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
03463     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
03464     break;
03465   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
03466     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
03467     break;
03468   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
03469     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
03470     break;
03471   case NVPTXISD::TexUnifiedCubeArrayS32Float:
03472     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
03473     break;
03474   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
03475     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
03476     break;
03477   case NVPTXISD::TexUnifiedCubeArrayU32Float:
03478     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
03479     break;
03480   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
03481     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
03482     break;
03483   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
03484     Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
03485     break;
03486   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
03487     Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
03488     break;
03489   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
03490     Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
03491     break;
03492   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
03493     Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
03494     break;
03495   case NVPTXISD::Tld4UnifiedR2DS64Float:
03496     Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
03497     break;
03498   case NVPTXISD::Tld4UnifiedG2DS64Float:
03499     Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
03500     break;
03501   case NVPTXISD::Tld4UnifiedB2DS64Float:
03502     Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
03503     break;
03504   case NVPTXISD::Tld4UnifiedA2DS64Float:
03505     Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
03506     break;
03507   case NVPTXISD::Tld4UnifiedR2DU64Float:
03508     Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
03509     break;
03510   case NVPTXISD::Tld4UnifiedG2DU64Float:
03511     Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
03512     break;
03513   case NVPTXISD::Tld4UnifiedB2DU64Float:
03514     Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
03515     break;
03516   case NVPTXISD::Tld4UnifiedA2DU64Float:
03517     Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
03518     break;
03519   }
03520 
03521   // Copy over operands
03522   for (unsigned i = 1; i < N->getNumOperands(); ++i) {
03523     Ops.push_back(N->getOperand(i));
03524   }
03525 
03526   Ops.push_back(Chain);
03527   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
03528   return Ret;
03529 }
03530 
03531 SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
03532   SDValue Chain = N->getOperand(0);
03533   SDValue TexHandle = N->getOperand(1);
03534   SDNode *Ret = nullptr;
03535   unsigned Opc = 0;
03536   SmallVector<SDValue, 8> Ops;
03537   switch (N->getOpcode()) {
03538   default: return nullptr;
03539   case NVPTXISD::Suld1DI8Clamp:
03540     Opc = NVPTX::SULD_1D_I8_CLAMP;
03541     Ops.push_back(TexHandle);
03542     Ops.push_back(N->getOperand(2));
03543     Ops.push_back(Chain);
03544     break;
03545   case NVPTXISD::Suld1DI16Clamp:
03546     Opc = NVPTX::SULD_1D_I16_CLAMP;
03547     Ops.push_back(TexHandle);
03548     Ops.push_back(N->getOperand(2));
03549     Ops.push_back(Chain);
03550     break;
03551   case NVPTXISD::Suld1DI32Clamp:
03552     Opc = NVPTX::SULD_1D_I32_CLAMP;
03553     Ops.push_back(TexHandle);
03554     Ops.push_back(N->getOperand(2));
03555     Ops.push_back(Chain);
03556     break;
03557   case NVPTXISD::Suld1DI64Clamp:
03558     Opc = NVPTX::SULD_1D_I64_CLAMP;
03559     Ops.push_back(TexHandle);
03560     Ops.push_back(N->getOperand(2));
03561     Ops.push_back(Chain);
03562     break;
03563   case NVPTXISD::Suld1DV2I8Clamp:
03564     Opc = NVPTX::SULD_1D_V2I8_CLAMP;
03565     Ops.push_back(TexHandle);
03566     Ops.push_back(N->getOperand(2));
03567     Ops.push_back(Chain);
03568     break;
03569   case NVPTXISD::Suld1DV2I16Clamp:
03570     Opc = NVPTX::SULD_1D_V2I16_CLAMP;
03571     Ops.push_back(TexHandle);
03572     Ops.push_back(N->getOperand(2));
03573     Ops.push_back(Chain);
03574     break;
03575   case NVPTXISD::Suld1DV2I32Clamp:
03576     Opc = NVPTX::SULD_1D_V2I32_CLAMP;
03577     Ops.push_back(TexHandle);
03578     Ops.push_back(N->getOperand(2));
03579     Ops.push_back(Chain);
03580     break;
03581   case NVPTXISD::Suld1DV2I64Clamp:
03582     Opc = NVPTX::SULD_1D_V2I64_CLAMP;
03583     Ops.push_back(TexHandle);
03584     Ops.push_back(N->getOperand(2));
03585     Ops.push_back(Chain);
03586     break;
03587   case NVPTXISD::Suld1DV4I8Clamp:
03588     Opc = NVPTX::SULD_1D_V4I8_CLAMP;
03589     Ops.push_back(TexHandle);
03590     Ops.push_back(N->getOperand(2));
03591     Ops.push_back(Chain);
03592     break;
03593   case NVPTXISD::Suld1DV4I16Clamp:
03594     Opc = NVPTX::SULD_1D_V4I16_CLAMP;
03595     Ops.push_back(TexHandle);
03596     Ops.push_back(N->getOperand(2));
03597     Ops.push_back(Chain);
03598     break;
03599   case NVPTXISD::Suld1DV4I32Clamp:
03600     Opc = NVPTX::SULD_1D_V4I32_CLAMP;
03601     Ops.push_back(TexHandle);
03602     Ops.push_back(N->getOperand(2));
03603     Ops.push_back(Chain);
03604     break;
03605   case NVPTXISD::Suld1DArrayI8Clamp:
03606     Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
03607     Ops.push_back(TexHandle);
03608     Ops.push_back(N->getOperand(2));
03609     Ops.push_back(N->getOperand(3));
03610     Ops.push_back(Chain);
03611     break;
03612   case NVPTXISD::Suld1DArrayI16Clamp:
03613     Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
03614     Ops.push_back(TexHandle);
03615     Ops.push_back(N->getOperand(2));
03616     Ops.push_back(N->getOperand(3));
03617     Ops.push_back(Chain);
03618     break;
03619   case NVPTXISD::Suld1DArrayI32Clamp:
03620     Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
03621     Ops.push_back(TexHandle);
03622     Ops.push_back(N->getOperand(2));
03623     Ops.push_back(N->getOperand(3));
03624     Ops.push_back(Chain);
03625     break;
03626   case NVPTXISD::Suld1DArrayI64Clamp:
03627     Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
03628     Ops.push_back(TexHandle);
03629     Ops.push_back(N->getOperand(2));
03630     Ops.push_back(N->getOperand(3));
03631     Ops.push_back(Chain);
03632     break;
03633   case NVPTXISD::Suld1DArrayV2I8Clamp:
03634     Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
03635     Ops.push_back(TexHandle);
03636     Ops.push_back(N->getOperand(2));
03637     Ops.push_back(N->getOperand(3));
03638     Ops.push_back(Chain);
03639     break;
03640   case NVPTXISD::Suld1DArrayV2I16Clamp:
03641     Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
03642     Ops.push_back(TexHandle);
03643     Ops.push_back(N->getOperand(2));
03644     Ops.push_back(N->getOperand(3));
03645     Ops.push_back(Chain);
03646     break;
03647   case NVPTXISD::Suld1DArrayV2I32Clamp:
03648     Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
03649     Ops.push_back(TexHandle);
03650     Ops.push_back(N->getOperand(2));
03651     Ops.push_back(N->getOperand(3));
03652     Ops.push_back(Chain);
03653     break;
03654   case NVPTXISD::Suld1DArrayV2I64Clamp:
03655     Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
03656     Ops.push_back(TexHandle);
03657     Ops.push_back(N->getOperand(2));
03658     Ops.push_back(N->getOperand(3));
03659     Ops.push_back(Chain);
03660     break;
03661   case NVPTXISD::Suld1DArrayV4I8Clamp:
03662     Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
03663     Ops.push_back(TexHandle);
03664     Ops.push_back(N->getOperand(2));
03665     Ops.push_back(N->getOperand(3));
03666     Ops.push_back(Chain);
03667     break;
03668   case NVPTXISD::Suld1DArrayV4I16Clamp:
03669     Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
03670     Ops.push_back(TexHandle);
03671     Ops.push_back(N->getOperand(2));
03672     Ops.push_back(N->getOperand(3));
03673     Ops.push_back(Chain);
03674     break;
03675   case NVPTXISD::Suld1DArrayV4I32Clamp:
03676     Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
03677     Ops.push_back(TexHandle);
03678     Ops.push_back(N->getOperand(2));
03679     Ops.push_back(N->getOperand(3));
03680     Ops.push_back(Chain);
03681     break;
03682   case NVPTXISD::Suld2DI8Clamp:
03683     Opc = NVPTX::SULD_2D_I8_CLAMP;
03684     Ops.push_back(TexHandle);
03685     Ops.push_back(N->getOperand(2));
03686     Ops.push_back(N->getOperand(3));
03687     Ops.push_back(Chain);
03688     break;
03689   case NVPTXISD::Suld2DI16Clamp:
03690     Opc = NVPTX::SULD_2D_I16_CLAMP;
03691     Ops.push_back(TexHandle);
03692     Ops.push_back(N->getOperand(2));
03693     Ops.push_back(N->getOperand(3));
03694     Ops.push_back(Chain);
03695     break;
03696   case NVPTXISD::Suld2DI32Clamp:
03697     Opc = NVPTX::SULD_2D_I32_CLAMP;
03698     Ops.push_back(TexHandle);
03699     Ops.push_back(N->getOperand(2));
03700     Ops.push_back(N->getOperand(3));
03701     Ops.push_back(Chain);
03702     break;
03703   case NVPTXISD::Suld2DI64Clamp:
03704     Opc = NVPTX::SULD_2D_I64_CLAMP;
03705     Ops.push_back(TexHandle);
03706     Ops.push_back(N->getOperand(2));
03707     Ops.push_back(N->getOperand(3));
03708     Ops.push_back(Chain);
03709     break;
03710   case NVPTXISD::Suld2DV2I8Clamp:
03711     Opc = NVPTX::SULD_2D_V2I8_CLAMP;
03712     Ops.push_back(TexHandle);
03713     Ops.push_back(N->getOperand(2));
03714     Ops.push_back(N->getOperand(3));
03715     Ops.push_back(Chain);
03716     break;
03717   case NVPTXISD::Suld2DV2I16Clamp:
03718     Opc = NVPTX::SULD_2D_V2I16_CLAMP;
03719     Ops.push_back(TexHandle);
03720     Ops.push_back(N->getOperand(2));
03721     Ops.push_back(N->getOperand(3));
03722     Ops.push_back(Chain);
03723     break;
03724   case NVPTXISD::Suld2DV2I32Clamp:
03725     Opc = NVPTX::SULD_2D_V2I32_CLAMP;
03726     Ops.push_back(TexHandle);
03727     Ops.push_back(N->getOperand(2));
03728     Ops.push_back(N->getOperand(3));
03729     Ops.push_back(Chain);
03730     break;
03731   case NVPTXISD::Suld2DV2I64Clamp:
03732     Opc = NVPTX::SULD_2D_V2I64_CLAMP;
03733     Ops.push_back(TexHandle);
03734     Ops.push_back(N->getOperand(2));
03735     Ops.push_back(N->getOperand(3));
03736     Ops.push_back(Chain);
03737     break;
03738   case NVPTXISD::Suld2DV4I8Clamp:
03739     Opc = NVPTX::SULD_2D_V4I8_CLAMP;
03740     Ops.push_back(TexHandle);
03741     Ops.push_back(N->getOperand(2));
03742     Ops.push_back(N->getOperand(3));
03743     Ops.push_back(Chain);
03744     break;
03745   case NVPTXISD::Suld2DV4I16Clamp:
03746     Opc = NVPTX::SULD_2D_V4I16_CLAMP;
03747     Ops.push_back(TexHandle);
03748     Ops.push_back(N->getOperand(2));
03749     Ops.push_back(N->getOperand(3));
03750     Ops.push_back(Chain);
03751     break;
03752   case NVPTXISD::Suld2DV4I32Clamp:
03753     Opc = NVPTX::SULD_2D_V4I32_CLAMP;
03754     Ops.push_back(TexHandle);
03755     Ops.push_back(N->getOperand(2));
03756     Ops.push_back(N->getOperand(3));
03757     Ops.push_back(Chain);
03758     break;
03759   case NVPTXISD::Suld2DArrayI8Clamp:
03760     Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
03761     Ops.push_back(TexHandle);
03762     Ops.push_back(N->getOperand(2));
03763     Ops.push_back(N->getOperand(3));
03764     Ops.push_back(N->getOperand(4));
03765     Ops.push_back(Chain);
03766     break;
03767   case NVPTXISD::Suld2DArrayI16Clamp:
03768     Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
03769     Ops.push_back(TexHandle);
03770     Ops.push_back(N->getOperand(2));
03771     Ops.push_back(N->getOperand(3));
03772     Ops.push_back(N->getOperand(4));
03773     Ops.push_back(Chain);
03774     break;
03775   case NVPTXISD::Suld2DArrayI32Clamp:
03776     Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
03777     Ops.push_back(TexHandle);
03778     Ops.push_back(N->getOperand(2));
03779     Ops.push_back(N->getOperand(3));
03780     Ops.push_back(N->getOperand(4));
03781     Ops.push_back(Chain);
03782     break;
03783   case NVPTXISD::Suld2DArrayI64Clamp:
03784     Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
03785     Ops.push_back(TexHandle);
03786     Ops.push_back(N->getOperand(2));
03787     Ops.push_back(N->getOperand(3));
03788     Ops.push_back(N->getOperand(4));
03789     Ops.push_back(Chain);
03790     break;
03791   case NVPTXISD::Suld2DArrayV2I8Clamp:
03792     Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
03793     Ops.push_back(TexHandle);
03794     Ops.push_back(N->getOperand(2));
03795     Ops.push_back(N->getOperand(3));
03796     Ops.push_back(N->getOperand(4));
03797     Ops.push_back(Chain);
03798     break;
03799   case NVPTXISD::Suld2DArrayV2I16Clamp:
03800     Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
03801     Ops.push_back(TexHandle);
03802     Ops.push_back(N->getOperand(2));
03803     Ops.push_back(N->getOperand(3));
03804     Ops.push_back(N->getOperand(4));
03805     Ops.push_back(Chain);
03806     break;
03807   case NVPTXISD::Suld2DArrayV2I32Clamp:
03808     Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
03809     Ops.push_back(TexHandle);
03810     Ops.push_back(N->getOperand(2));
03811     Ops.push_back(N->getOperand(3));
03812     Ops.push_back(N->getOperand(4));
03813     Ops.push_back(Chain);
03814     break;
03815   case NVPTXISD::Suld2DArrayV2I64Clamp:
03816     Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
03817     Ops.push_back(TexHandle);
03818     Ops.push_back(N->getOperand(2));
03819     Ops.push_back(N->getOperand(3));
03820     Ops.push_back(N->getOperand(4));
03821     Ops.push_back(Chain);
03822     break;
03823   case NVPTXISD::Suld2DArrayV4I8Clamp:
03824     Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
03825     Ops.push_back(TexHandle);
03826     Ops.push_back(N->getOperand(2));
03827     Ops.push_back(N->getOperand(3));
03828     Ops.push_back(N->getOperand(4));
03829     Ops.push_back(Chain);
03830     break;
03831   case NVPTXISD::Suld2DArrayV4I16Clamp:
03832     Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
03833     Ops.push_back(TexHandle);
03834     Ops.push_back(N->getOperand(2));
03835     Ops.push_back(N->getOperand(3));
03836     Ops.push_back(N->getOperand(4));
03837     Ops.push_back(Chain);
03838     break;
03839   case NVPTXISD::Suld2DArrayV4I32Clamp:
03840     Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
03841     Ops.push_back(TexHandle);
03842     Ops.push_back(N->getOperand(2));
03843     Ops.push_back(N->getOperand(3));
03844     Ops.push_back(N->getOperand(4));
03845     Ops.push_back(Chain);
03846     break;
03847   case NVPTXISD::Suld3DI8Clamp:
03848     Opc = NVPTX::SULD_3D_I8_CLAMP;
03849     Ops.push_back(TexHandle);
03850     Ops.push_back(N->getOperand(2));
03851     Ops.push_back(N->getOperand(3));
03852     Ops.push_back(N->getOperand(4));
03853     Ops.push_back(Chain);
03854     break;
03855   case NVPTXISD::Suld3DI16Clamp:
03856     Opc = NVPTX::SULD_3D_I16_CLAMP;
03857     Ops.push_back(TexHandle);
03858     Ops.push_back(N->getOperand(2));
03859     Ops.push_back(N->getOperand(3));
03860     Ops.push_back(N->getOperand(4));
03861     Ops.push_back(Chain);
03862     break;
03863   case NVPTXISD::Suld3DI32Clamp:
03864     Opc = NVPTX::SULD_3D_I32_CLAMP;
03865     Ops.push_back(TexHandle);
03866     Ops.push_back(N->getOperand(2));
03867     Ops.push_back(N->getOperand(3));
03868     Ops.push_back(N->getOperand(4));
03869     Ops.push_back(Chain);
03870     break;
03871   case NVPTXISD::Suld3DI64Clamp:
03872     Opc = NVPTX::SULD_3D_I64_CLAMP;
03873     Ops.push_back(TexHandle);
03874     Ops.push_back(N->getOperand(2));
03875     Ops.push_back(N->getOperand(3));
03876     Ops.push_back(N->getOperand(4));
03877     Ops.push_back(Chain);
03878     break;
03879   case NVPTXISD::Suld3DV2I8Clamp:
03880     Opc = NVPTX::SULD_3D_V2I8_CLAMP;
03881     Ops.push_back(TexHandle);
03882     Ops.push_back(N->getOperand(2));
03883     Ops.push_back(N->getOperand(3));
03884     Ops.push_back(N->getOperand(4));
03885     Ops.push_back(Chain);
03886     break;
03887   case NVPTXISD::Suld3DV2I16Clamp:
03888     Opc = NVPTX::SULD_3D_V2I16_CLAMP;
03889     Ops.push_back(TexHandle);
03890     Ops.push_back(N->getOperand(2));
03891     Ops.push_back(N->getOperand(3));
03892     Ops.push_back(N->getOperand(4));
03893     Ops.push_back(Chain);
03894     break;
03895   case NVPTXISD::Suld3DV2I32Clamp:
03896     Opc = NVPTX::SULD_3D_V2I32_CLAMP;
03897     Ops.push_back(TexHandle);
03898     Ops.push_back(N->getOperand(2));
03899     Ops.push_back(N->getOperand(3));
03900     Ops.push_back(N->getOperand(4));
03901     Ops.push_back(Chain);
03902     break;
03903   case NVPTXISD::Suld3DV2I64Clamp:
03904     Opc = NVPTX::SULD_3D_V2I64_CLAMP;
03905     Ops.push_back(TexHandle);
03906     Ops.push_back(N->getOperand(2));
03907     Ops.push_back(N->getOperand(3));
03908     Ops.push_back(N->getOperand(4));
03909     Ops.push_back(Chain);
03910     break;
03911   case NVPTXISD::Suld3DV4I8Clamp:
03912     Opc = NVPTX::SULD_3D_V4I8_CLAMP;
03913     Ops.push_back(TexHandle);
03914     Ops.push_back(N->getOperand(2));
03915     Ops.push_back(N->getOperand(3));
03916     Ops.push_back(N->getOperand(4));
03917     Ops.push_back(Chain);
03918     break;
03919   case NVPTXISD::Suld3DV4I16Clamp:
03920     Opc = NVPTX::SULD_3D_V4I16_CLAMP;
03921     Ops.push_back(TexHandle);
03922     Ops.push_back(N->getOperand(2));
03923     Ops.push_back(N->getOperand(3));
03924     Ops.push_back(N->getOperand(4));
03925     Ops.push_back(Chain);
03926     break;
03927   case NVPTXISD::Suld3DV4I32Clamp:
03928     Opc = NVPTX::SULD_3D_V4I32_CLAMP;
03929     Ops.push_back(TexHandle);
03930     Ops.push_back(N->getOperand(2));
03931     Ops.push_back(N->getOperand(3));
03932     Ops.push_back(N->getOperand(4));
03933     Ops.push_back(Chain);
03934     break;
03935   case NVPTXISD::Suld1DI8Trap:
03936     Opc = NVPTX::SULD_1D_I8_TRAP;
03937     Ops.push_back(TexHandle);
03938     Ops.push_back(N->getOperand(2));
03939     Ops.push_back(Chain);
03940     break;
03941   case NVPTXISD::Suld1DI16Trap:
03942     Opc = NVPTX::SULD_1D_I16_TRAP;
03943     Ops.push_back(TexHandle);
03944     Ops.push_back(N->getOperand(2));
03945     Ops.push_back(Chain);
03946     break;
03947   case NVPTXISD::Suld1DI32Trap:
03948     Opc = NVPTX::SULD_1D_I32_TRAP;
03949     Ops.push_back(TexHandle);
03950     Ops.push_back(N->getOperand(2));
03951     Ops.push_back(Chain);
03952     break;
03953   case NVPTXISD::Suld1DI64Trap:
03954     Opc = NVPTX::SULD_1D_I64_TRAP;
03955     Ops.push_back(TexHandle);
03956     Ops.push_back(N->getOperand(2));
03957     Ops.push_back(Chain);
03958     break;
03959   case NVPTXISD::Suld1DV2I8Trap:
03960     Opc = NVPTX::SULD_1D_V2I8_TRAP;
03961     Ops.push_back(TexHandle);
03962     Ops.push_back(N->getOperand(2));
03963     Ops.push_back(Chain);
03964     break;
03965   case NVPTXISD::Suld1DV2I16Trap:
03966     Opc = NVPTX::SULD_1D_V2I16_TRAP;
03967     Ops.push_back(TexHandle);
03968     Ops.push_back(N->getOperand(2));
03969     Ops.push_back(Chain);
03970     break;
03971   case NVPTXISD::Suld1DV2I32Trap:
03972     Opc = NVPTX::SULD_1D_V2I32_TRAP;
03973     Ops.push_back(TexHandle);
03974     Ops.push_back(N->getOperand(2));
03975     Ops.push_back(Chain);
03976     break;
03977   case NVPTXISD::Suld1DV2I64Trap:
03978     Opc = NVPTX::SULD_1D_V2I64_TRAP;
03979     Ops.push_back(TexHandle);
03980     Ops.push_back(N->getOperand(2));
03981     Ops.push_back(Chain);
03982     break;
03983   case NVPTXISD::Suld1DV4I8Trap:
03984     Opc = NVPTX::SULD_1D_V4I8_TRAP;
03985     Ops.push_back(TexHandle);
03986     Ops.push_back(N->getOperand(2));
03987     Ops.push_back(Chain);
03988     break;
03989   case NVPTXISD::Suld1DV4I16Trap:
03990     Opc = NVPTX::SULD_1D_V4I16_TRAP;
03991     Ops.push_back(TexHandle);
03992     Ops.push_back(N->getOperand(2));
03993     Ops.push_back(Chain);
03994     break;
03995   case NVPTXISD::Suld1DV4I32Trap:
03996     Opc = NVPTX::SULD_1D_V4I32_TRAP;
03997     Ops.push_back(TexHandle);
03998     Ops.push_back(N->getOperand(2));
03999     Ops.push_back(Chain);
04000     break;
04001   case NVPTXISD::Suld1DArrayI8Trap:
04002     Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
04003     Ops.push_back(TexHandle);
04004     Ops.push_back(N->getOperand(2));
04005     Ops.push_back(N->getOperand(3));
04006     Ops.push_back(Chain);
04007     break;
04008   case NVPTXISD::Suld1DArrayI16Trap:
04009     Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
04010     Ops.push_back(TexHandle);
04011     Ops.push_back(N->getOperand(2));
04012     Ops.push_back(N->getOperand(3));
04013     Ops.push_back(Chain);
04014     break;
04015   case NVPTXISD::Suld1DArrayI32Trap:
04016     Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
04017     Ops.push_back(TexHandle);
04018     Ops.push_back(N->getOperand(2));
04019     Ops.push_back(N->getOperand(3));
04020     Ops.push_back(Chain);
04021     break;
04022   case NVPTXISD::Suld1DArrayI64Trap:
04023     Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
04024     Ops.push_back(TexHandle);
04025     Ops.push_back(N->getOperand(2));
04026     Ops.push_back(N->getOperand(3));
04027     Ops.push_back(Chain);
04028     break;
04029   case NVPTXISD::Suld1DArrayV2I8Trap:
04030     Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
04031     Ops.push_back(TexHandle);
04032     Ops.push_back(N->getOperand(2));
04033     Ops.push_back(N->getOperand(3));
04034     Ops.push_back(Chain);
04035     break;
04036   case NVPTXISD::Suld1DArrayV2I16Trap:
04037     Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
04038     Ops.push_back(TexHandle);
04039     Ops.push_back(N->getOperand(2));
04040     Ops.push_back(N->getOperand(3));
04041     Ops.push_back(Chain);
04042     break;
04043   case NVPTXISD::Suld1DArrayV2I32Trap:
04044     Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
04045     Ops.push_back(TexHandle);
04046     Ops.push_back(N->getOperand(2));
04047     Ops.push_back(N->getOperand(3));
04048     Ops.push_back(Chain);
04049     break;
04050   case NVPTXISD::Suld1DArrayV2I64Trap:
04051     Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
04052     Ops.push_back(TexHandle);
04053     Ops.push_back(N->getOperand(2));
04054     Ops.push_back(N->getOperand(3));
04055     Ops.push_back(Chain);
04056     break;
04057   case NVPTXISD::Suld1DArrayV4I8Trap:
04058     Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
04059     Ops.push_back(TexHandle);
04060     Ops.push_back(N->getOperand(2));
04061     Ops.push_back(N->getOperand(3));
04062     Ops.push_back(Chain);
04063     break;
04064   case NVPTXISD::Suld1DArrayV4I16Trap:
04065     Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
04066     Ops.push_back(TexHandle);
04067     Ops.push_back(N->getOperand(2));
04068     Ops.push_back(N->getOperand(3));
04069     Ops.push_back(Chain);
04070     break;
04071   case NVPTXISD::Suld1DArrayV4I32Trap:
04072     Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
04073     Ops.push_back(TexHandle);
04074     Ops.push_back(N->getOperand(2));
04075     Ops.push_back(N->getOperand(3));
04076     Ops.push_back(Chain);
04077     break;
04078   case NVPTXISD::Suld2DI8Trap:
04079     Opc = NVPTX::SULD_2D_I8_TRAP;
04080     Ops.push_back(TexHandle);
04081     Ops.push_back(N->getOperand(2));
04082     Ops.push_back(N->getOperand(3));
04083     Ops.push_back(Chain);
04084     break;
04085   case NVPTXISD::Suld2DI16Trap:
04086     Opc = NVPTX::SULD_2D_I16_TRAP;
04087     Ops.push_back(TexHandle);
04088     Ops.push_back(N->getOperand(2));
04089     Ops.push_back(N->getOperand(3));
04090     Ops.push_back(Chain);
04091     break;
04092   case NVPTXISD::Suld2DI32Trap:
04093     Opc = NVPTX::SULD_2D_I32_TRAP;
04094     Ops.push_back(TexHandle);
04095     Ops.push_back(N->getOperand(2));
04096     Ops.push_back(N->getOperand(3));
04097     Ops.push_back(Chain);
04098     break;
04099   case NVPTXISD::Suld2DI64Trap:
04100     Opc = NVPTX::SULD_2D_I64_TRAP;
04101     Ops.push_back(TexHandle);
04102     Ops.push_back(N->getOperand(2));
04103     Ops.push_back(N->getOperand(3));
04104     Ops.push_back(Chain);
04105     break;
04106   case NVPTXISD::Suld2DV2I8Trap:
04107     Opc = NVPTX::SULD_2D_V2I8_TRAP;
04108     Ops.push_back(TexHandle);
04109     Ops.push_back(N->getOperand(2));
04110     Ops.push_back(N->getOperand(3));
04111     Ops.push_back(Chain);
04112     break;
04113   case NVPTXISD::Suld2DV2I16Trap:
04114     Opc = NVPTX::SULD_2D_V2I16_TRAP;
04115     Ops.push_back(TexHandle);
04116     Ops.push_back(N->getOperand(2));
04117     Ops.push_back(N->getOperand(3));
04118     Ops.push_back(Chain);
04119     break;
04120   case NVPTXISD::Suld2DV2I32Trap:
04121     Opc = NVPTX::SULD_2D_V2I32_TRAP;
04122     Ops.push_back(TexHandle);
04123     Ops.push_back(N->getOperand(2));
04124     Ops.push_back(N->getOperand(3));
04125     Ops.push_back(Chain);
04126     break;
04127   case NVPTXISD::Suld2DV2I64Trap:
04128     Opc = NVPTX::SULD_2D_V2I64_TRAP;
04129     Ops.push_back(TexHandle);
04130     Ops.push_back(N->getOperand(2));
04131     Ops.push_back(N->getOperand(3));
04132     Ops.push_back(Chain);
04133     break;
04134   case NVPTXISD::Suld2DV4I8Trap:
04135     Opc = NVPTX::SULD_2D_V4I8_TRAP;
04136     Ops.push_back(TexHandle);
04137     Ops.push_back(N->getOperand(2));
04138     Ops.push_back(N->getOperand(3));
04139     Ops.push_back(Chain);
04140     break;
04141   case NVPTXISD::Suld2DV4I16Trap:
04142     Opc = NVPTX::SULD_2D_V4I16_TRAP;
04143     Ops.push_back(TexHandle);
04144     Ops.push_back(N->getOperand(2));
04145     Ops.push_back(N->getOperand(3));
04146     Ops.push_back(Chain);
04147     break;
04148   case NVPTXISD::Suld2DV4I32Trap:
04149     Opc = NVPTX::SULD_2D_V4I32_TRAP;
04150     Ops.push_back(TexHandle);
04151     Ops.push_back(N->getOperand(2));
04152     Ops.push_back(N->getOperand(3));
04153     Ops.push_back(Chain);
04154     break;
04155   case NVPTXISD::Suld2DArrayI8Trap:
04156     Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
04157     Ops.push_back(TexHandle);
04158     Ops.push_back(N->getOperand(2));
04159     Ops.push_back(N->getOperand(3));
04160     Ops.push_back(N->getOperand(4));
04161     Ops.push_back(Chain);
04162     break;
04163   case NVPTXISD::Suld2DArrayI16Trap:
04164     Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
04165     Ops.push_back(TexHandle);
04166     Ops.push_back(N->getOperand(2));
04167     Ops.push_back(N->getOperand(3));
04168     Ops.push_back(N->getOperand(4));
04169     Ops.push_back(Chain);
04170     break;
04171   case NVPTXISD::Suld2DArrayI32Trap:
04172     Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
04173     Ops.push_back(TexHandle);
04174     Ops.push_back(N->getOperand(2));
04175     Ops.push_back(N->getOperand(3));
04176     Ops.push_back(N->getOperand(4));
04177     Ops.push_back(Chain);
04178     break;
04179   case NVPTXISD::Suld2DArrayI64Trap:
04180     Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
04181     Ops.push_back(TexHandle);
04182     Ops.push_back(N->getOperand(2));
04183     Ops.push_back(N->getOperand(3));
04184     Ops.push_back(N->getOperand(4));
04185     Ops.push_back(Chain);
04186     break;
04187   case NVPTXISD::Suld2DArrayV2I8Trap:
04188     Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
04189     Ops.push_back(TexHandle);
04190     Ops.push_back(N->getOperand(2));
04191     Ops.push_back(N->getOperand(3));
04192     Ops.push_back(N->getOperand(4));
04193     Ops.push_back(Chain);
04194     break;
04195   case NVPTXISD::Suld2DArrayV2I16Trap:
04196     Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
04197     Ops.push_back(TexHandle);
04198     Ops.push_back(N->getOperand(2));
04199     Ops.push_back(N->getOperand(3));
04200     Ops.push_back(N->getOperand(4));
04201     Ops.push_back(Chain);
04202     break;
04203   case NVPTXISD::Suld2DArrayV2I32Trap:
04204     Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
04205     Ops.push_back(TexHandle);
04206     Ops.push_back(N->getOperand(2));
04207     Ops.push_back(N->getOperand(3));
04208     Ops.push_back(N->getOperand(4));
04209     Ops.push_back(Chain);
04210     break;
04211   case NVPTXISD::Suld2DArrayV2I64Trap:
04212     Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
04213     Ops.push_back(TexHandle);
04214     Ops.push_back(N->getOperand(2));
04215     Ops.push_back(N->getOperand(3));
04216     Ops.push_back(N->getOperand(4));
04217     Ops.push_back(Chain);
04218     break;
04219   case NVPTXISD::Suld2DArrayV4I8Trap:
04220     Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
04221     Ops.push_back(TexHandle);
04222     Ops.push_back(N->getOperand(2));
04223     Ops.push_back(N->getOperand(3));
04224     Ops.push_back(N->getOperand(4));
04225     Ops.push_back(Chain);
04226     break;
04227   case NVPTXISD::Suld2DArrayV4I16Trap:
04228     Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
04229     Ops.push_back(TexHandle);
04230     Ops.push_back(N->getOperand(2));
04231     Ops.push_back(N->getOperand(3));
04232     Ops.push_back(N->getOperand(4));
04233     Ops.push_back(Chain);
04234     break;
04235   case NVPTXISD::Suld2DArrayV4I32Trap:
04236     Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
04237     Ops.push_back(TexHandle);
04238     Ops.push_back(N->getOperand(2));
04239     Ops.push_back(N->getOperand(3));
04240     Ops.push_back(N->getOperand(4));
04241     Ops.push_back(Chain);
04242     break;
04243   case NVPTXISD::Suld3DI8Trap:
04244     Opc = NVPTX::SULD_3D_I8_TRAP;
04245     Ops.push_back(TexHandle);
04246     Ops.push_back(N->getOperand(2));
04247     Ops.push_back(N->getOperand(3));
04248     Ops.push_back(N->getOperand(4));
04249     Ops.push_back(Chain);
04250     break;
04251   case NVPTXISD::Suld3DI16Trap:
04252     Opc = NVPTX::SULD_3D_I16_TRAP;
04253     Ops.push_back(TexHandle);
04254     Ops.push_back(N->getOperand(2));
04255     Ops.push_back(N->getOperand(3));
04256     Ops.push_back(N->getOperand(4));
04257     Ops.push_back(Chain);
04258     break;
04259   case NVPTXISD::Suld3DI32Trap:
04260     Opc = NVPTX::SULD_3D_I32_TRAP;
04261     Ops.push_back(TexHandle);
04262     Ops.push_back(N->getOperand(2));
04263     Ops.push_back(N->getOperand(3));
04264     Ops.push_back(N->getOperand(4));
04265     Ops.push_back(Chain);
04266     break;
04267   case NVPTXISD::Suld3DI64Trap:
04268     Opc = NVPTX::SULD_3D_I64_TRAP;
04269     Ops.push_back(TexHandle);
04270     Ops.push_back(N->getOperand(2));
04271     Ops.push_back(N->getOperand(3));
04272     Ops.push_back(N->getOperand(4));
04273     Ops.push_back(Chain);
04274     break;
04275   case NVPTXISD::Suld3DV2I8Trap:
04276     Opc = NVPTX::SULD_3D_V2I8_TRAP;
04277     Ops.push_back(TexHandle);
04278     Ops.push_back(N->getOperand(2));
04279     Ops.push_back(N->getOperand(3));
04280     Ops.push_back(N->getOperand(4));
04281     Ops.push_back(Chain);
04282     break;
04283   case NVPTXISD::Suld3DV2I16Trap:
04284     Opc = NVPTX::SULD_3D_V2I16_TRAP;
04285     Ops.push_back(TexHandle);
04286     Ops.push_back(N->getOperand(2));
04287     Ops.push_back(N->getOperand(3));
04288     Ops.push_back(N->getOperand(4));
04289     Ops.push_back(Chain);
04290     break;
04291   case NVPTXISD::Suld3DV2I32Trap:
04292     Opc = NVPTX::SULD_3D_V2I32_TRAP;
04293     Ops.push_back(TexHandle);
04294     Ops.push_back(N->getOperand(2));
04295     Ops.push_back(N->getOperand(3));
04296     Ops.push_back(N->getOperand(4));
04297     Ops.push_back(Chain);
04298     break;
04299   case NVPTXISD::Suld3DV2I64Trap:
04300     Opc = NVPTX::SULD_3D_V2I64_TRAP;
04301     Ops.push_back(TexHandle);
04302     Ops.push_back(N->getOperand(2));
04303     Ops.push_back(N->getOperand(3));
04304     Ops.push_back(N->getOperand(4));
04305     Ops.push_back(Chain);
04306     break;
04307   case NVPTXISD::Suld3DV4I8Trap:
04308     Opc = NVPTX::SULD_3D_V4I8_TRAP;
04309     Ops.push_back(TexHandle);
04310     Ops.push_back(N->getOperand(2));
04311     Ops.push_back(N->getOperand(3));
04312     Ops.push_back(N->getOperand(4));
04313     Ops.push_back(Chain);
04314     break;
04315   case NVPTXISD::Suld3DV4I16Trap:
04316     Opc = NVPTX::SULD_3D_V4I16_TRAP;
04317     Ops.push_back(TexHandle);
04318     Ops.push_back(N->getOperand(2));
04319     Ops.push_back(N->getOperand(3));
04320     Ops.push_back(N->getOperand(4));
04321     Ops.push_back(Chain);
04322     break;
04323   case NVPTXISD::Suld3DV4I32Trap:
04324     Opc = NVPTX::SULD_3D_V4I32_TRAP;
04325     Ops.push_back(TexHandle);
04326     Ops.push_back(N->getOperand(2));
04327     Ops.push_back(N->getOperand(3));
04328     Ops.push_back(N->getOperand(4));
04329     Ops.push_back(Chain);
04330     break;
04331   case NVPTXISD::Suld1DI8Zero:
04332     Opc = NVPTX::SULD_1D_I8_ZERO;
04333     Ops.push_back(TexHandle);
04334     Ops.push_back(N->getOperand(2));
04335     Ops.push_back(Chain);
04336     break;
04337   case NVPTXISD::Suld1DI16Zero:
04338     Opc = NVPTX::SULD_1D_I16_ZERO;
04339     Ops.push_back(TexHandle);
04340     Ops.push_back(N->getOperand(2));
04341     Ops.push_back(Chain);
04342     break;
04343   case NVPTXISD::Suld1DI32Zero:
04344     Opc = NVPTX::SULD_1D_I32_ZERO;
04345     Ops.push_back(TexHandle);
04346     Ops.push_back(N->getOperand(2));
04347     Ops.push_back(Chain);
04348     break;
04349   case NVPTXISD::Suld1DI64Zero:
04350     Opc = NVPTX::SULD_1D_I64_ZERO;
04351     Ops.push_back(TexHandle);
04352     Ops.push_back(N->getOperand(2));
04353     Ops.push_back(Chain);
04354     break;
04355   case NVPTXISD::Suld1DV2I8Zero:
04356     Opc = NVPTX::SULD_1D_V2I8_ZERO;
04357     Ops.push_back(TexHandle);
04358     Ops.push_back(N->getOperand(2));
04359     Ops.push_back(Chain);
04360     break;
04361   case NVPTXISD::Suld1DV2I16Zero:
04362     Opc = NVPTX::SULD_1D_V2I16_ZERO;
04363     Ops.push_back(TexHandle);
04364     Ops.push_back(N->getOperand(2));
04365     Ops.push_back(Chain);
04366     break;
04367   case NVPTXISD::Suld1DV2I32Zero:
04368     Opc = NVPTX::SULD_1D_V2I32_ZERO;
04369     Ops.push_back(TexHandle);
04370     Ops.push_back(N->getOperand(2));
04371     Ops.push_back(Chain);
04372     break;
04373   case NVPTXISD::Suld1DV2I64Zero:
04374     Opc = NVPTX::SULD_1D_V2I64_ZERO;
04375     Ops.push_back(TexHandle);
04376     Ops.push_back(N->getOperand(2));
04377     Ops.push_back(Chain);
04378     break;
04379   case NVPTXISD::Suld1DV4I8Zero:
04380     Opc = NVPTX::SULD_1D_V4I8_ZERO;
04381     Ops.push_back(TexHandle);
04382     Ops.push_back(N->getOperand(2));
04383     Ops.push_back(Chain);
04384     break;
04385   case NVPTXISD::Suld1DV4I16Zero:
04386     Opc = NVPTX::SULD_1D_V4I16_ZERO;
04387     Ops.push_back(TexHandle);
04388     Ops.push_back(N->getOperand(2));
04389     Ops.push_back(Chain);
04390     break;
04391   case NVPTXISD::Suld1DV4I32Zero:
04392     Opc = NVPTX::SULD_1D_V4I32_ZERO;
04393     Ops.push_back(TexHandle);
04394     Ops.push_back(N->getOperand(2));
04395     Ops.push_back(Chain);
04396     break;
04397   case NVPTXISD::Suld1DArrayI8Zero:
04398     Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
04399     Ops.push_back(TexHandle);
04400     Ops.push_back(N->getOperand(2));
04401     Ops.push_back(N->getOperand(3));
04402     Ops.push_back(Chain);
04403     break;
04404   case NVPTXISD::Suld1DArrayI16Zero:
04405     Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
04406     Ops.push_back(TexHandle);
04407     Ops.push_back(N->getOperand(2));
04408     Ops.push_back(N->getOperand(3));
04409     Ops.push_back(Chain);
04410     break;
04411   case NVPTXISD::Suld1DArrayI32Zero:
04412     Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
04413     Ops.push_back(TexHandle);
04414     Ops.push_back(N->getOperand(2));
04415     Ops.push_back(N->getOperand(3));
04416     Ops.push_back(Chain);
04417     break;
04418   case NVPTXISD::Suld1DArrayI64Zero:
04419     Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
04420     Ops.push_back(TexHandle);
04421     Ops.push_back(N->getOperand(2));
04422     Ops.push_back(N->getOperand(3));
04423     Ops.push_back(Chain);
04424     break;
04425   case NVPTXISD::Suld1DArrayV2I8Zero:
04426     Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
04427     Ops.push_back(TexHandle);
04428     Ops.push_back(N->getOperand(2));
04429     Ops.push_back(N->getOperand(3));
04430     Ops.push_back(Chain);
04431     break;
04432   case NVPTXISD::Suld1DArrayV2I16Zero:
04433     Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
04434     Ops.push_back(TexHandle);
04435     Ops.push_back(N->getOperand(2));
04436     Ops.push_back(N->getOperand(3));
04437     Ops.push_back(Chain);
04438     break;
04439   case NVPTXISD::Suld1DArrayV2I32Zero:
04440     Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
04441     Ops.push_back(TexHandle);
04442     Ops.push_back(N->getOperand(2));
04443     Ops.push_back(N->getOperand(3));
04444     Ops.push_back(Chain);
04445     break;
04446   case NVPTXISD::Suld1DArrayV2I64Zero:
04447     Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
04448     Ops.push_back(TexHandle);
04449     Ops.push_back(N->getOperand(2));
04450     Ops.push_back(N->getOperand(3));
04451     Ops.push_back(Chain);
04452     break;
04453   case NVPTXISD::Suld1DArrayV4I8Zero:
04454     Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
04455     Ops.push_back(TexHandle);
04456     Ops.push_back(N->getOperand(2));
04457     Ops.push_back(N->getOperand(3));
04458     Ops.push_back(Chain);
04459     break;
04460   case NVPTXISD::Suld1DArrayV4I16Zero:
04461     Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
04462     Ops.push_back(TexHandle);
04463     Ops.push_back(N->getOperand(2));
04464     Ops.push_back(N->getOperand(3));
04465     Ops.push_back(Chain);
04466     break;
04467   case NVPTXISD::Suld1DArrayV4I32Zero:
04468     Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
04469     Ops.push_back(TexHandle);
04470     Ops.push_back(N->getOperand(2));
04471     Ops.push_back(N->getOperand(3));
04472     Ops.push_back(Chain);
04473     break;
04474   case NVPTXISD::Suld2DI8Zero:
04475     Opc = NVPTX::SULD_2D_I8_ZERO;
04476     Ops.push_back(TexHandle);
04477     Ops.push_back(N->getOperand(2));
04478     Ops.push_back(N->getOperand(3));
04479     Ops.push_back(Chain);
04480     break;
04481   case NVPTXISD::Suld2DI16Zero:
04482     Opc = NVPTX::SULD_2D_I16_ZERO;
04483     Ops.push_back(TexHandle);
04484     Ops.push_back(N->getOperand(2));
04485     Ops.push_back(N->getOperand(3));
04486     Ops.push_back(Chain);
04487     break;
04488   case NVPTXISD::Suld2DI32Zero:
04489     Opc = NVPTX::SULD_2D_I32_ZERO;
04490     Ops.push_back(TexHandle);
04491     Ops.push_back(N->getOperand(2));
04492     Ops.push_back(N->getOperand(3));
04493     Ops.push_back(Chain);
04494     break;
04495   case NVPTXISD::Suld2DI64Zero:
04496     Opc = NVPTX::SULD_2D_I64_ZERO;
04497     Ops.push_back(TexHandle);
04498     Ops.push_back(N->getOperand(2));
04499     Ops.push_back(N->getOperand(3));
04500     Ops.push_back(Chain);
04501     break;
04502   case NVPTXISD::Suld2DV2I8Zero:
04503     Opc = NVPTX::SULD_2D_V2I8_ZERO;
04504     Ops.push_back(TexHandle);
04505     Ops.push_back(N->getOperand(2));
04506     Ops.push_back(N->getOperand(3));
04507     Ops.push_back(Chain);
04508     break;
04509   case NVPTXISD::Suld2DV2I16Zero:
04510     Opc = NVPTX::SULD_2D_V2I16_ZERO;
04511     Ops.push_back(TexHandle);
04512     Ops.push_back(N->getOperand(2));
04513     Ops.push_back(N->getOperand(3));
04514     Ops.push_back(Chain);
04515     break;
04516   case NVPTXISD::Suld2DV2I32Zero:
04517     Opc = NVPTX::SULD_2D_V2I32_ZERO;
04518     Ops.push_back(TexHandle);
04519     Ops.push_back(N->getOperand(2));
04520     Ops.push_back(N->getOperand(3));
04521     Ops.push_back(Chain);
04522     break;
04523   case NVPTXISD::Suld2DV2I64Zero:
04524     Opc = NVPTX::SULD_2D_V2I64_ZERO;
04525     Ops.push_back(TexHandle);
04526     Ops.push_back(N->getOperand(2));
04527     Ops.push_back(N->getOperand(3));
04528     Ops.push_back(Chain);
04529     break;
04530   case NVPTXISD::Suld2DV4I8Zero:
04531     Opc = NVPTX::SULD_2D_V4I8_ZERO;
04532     Ops.push_back(TexHandle);
04533     Ops.push_back(N->getOperand(2));
04534     Ops.push_back(N->getOperand(3));
04535     Ops.push_back(Chain);
04536     break;
04537   case NVPTXISD::Suld2DV4I16Zero:
04538     Opc = NVPTX::SULD_2D_V4I16_ZERO;
04539     Ops.push_back(TexHandle);
04540     Ops.push_back(N->getOperand(2));
04541     Ops.push_back(N->getOperand(3));
04542     Ops.push_back(Chain);
04543     break;
04544   case NVPTXISD::Suld2DV4I32Zero:
04545     Opc = NVPTX::SULD_2D_V4I32_ZERO;
04546     Ops.push_back(TexHandle);
04547     Ops.push_back(N->getOperand(2));
04548     Ops.push_back(N->getOperand(3));
04549     Ops.push_back(Chain);
04550     break;
04551   case NVPTXISD::Suld2DArrayI8Zero:
04552     Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
04553     Ops.push_back(TexHandle);
04554     Ops.push_back(N->getOperand(2));
04555     Ops.push_back(N->getOperand(3));
04556     Ops.push_back(N->getOperand(4));
04557     Ops.push_back(Chain);
04558     break;
04559   case NVPTXISD::Suld2DArrayI16Zero:
04560     Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
04561     Ops.push_back(TexHandle);
04562     Ops.push_back(N->getOperand(2));
04563     Ops.push_back(N->getOperand(3));
04564     Ops.push_back(N->getOperand(4));
04565     Ops.push_back(Chain);
04566     break;
04567   case NVPTXISD::Suld2DArrayI32Zero:
04568     Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
04569     Ops.push_back(TexHandle);
04570     Ops.push_back(N->getOperand(2));
04571     Ops.push_back(N->getOperand(3));
04572     Ops.push_back(N->getOperand(4));
04573     Ops.push_back(Chain);
04574     break;
04575   case NVPTXISD::Suld2DArrayI64Zero:
04576     Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
04577     Ops.push_back(TexHandle);
04578     Ops.push_back(N->getOperand(2));
04579     Ops.push_back(N->getOperand(3));
04580     Ops.push_back(N->getOperand(4));
04581     Ops.push_back(Chain);
04582     break;
04583   case NVPTXISD::Suld2DArrayV2I8Zero:
04584     Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
04585     Ops.push_back(TexHandle);
04586     Ops.push_back(N->getOperand(2));
04587     Ops.push_back(N->getOperand(3));
04588     Ops.push_back(N->getOperand(4));
04589     Ops.push_back(Chain);
04590     break;
04591   case NVPTXISD::Suld2DArrayV2I16Zero:
04592     Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
04593     Ops.push_back(TexHandle);
04594     Ops.push_back(N->getOperand(2));
04595     Ops.push_back(N->getOperand(3));
04596     Ops.push_back(N->getOperand(4));
04597     Ops.push_back(Chain);
04598     break;
04599   case NVPTXISD::Suld2DArrayV2I32Zero:
04600     Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
04601     Ops.push_back(TexHandle);
04602     Ops.push_back(N->getOperand(2));
04603     Ops.push_back(N->getOperand(3));
04604     Ops.push_back(N->getOperand(4));
04605     Ops.push_back(Chain);
04606     break;
04607   case NVPTXISD::Suld2DArrayV2I64Zero:
04608     Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
04609     Ops.push_back(TexHandle);
04610     Ops.push_back(N->getOperand(2));
04611     Ops.push_back(N->getOperand(3));
04612     Ops.push_back(N->getOperand(4));
04613     Ops.push_back(Chain);
04614     break;
04615   case NVPTXISD::Suld2DArrayV4I8Zero:
04616     Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
04617     Ops.push_back(TexHandle);
04618     Ops.push_back(N->getOperand(2));
04619     Ops.push_back(N->getOperand(3));
04620     Ops.push_back(N->getOperand(4));
04621     Ops.push_back(Chain);
04622     break;
04623   case NVPTXISD::Suld2DArrayV4I16Zero:
04624     Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
04625     Ops.push_back(TexHandle);
04626     Ops.push_back(N->getOperand(2));
04627     Ops.push_back(N->getOperand(3));
04628     Ops.push_back(N->getOperand(4));
04629     Ops.push_back(Chain);
04630     break;
04631   case NVPTXISD::Suld2DArrayV4I32Zero:
04632     Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
04633     Ops.push_back(TexHandle);
04634     Ops.push_back(N->getOperand(2));
04635     Ops.push_back(N->getOperand(3));
04636     Ops.push_back(N->getOperand(4));
04637     Ops.push_back(Chain);
04638     break;
04639   case NVPTXISD::Suld3DI8Zero:
04640     Opc = NVPTX::SULD_3D_I8_ZERO;
04641     Ops.push_back(TexHandle);
04642     Ops.push_back(N->getOperand(2));
04643     Ops.push_back(N->getOperand(3));
04644     Ops.push_back(N->getOperand(4));
04645     Ops.push_back(Chain);
04646     break;
04647   case NVPTXISD::Suld3DI16Zero:
04648     Opc = NVPTX::SULD_3D_I16_ZERO;
04649     Ops.push_back(TexHandle);
04650     Ops.push_back(N->getOperand(2));
04651     Ops.push_back(N->getOperand(3));
04652     Ops.push_back(N->getOperand(4));
04653     Ops.push_back(Chain);
04654     break;
04655   case NVPTXISD::Suld3DI32Zero:
04656     Opc = NVPTX::SULD_3D_I32_ZERO;
04657     Ops.push_back(TexHandle);
04658     Ops.push_back(N->getOperand(2));
04659     Ops.push_back(N->getOperand(3));
04660     Ops.push_back(N->getOperand(4));
04661     Ops.push_back(Chain);
04662     break;
04663   case NVPTXISD::Suld3DI64Zero:
04664     Opc = NVPTX::SULD_3D_I64_ZERO;
04665     Ops.push_back(TexHandle);
04666     Ops.push_back(N->getOperand(2));
04667     Ops.push_back(N->getOperand(3));
04668     Ops.push_back(N->getOperand(4));
04669     Ops.push_back(Chain);
04670     break;
04671   case NVPTXISD::Suld3DV2I8Zero:
04672     Opc = NVPTX::SULD_3D_V2I8_ZERO;
04673     Ops.push_back(TexHandle);
04674     Ops.push_back(N->getOperand(2));
04675     Ops.push_back(N->getOperand(3));
04676     Ops.push_back(N->getOperand(4));
04677     Ops.push_back(Chain);
04678     break;
04679   case NVPTXISD::Suld3DV2I16Zero:
04680     Opc = NVPTX::SULD_3D_V2I16_ZERO;
04681     Ops.push_back(TexHandle);
04682     Ops.push_back(N->getOperand(2));
04683     Ops.push_back(N->getOperand(3));
04684     Ops.push_back(N->getOperand(4));
04685     Ops.push_back(Chain);
04686     break;
04687   case NVPTXISD::Suld3DV2I32Zero:
04688     Opc = NVPTX::SULD_3D_V2I32_ZERO;
04689     Ops.push_back(TexHandle);
04690     Ops.push_back(N->getOperand(2));
04691     Ops.push_back(N->getOperand(3));
04692     Ops.push_back(N->getOperand(4));
04693     Ops.push_back(Chain);
04694     break;
04695   case NVPTXISD::Suld3DV2I64Zero:
04696     Opc = NVPTX::SULD_3D_V2I64_ZERO;
04697     Ops.push_back(TexHandle);
04698     Ops.push_back(N->getOperand(2));
04699     Ops.push_back(N->getOperand(3));
04700     Ops.push_back(N->getOperand(4));
04701     Ops.push_back(Chain);
04702     break;
04703   case NVPTXISD::Suld3DV4I8Zero:
04704     Opc = NVPTX::SULD_3D_V4I8_ZERO;
04705     Ops.push_back(TexHandle);
04706     Ops.push_back(N->getOperand(2));
04707     Ops.push_back(N->getOperand(3));
04708     Ops.push_back(N->getOperand(4));
04709     Ops.push_back(Chain);
04710     break;
04711   case NVPTXISD::Suld3DV4I16Zero:
04712     Opc = NVPTX::SULD_3D_V4I16_ZERO;
04713     Ops.push_back(TexHandle);
04714     Ops.push_back(N->getOperand(2));
04715     Ops.push_back(N->getOperand(3));
04716     Ops.push_back(N->getOperand(4));
04717     Ops.push_back(Chain);
04718     break;
04719   case NVPTXISD::Suld3DV4I32Zero:
04720     Opc = NVPTX::SULD_3D_V4I32_ZERO;
04721     Ops.push_back(TexHandle);
04722     Ops.push_back(N->getOperand(2));
04723     Ops.push_back(N->getOperand(3));
04724     Ops.push_back(N->getOperand(4));
04725     Ops.push_back(Chain);
04726     break;
04727   }
04728   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
04729   return Ret;
04730 }
04731 
04732 
04733 /// SelectBFE - Look for instruction sequences that can be made more efficient
04734 /// by using the 'bfe' (bit-field extract) PTX instruction
04735 SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
04736   SDLoc DL(N);
04737   SDValue LHS = N->getOperand(0);
04738   SDValue RHS = N->getOperand(1);
04739   SDValue Len;
04740   SDValue Start;
04741   SDValue Val;
04742   bool IsSigned = false;
04743 
04744   if (N->getOpcode() == ISD::AND) {
04745     // Canonicalize the operands
04746     // We want 'and %val, %mask'
04747     if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
04748       std::swap(LHS, RHS);
04749     }
04750 
04751     ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
04752     if (!Mask) {
04753       // We need a constant mask on the RHS of the AND
04754       return NULL;
04755     }
04756 
04757     // Extract the mask bits
04758     uint64_t MaskVal = Mask->getZExtValue();
04759     if (!isMask_64(MaskVal)) {
04760       // We *could* handle shifted masks here, but doing so would require an
04761       // 'and' operation to fix up the low-order bits so we would trade
04762       // shr+and for bfe+and, which has the same throughput
04763       return NULL;
04764     }
04765 
04766     // How many bits are in our mask?
04767     uint64_t NumBits = countTrailingOnes(MaskVal);
04768     Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
04769 
04770     if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
04771       // We have a 'srl/and' pair, extract the effective start bit and length
04772       Val = LHS.getNode()->getOperand(0);
04773       Start = LHS.getNode()->getOperand(1);
04774       ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
04775       if (StartConst) {
04776         uint64_t StartVal = StartConst->getZExtValue();
04777         // How many "good" bits do we have left?  "good" is defined here as bits
04778         // that exist in the original value, not shifted in.
04779         uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
04780         if (NumBits > GoodBits) {
04781           // Do not handle the case where bits have been shifted in. In theory
04782           // we could handle this, but the cost is likely higher than just
04783           // emitting the srl/and pair.
04784           return NULL;
04785         }
04786         Start = CurDAG->getTargetConstant(StartVal, DL, MVT::i32);
04787       } else {
04788         // Do not handle the case where the shift amount (can be zero if no srl
04789         // was found) is not constant. We could handle this case, but it would
04790         // require run-time logic that would be more expensive than just
04791         // emitting the srl/and pair.
04792         return NULL;
04793       }
04794     } else {
04795       // Do not handle the case where the LHS of the and is not a shift. While
04796       // it would be trivial to handle this case, it would just transform
04797       // 'and' -> 'bfe', but 'and' has higher-throughput.
04798       return NULL;
04799     }
04800   } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
04801     if (LHS->getOpcode() == ISD::AND) {
04802       ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
04803       if (!ShiftCnst) {
04804         // Shift amount must be constant
04805         return NULL;
04806       }
04807 
04808       uint64_t ShiftAmt = ShiftCnst->getZExtValue();
04809 
04810       SDValue AndLHS = LHS->getOperand(0);
04811       SDValue AndRHS = LHS->getOperand(1);
04812 
04813       // Canonicalize the AND to have the mask on the RHS
04814       if (isa<ConstantSDNode>(AndLHS)) {
04815         std::swap(AndLHS, AndRHS);
04816       }
04817 
04818       ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
04819       if (!MaskCnst) {
04820         // Mask must be constant
04821         return NULL;
04822       }
04823 
04824       uint64_t MaskVal = MaskCnst->getZExtValue();
04825       uint64_t NumZeros;
04826       uint64_t NumBits;
04827       if (isMask_64(MaskVal)) {
04828         NumZeros = 0;
04829         // The number of bits in the result bitfield will be the number of
04830         // trailing ones (the AND) minus the number of bits we shift off
04831         NumBits = countTrailingOnes(MaskVal) - ShiftAmt;
04832       } else if (isShiftedMask_64(MaskVal)) {
04833         NumZeros = countTrailingZeros(MaskVal);
04834         unsigned NumOnes = countTrailingOnes(MaskVal >> NumZeros);
04835         // The number of bits in the result bitfield will be the number of
04836         // trailing zeros plus the number of set bits in the mask minus the
04837         // number of bits we shift off
04838         NumBits = NumZeros + NumOnes - ShiftAmt;
04839       } else {
04840         // This is not a mask we can handle
04841         return NULL;
04842       }
04843 
04844       if (ShiftAmt < NumZeros) {
04845         // Handling this case would require extra logic that would make this
04846         // transformation non-profitable
04847         return NULL;
04848       }
04849 
04850       Val = AndLHS;
04851       Start = CurDAG->getTargetConstant(ShiftAmt, DL, MVT::i32);
04852       Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
04853     } else if (LHS->getOpcode() == ISD::SHL) {
04854       // Here, we have a pattern like:
04855       //
04856       // (sra (shl val, NN), MM)
04857       // or
04858       // (srl (shl val, NN), MM)
04859       //
04860       // If MM >= NN, we can efficiently optimize this with bfe
04861       Val = LHS->getOperand(0);
04862 
04863       SDValue ShlRHS = LHS->getOperand(1);
04864       ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
04865       if (!ShlCnst) {
04866         // Shift amount must be constant
04867         return NULL;
04868       }
04869       uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
04870 
04871       SDValue ShrRHS = RHS;
04872       ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
04873       if (!ShrCnst) {
04874         // Shift amount must be constant
04875         return NULL;
04876       }
04877       uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
04878 
04879       // To avoid extra codegen and be profitable, we need Outer >= Inner
04880       if (OuterShiftAmt < InnerShiftAmt) {
04881         return NULL;
04882       }
04883 
04884       // If the outer shift is more than the type size, we have no bitfield to
04885       // extract (since we also check that the inner shift is <= the outer shift
04886       // then this also implies that the inner shift is < the type size)
04887       if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
04888         return NULL;
04889       }
04890 
04891       Start =
04892         CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, DL, MVT::i32);
04893       Len =
04894         CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
04895                                   OuterShiftAmt, DL, MVT::i32);
04896 
04897       if (N->getOpcode() == ISD::SRA) {
04898         // If we have a arithmetic right shift, we need to use the signed bfe
04899         // variant
04900         IsSigned = true;
04901       }
04902     } else {
04903       // No can do...
04904       return NULL;
04905     }
04906   } else {
04907     // No can do...
04908     return NULL;
04909   }
04910 
04911 
04912   unsigned Opc;
04913   // For the BFE operations we form here from "and" and "srl", always use the
04914   // unsigned variants.
04915   if (Val.getValueType() == MVT::i32) {
04916     if (IsSigned) {
04917       Opc = NVPTX::BFE_S32rii;
04918     } else {
04919       Opc = NVPTX::BFE_U32rii;
04920     }
04921   } else if (Val.getValueType() == MVT::i64) {
04922     if (IsSigned) {
04923       Opc = NVPTX::BFE_S64rii;
04924     } else {
04925       Opc = NVPTX::BFE_U64rii;
04926     }
04927   } else {
04928     // We cannot handle this type
04929     return NULL;
04930   }
04931 
04932   SDValue Ops[] = {
04933     Val, Start, Len
04934   };
04935 
04936   return CurDAG->getMachineNode(Opc, DL, N->getVTList(), Ops);
04937 }
04938 
04939 // SelectDirectAddr - Match a direct address for DAG.
04940 // A direct address could be a globaladdress or externalsymbol.
04941 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
04942   // Return true if TGA or ES.
04943   if (N.getOpcode() == ISD::TargetGlobalAddress ||
04944       N.getOpcode() == ISD::TargetExternalSymbol) {
04945     Address = N;
04946     return true;
04947   }
04948   if (N.getOpcode() == NVPTXISD::Wrapper) {
04949     Address = N.getOperand(0);
04950     return true;
04951   }
04952   if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
04953     unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
04954     if (IID == Intrinsic::nvvm_ptr_gen_to_param)
04955       if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
04956         return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
04957   }
04958   return false;
04959 }
04960 
04961 // symbol+offset
04962 bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
04963     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
04964   if (Addr.getOpcode() == ISD::ADD) {
04965     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
04966       SDValue base = Addr.getOperand(0);
04967       if (SelectDirectAddr(base, Base)) {
04968         Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
04969                                            mvt);
04970         return true;
04971       }
04972     }
04973   }
04974   return false;
04975 }
04976 
04977 // symbol+offset
04978 bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
04979                                      SDValue &Base, SDValue &Offset) {
04980   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
04981 }
04982 
04983 // symbol+offset
04984 bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
04985                                        SDValue &Base, SDValue &Offset) {
04986   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
04987 }
04988 
04989 // register+offset
04990 bool NVPTXDAGToDAGISel::SelectADDRri_imp(
04991     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
04992   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
04993     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
04994     Offset = CurDAG->getTargetConstant(0, SDLoc(OpNode), mvt);
04995     return true;
04996   }
04997   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
04998       Addr.getOpcode() == ISD::TargetGlobalAddress)
04999     return false; // direct calls.
05000 
05001   if (Addr.getOpcode() == ISD::ADD) {
05002     if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
05003       return false;
05004     }
05005     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
05006       if (FrameIndexSDNode *FIN =
05007               dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
05008         // Constant offset from frame ref.
05009         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
05010       else
05011         Base = Addr.getOperand(0);
05012       Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
05013                                          mvt);
05014       return true;
05015     }
05016   }
05017   return false;
05018 }
05019 
05020 // register+offset
05021 bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
05022                                      SDValue &Base, SDValue &Offset) {
05023   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
05024 }
05025 
05026 // register+offset
05027 bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
05028                                        SDValue &Base, SDValue &Offset) {
05029   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
05030 }
05031 
05032 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
05033                                                  unsigned int spN) const {
05034   const Value *Src = nullptr;
05035   if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
05036     if (spN == 0 && mN->getMemOperand()->getPseudoValue())
05037       return true;
05038     Src = mN->getMemOperand()->getValue();
05039   }
05040   if (!Src)
05041     return false;
05042   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
05043     return (PT->getAddressSpace() == spN);
05044   return false;
05045 }
05046 
05047 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
05048 /// inline asm expressions.
05049 bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
05050     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
05051   SDValue Op0, Op1;
05052   switch (ConstraintID) {
05053   default:
05054     return true;
05055   case InlineAsm::Constraint_m: // memory
05056     if (SelectDirectAddr(Op, Op0)) {
05057       OutOps.push_back(Op0);
05058       OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
05059       return false;
05060     }
05061     if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
05062       OutOps.push_back(Op0);
05063       OutOps.push_back(Op1);
05064       return false;
05065     }
05066     break;
05067   }
05068   return true;
05069 }