LLVM API Documentation

NVPTXISelDAGToDAG.cpp
Go to the documentation of this file.
00001 //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file defines an instruction selector for the NVPTX target.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "NVPTXISelDAGToDAG.h"
00015 #include "llvm/IR/GlobalValue.h"
00016 #include "llvm/IR/Instructions.h"
00017 #include "llvm/Support/CommandLine.h"
00018 #include "llvm/Support/Debug.h"
00019 #include "llvm/Support/ErrorHandling.h"
00020 #include "llvm/Support/raw_ostream.h"
00021 #include "llvm/Target/TargetIntrinsicInfo.h"
00022 
00023 using namespace llvm;
00024 
00025 #define DEBUG_TYPE "nvptx-isel"
00026 
00027 static cl::opt<int> UsePrecDivF32(
00028     "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
00029     cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
00030              " IEEE Compliant F32 div.rnd if available."),
00031     cl::init(2));
00032 
00033 static cl::opt<bool>
00034 UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
00035           cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
00036           cl::init(true));
00037 
00038 static cl::opt<bool>
00039 FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
00040            cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
00041            cl::init(false));
00042 
00043 
00044 /// createNVPTXISelDag - This pass converts a legalized DAG into a
00045 /// NVPTX-specific DAG, ready for instruction scheduling.
00046 FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
00047                                        llvm::CodeGenOpt::Level OptLevel) {
00048   return new NVPTXDAGToDAGISel(TM, OptLevel);
00049 }
00050 
00051 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
00052                                      CodeGenOpt::Level OptLevel)
00053     : SelectionDAGISel(tm, OptLevel), TM(tm) {
00054   doMulWide = (OptLevel > 0);
00055 }
00056 
00057 bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
00058     Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget());
00059     return SelectionDAGISel::runOnMachineFunction(MF);
00060 }
00061 
00062 int NVPTXDAGToDAGISel::getDivF32Level() const {
00063   if (UsePrecDivF32.getNumOccurrences() > 0) {
00064     // If nvptx-prec-div32=N is used on the command-line, always honor it
00065     return UsePrecDivF32;
00066   } else {
00067     // Otherwise, use div.approx if fast math is enabled
00068     if (TM.Options.UnsafeFPMath)
00069       return 0;
00070     else
00071       return 2;
00072   }
00073 }
00074 
00075 bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
00076   if (UsePrecSqrtF32.getNumOccurrences() > 0) {
00077     // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
00078     return UsePrecSqrtF32;
00079   } else {
00080     // Otherwise, use sqrt.approx if fast math is enabled
00081     if (TM.Options.UnsafeFPMath)
00082       return false;
00083     else
00084       return true;
00085   }
00086 }
00087 
00088 bool NVPTXDAGToDAGISel::useF32FTZ() const {
00089   if (FtzEnabled.getNumOccurrences() > 0) {
00090     // If nvptx-f32ftz is used on the command-line, always honor it
00091     return FtzEnabled;
00092   } else {
00093     const Function *F = MF->getFunction();
00094     // Otherwise, check for an nvptx-f32ftz attribute on the function
00095     if (F->hasFnAttribute("nvptx-f32ftz"))
00096       return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true";
00097     else
00098       return false;
00099   }
00100 }
00101 
00102 bool NVPTXDAGToDAGISel::allowFMA() const {
00103   const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
00104   return TL->allowFMA(*MF, OptLevel);
00105 }
00106 
00107 /// Select - Select instructions not customized! Used for
00108 /// expanded, promoted and normal instructions.
00109 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
00110 
00111   if (N->isMachineOpcode()) {
00112     N->setNodeId(-1);
00113     return nullptr; // Already selected.
00114   }
00115 
00116   SDNode *ResNode = nullptr;
00117   switch (N->getOpcode()) {
00118   case ISD::LOAD:
00119     ResNode = SelectLoad(N);
00120     break;
00121   case ISD::STORE:
00122     ResNode = SelectStore(N);
00123     break;
00124   case NVPTXISD::LoadV2:
00125   case NVPTXISD::LoadV4:
00126     ResNode = SelectLoadVector(N);
00127     break;
00128   case NVPTXISD::LDGV2:
00129   case NVPTXISD::LDGV4:
00130   case NVPTXISD::LDUV2:
00131   case NVPTXISD::LDUV4:
00132     ResNode = SelectLDGLDU(N);
00133     break;
00134   case NVPTXISD::StoreV2:
00135   case NVPTXISD::StoreV4:
00136     ResNode = SelectStoreVector(N);
00137     break;
00138   case NVPTXISD::LoadParam:
00139   case NVPTXISD::LoadParamV2:
00140   case NVPTXISD::LoadParamV4:
00141     ResNode = SelectLoadParam(N);
00142     break;
00143   case NVPTXISD::StoreRetval:
00144   case NVPTXISD::StoreRetvalV2:
00145   case NVPTXISD::StoreRetvalV4:
00146     ResNode = SelectStoreRetval(N);
00147     break;
00148   case NVPTXISD::StoreParam:
00149   case NVPTXISD::StoreParamV2:
00150   case NVPTXISD::StoreParamV4:
00151   case NVPTXISD::StoreParamS32:
00152   case NVPTXISD::StoreParamU32:
00153     ResNode = SelectStoreParam(N);
00154     break;
00155   case ISD::INTRINSIC_WO_CHAIN:
00156     ResNode = SelectIntrinsicNoChain(N);
00157     break;
00158   case ISD::INTRINSIC_W_CHAIN:
00159     ResNode = SelectIntrinsicChain(N);
00160     break;
00161   case NVPTXISD::Tex1DFloatS32:
00162   case NVPTXISD::Tex1DFloatFloat:
00163   case NVPTXISD::Tex1DFloatFloatLevel:
00164   case NVPTXISD::Tex1DFloatFloatGrad:
00165   case NVPTXISD::Tex1DS32S32:
00166   case NVPTXISD::Tex1DS32Float:
00167   case NVPTXISD::Tex1DS32FloatLevel:
00168   case NVPTXISD::Tex1DS32FloatGrad:
00169   case NVPTXISD::Tex1DU32S32:
00170   case NVPTXISD::Tex1DU32Float:
00171   case NVPTXISD::Tex1DU32FloatLevel:
00172   case NVPTXISD::Tex1DU32FloatGrad:
00173   case NVPTXISD::Tex1DArrayFloatS32:
00174   case NVPTXISD::Tex1DArrayFloatFloat:
00175   case NVPTXISD::Tex1DArrayFloatFloatLevel:
00176   case NVPTXISD::Tex1DArrayFloatFloatGrad:
00177   case NVPTXISD::Tex1DArrayS32S32:
00178   case NVPTXISD::Tex1DArrayS32Float:
00179   case NVPTXISD::Tex1DArrayS32FloatLevel:
00180   case NVPTXISD::Tex1DArrayS32FloatGrad:
00181   case NVPTXISD::Tex1DArrayU32S32:
00182   case NVPTXISD::Tex1DArrayU32Float:
00183   case NVPTXISD::Tex1DArrayU32FloatLevel:
00184   case NVPTXISD::Tex1DArrayU32FloatGrad:
00185   case NVPTXISD::Tex2DFloatS32:
00186   case NVPTXISD::Tex2DFloatFloat:
00187   case NVPTXISD::Tex2DFloatFloatLevel:
00188   case NVPTXISD::Tex2DFloatFloatGrad:
00189   case NVPTXISD::Tex2DS32S32:
00190   case NVPTXISD::Tex2DS32Float:
00191   case NVPTXISD::Tex2DS32FloatLevel:
00192   case NVPTXISD::Tex2DS32FloatGrad:
00193   case NVPTXISD::Tex2DU32S32:
00194   case NVPTXISD::Tex2DU32Float:
00195   case NVPTXISD::Tex2DU32FloatLevel:
00196   case NVPTXISD::Tex2DU32FloatGrad:
00197   case NVPTXISD::Tex2DArrayFloatS32:
00198   case NVPTXISD::Tex2DArrayFloatFloat:
00199   case NVPTXISD::Tex2DArrayFloatFloatLevel:
00200   case NVPTXISD::Tex2DArrayFloatFloatGrad:
00201   case NVPTXISD::Tex2DArrayS32S32:
00202   case NVPTXISD::Tex2DArrayS32Float:
00203   case NVPTXISD::Tex2DArrayS32FloatLevel:
00204   case NVPTXISD::Tex2DArrayS32FloatGrad:
00205   case NVPTXISD::Tex2DArrayU32S32:
00206   case NVPTXISD::Tex2DArrayU32Float:
00207   case NVPTXISD::Tex2DArrayU32FloatLevel:
00208   case NVPTXISD::Tex2DArrayU32FloatGrad:
00209   case NVPTXISD::Tex3DFloatS32:
00210   case NVPTXISD::Tex3DFloatFloat:
00211   case NVPTXISD::Tex3DFloatFloatLevel:
00212   case NVPTXISD::Tex3DFloatFloatGrad:
00213   case NVPTXISD::Tex3DS32S32:
00214   case NVPTXISD::Tex3DS32Float:
00215   case NVPTXISD::Tex3DS32FloatLevel:
00216   case NVPTXISD::Tex3DS32FloatGrad:
00217   case NVPTXISD::Tex3DU32S32:
00218   case NVPTXISD::Tex3DU32Float:
00219   case NVPTXISD::Tex3DU32FloatLevel:
00220   case NVPTXISD::Tex3DU32FloatGrad:
00221   case NVPTXISD::TexCubeFloatFloat:
00222   case NVPTXISD::TexCubeFloatFloatLevel:
00223   case NVPTXISD::TexCubeS32Float:
00224   case NVPTXISD::TexCubeS32FloatLevel:
00225   case NVPTXISD::TexCubeU32Float:
00226   case NVPTXISD::TexCubeU32FloatLevel:
00227   case NVPTXISD::TexCubeArrayFloatFloat:
00228   case NVPTXISD::TexCubeArrayFloatFloatLevel:
00229   case NVPTXISD::TexCubeArrayS32Float:
00230   case NVPTXISD::TexCubeArrayS32FloatLevel:
00231   case NVPTXISD::TexCubeArrayU32Float:
00232   case NVPTXISD::TexCubeArrayU32FloatLevel:
00233   case NVPTXISD::Tld4R2DFloatFloat:
00234   case NVPTXISD::Tld4G2DFloatFloat:
00235   case NVPTXISD::Tld4B2DFloatFloat:
00236   case NVPTXISD::Tld4A2DFloatFloat:
00237   case NVPTXISD::Tld4R2DS64Float:
00238   case NVPTXISD::Tld4G2DS64Float:
00239   case NVPTXISD::Tld4B2DS64Float:
00240   case NVPTXISD::Tld4A2DS64Float:
00241   case NVPTXISD::Tld4R2DU64Float:
00242   case NVPTXISD::Tld4G2DU64Float:
00243   case NVPTXISD::Tld4B2DU64Float:
00244   case NVPTXISD::Tld4A2DU64Float:
00245   case NVPTXISD::TexUnified1DFloatS32:
00246   case NVPTXISD::TexUnified1DFloatFloat:
00247   case NVPTXISD::TexUnified1DFloatFloatLevel:
00248   case NVPTXISD::TexUnified1DFloatFloatGrad:
00249   case NVPTXISD::TexUnified1DS32S32:
00250   case NVPTXISD::TexUnified1DS32Float:
00251   case NVPTXISD::TexUnified1DS32FloatLevel:
00252   case NVPTXISD::TexUnified1DS32FloatGrad:
00253   case NVPTXISD::TexUnified1DU32S32:
00254   case NVPTXISD::TexUnified1DU32Float:
00255   case NVPTXISD::TexUnified1DU32FloatLevel:
00256   case NVPTXISD::TexUnified1DU32FloatGrad:
00257   case NVPTXISD::TexUnified1DArrayFloatS32:
00258   case NVPTXISD::TexUnified1DArrayFloatFloat:
00259   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
00260   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
00261   case NVPTXISD::TexUnified1DArrayS32S32:
00262   case NVPTXISD::TexUnified1DArrayS32Float:
00263   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
00264   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
00265   case NVPTXISD::TexUnified1DArrayU32S32:
00266   case NVPTXISD::TexUnified1DArrayU32Float:
00267   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
00268   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
00269   case NVPTXISD::TexUnified2DFloatS32:
00270   case NVPTXISD::TexUnified2DFloatFloat:
00271   case NVPTXISD::TexUnified2DFloatFloatLevel:
00272   case NVPTXISD::TexUnified2DFloatFloatGrad:
00273   case NVPTXISD::TexUnified2DS32S32:
00274   case NVPTXISD::TexUnified2DS32Float:
00275   case NVPTXISD::TexUnified2DS32FloatLevel:
00276   case NVPTXISD::TexUnified2DS32FloatGrad:
00277   case NVPTXISD::TexUnified2DU32S32:
00278   case NVPTXISD::TexUnified2DU32Float:
00279   case NVPTXISD::TexUnified2DU32FloatLevel:
00280   case NVPTXISD::TexUnified2DU32FloatGrad:
00281   case NVPTXISD::TexUnified2DArrayFloatS32:
00282   case NVPTXISD::TexUnified2DArrayFloatFloat:
00283   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
00284   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
00285   case NVPTXISD::TexUnified2DArrayS32S32:
00286   case NVPTXISD::TexUnified2DArrayS32Float:
00287   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
00288   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
00289   case NVPTXISD::TexUnified2DArrayU32S32:
00290   case NVPTXISD::TexUnified2DArrayU32Float:
00291   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
00292   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
00293   case NVPTXISD::TexUnified3DFloatS32:
00294   case NVPTXISD::TexUnified3DFloatFloat:
00295   case NVPTXISD::TexUnified3DFloatFloatLevel:
00296   case NVPTXISD::TexUnified3DFloatFloatGrad:
00297   case NVPTXISD::TexUnified3DS32S32:
00298   case NVPTXISD::TexUnified3DS32Float:
00299   case NVPTXISD::TexUnified3DS32FloatLevel:
00300   case NVPTXISD::TexUnified3DS32FloatGrad:
00301   case NVPTXISD::TexUnified3DU32S32:
00302   case NVPTXISD::TexUnified3DU32Float:
00303   case NVPTXISD::TexUnified3DU32FloatLevel:
00304   case NVPTXISD::TexUnified3DU32FloatGrad:
00305   case NVPTXISD::TexUnifiedCubeFloatFloat:
00306   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
00307   case NVPTXISD::TexUnifiedCubeS32Float:
00308   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
00309   case NVPTXISD::TexUnifiedCubeU32Float:
00310   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
00311   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
00312   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
00313   case NVPTXISD::TexUnifiedCubeArrayS32Float:
00314   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
00315   case NVPTXISD::TexUnifiedCubeArrayU32Float:
00316   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
00317   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
00318   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
00319   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
00320   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
00321   case NVPTXISD::Tld4UnifiedR2DS64Float:
00322   case NVPTXISD::Tld4UnifiedG2DS64Float:
00323   case NVPTXISD::Tld4UnifiedB2DS64Float:
00324   case NVPTXISD::Tld4UnifiedA2DS64Float:
00325   case NVPTXISD::Tld4UnifiedR2DU64Float:
00326   case NVPTXISD::Tld4UnifiedG2DU64Float:
00327   case NVPTXISD::Tld4UnifiedB2DU64Float:
00328   case NVPTXISD::Tld4UnifiedA2DU64Float:
00329     ResNode = SelectTextureIntrinsic(N);
00330     break;
00331   case NVPTXISD::Suld1DI8Clamp:
00332   case NVPTXISD::Suld1DI16Clamp:
00333   case NVPTXISD::Suld1DI32Clamp:
00334   case NVPTXISD::Suld1DI64Clamp:
00335   case NVPTXISD::Suld1DV2I8Clamp:
00336   case NVPTXISD::Suld1DV2I16Clamp:
00337   case NVPTXISD::Suld1DV2I32Clamp:
00338   case NVPTXISD::Suld1DV2I64Clamp:
00339   case NVPTXISD::Suld1DV4I8Clamp:
00340   case NVPTXISD::Suld1DV4I16Clamp:
00341   case NVPTXISD::Suld1DV4I32Clamp:
00342   case NVPTXISD::Suld1DArrayI8Clamp:
00343   case NVPTXISD::Suld1DArrayI16Clamp:
00344   case NVPTXISD::Suld1DArrayI32Clamp:
00345   case NVPTXISD::Suld1DArrayI64Clamp:
00346   case NVPTXISD::Suld1DArrayV2I8Clamp:
00347   case NVPTXISD::Suld1DArrayV2I16Clamp:
00348   case NVPTXISD::Suld1DArrayV2I32Clamp:
00349   case NVPTXISD::Suld1DArrayV2I64Clamp:
00350   case NVPTXISD::Suld1DArrayV4I8Clamp:
00351   case NVPTXISD::Suld1DArrayV4I16Clamp:
00352   case NVPTXISD::Suld1DArrayV4I32Clamp:
00353   case NVPTXISD::Suld2DI8Clamp:
00354   case NVPTXISD::Suld2DI16Clamp:
00355   case NVPTXISD::Suld2DI32Clamp:
00356   case NVPTXISD::Suld2DI64Clamp:
00357   case NVPTXISD::Suld2DV2I8Clamp:
00358   case NVPTXISD::Suld2DV2I16Clamp:
00359   case NVPTXISD::Suld2DV2I32Clamp:
00360   case NVPTXISD::Suld2DV2I64Clamp:
00361   case NVPTXISD::Suld2DV4I8Clamp:
00362   case NVPTXISD::Suld2DV4I16Clamp:
00363   case NVPTXISD::Suld2DV4I32Clamp:
00364   case NVPTXISD::Suld2DArrayI8Clamp:
00365   case NVPTXISD::Suld2DArrayI16Clamp:
00366   case NVPTXISD::Suld2DArrayI32Clamp:
00367   case NVPTXISD::Suld2DArrayI64Clamp:
00368   case NVPTXISD::Suld2DArrayV2I8Clamp:
00369   case NVPTXISD::Suld2DArrayV2I16Clamp:
00370   case NVPTXISD::Suld2DArrayV2I32Clamp:
00371   case NVPTXISD::Suld2DArrayV2I64Clamp:
00372   case NVPTXISD::Suld2DArrayV4I8Clamp:
00373   case NVPTXISD::Suld2DArrayV4I16Clamp:
00374   case NVPTXISD::Suld2DArrayV4I32Clamp:
00375   case NVPTXISD::Suld3DI8Clamp:
00376   case NVPTXISD::Suld3DI16Clamp:
00377   case NVPTXISD::Suld3DI32Clamp:
00378   case NVPTXISD::Suld3DI64Clamp:
00379   case NVPTXISD::Suld3DV2I8Clamp:
00380   case NVPTXISD::Suld3DV2I16Clamp:
00381   case NVPTXISD::Suld3DV2I32Clamp:
00382   case NVPTXISD::Suld3DV2I64Clamp:
00383   case NVPTXISD::Suld3DV4I8Clamp:
00384   case NVPTXISD::Suld3DV4I16Clamp:
00385   case NVPTXISD::Suld3DV4I32Clamp:
00386   case NVPTXISD::Suld1DI8Trap:
00387   case NVPTXISD::Suld1DI16Trap:
00388   case NVPTXISD::Suld1DI32Trap:
00389   case NVPTXISD::Suld1DI64Trap:
00390   case NVPTXISD::Suld1DV2I8Trap:
00391   case NVPTXISD::Suld1DV2I16Trap:
00392   case NVPTXISD::Suld1DV2I32Trap:
00393   case NVPTXISD::Suld1DV2I64Trap:
00394   case NVPTXISD::Suld1DV4I8Trap:
00395   case NVPTXISD::Suld1DV4I16Trap:
00396   case NVPTXISD::Suld1DV4I32Trap:
00397   case NVPTXISD::Suld1DArrayI8Trap:
00398   case NVPTXISD::Suld1DArrayI16Trap:
00399   case NVPTXISD::Suld1DArrayI32Trap:
00400   case NVPTXISD::Suld1DArrayI64Trap:
00401   case NVPTXISD::Suld1DArrayV2I8Trap:
00402   case NVPTXISD::Suld1DArrayV2I16Trap:
00403   case NVPTXISD::Suld1DArrayV2I32Trap:
00404   case NVPTXISD::Suld1DArrayV2I64Trap:
00405   case NVPTXISD::Suld1DArrayV4I8Trap:
00406   case NVPTXISD::Suld1DArrayV4I16Trap:
00407   case NVPTXISD::Suld1DArrayV4I32Trap:
00408   case NVPTXISD::Suld2DI8Trap:
00409   case NVPTXISD::Suld2DI16Trap:
00410   case NVPTXISD::Suld2DI32Trap:
00411   case NVPTXISD::Suld2DI64Trap:
00412   case NVPTXISD::Suld2DV2I8Trap:
00413   case NVPTXISD::Suld2DV2I16Trap:
00414   case NVPTXISD::Suld2DV2I32Trap:
00415   case NVPTXISD::Suld2DV2I64Trap:
00416   case NVPTXISD::Suld2DV4I8Trap:
00417   case NVPTXISD::Suld2DV4I16Trap:
00418   case NVPTXISD::Suld2DV4I32Trap:
00419   case NVPTXISD::Suld2DArrayI8Trap:
00420   case NVPTXISD::Suld2DArrayI16Trap:
00421   case NVPTXISD::Suld2DArrayI32Trap:
00422   case NVPTXISD::Suld2DArrayI64Trap:
00423   case NVPTXISD::Suld2DArrayV2I8Trap:
00424   case NVPTXISD::Suld2DArrayV2I16Trap:
00425   case NVPTXISD::Suld2DArrayV2I32Trap:
00426   case NVPTXISD::Suld2DArrayV2I64Trap:
00427   case NVPTXISD::Suld2DArrayV4I8Trap:
00428   case NVPTXISD::Suld2DArrayV4I16Trap:
00429   case NVPTXISD::Suld2DArrayV4I32Trap:
00430   case NVPTXISD::Suld3DI8Trap:
00431   case NVPTXISD::Suld3DI16Trap:
00432   case NVPTXISD::Suld3DI32Trap:
00433   case NVPTXISD::Suld3DI64Trap:
00434   case NVPTXISD::Suld3DV2I8Trap:
00435   case NVPTXISD::Suld3DV2I16Trap:
00436   case NVPTXISD::Suld3DV2I32Trap:
00437   case NVPTXISD::Suld3DV2I64Trap:
00438   case NVPTXISD::Suld3DV4I8Trap:
00439   case NVPTXISD::Suld3DV4I16Trap:
00440   case NVPTXISD::Suld3DV4I32Trap:
00441   case NVPTXISD::Suld1DI8Zero:
00442   case NVPTXISD::Suld1DI16Zero:
00443   case NVPTXISD::Suld1DI32Zero:
00444   case NVPTXISD::Suld1DI64Zero:
00445   case NVPTXISD::Suld1DV2I8Zero:
00446   case NVPTXISD::Suld1DV2I16Zero:
00447   case NVPTXISD::Suld1DV2I32Zero:
00448   case NVPTXISD::Suld1DV2I64Zero:
00449   case NVPTXISD::Suld1DV4I8Zero:
00450   case NVPTXISD::Suld1DV4I16Zero:
00451   case NVPTXISD::Suld1DV4I32Zero:
00452   case NVPTXISD::Suld1DArrayI8Zero:
00453   case NVPTXISD::Suld1DArrayI16Zero:
00454   case NVPTXISD::Suld1DArrayI32Zero:
00455   case NVPTXISD::Suld1DArrayI64Zero:
00456   case NVPTXISD::Suld1DArrayV2I8Zero:
00457   case NVPTXISD::Suld1DArrayV2I16Zero:
00458   case NVPTXISD::Suld1DArrayV2I32Zero:
00459   case NVPTXISD::Suld1DArrayV2I64Zero:
00460   case NVPTXISD::Suld1DArrayV4I8Zero:
00461   case NVPTXISD::Suld1DArrayV4I16Zero:
00462   case NVPTXISD::Suld1DArrayV4I32Zero:
00463   case NVPTXISD::Suld2DI8Zero:
00464   case NVPTXISD::Suld2DI16Zero:
00465   case NVPTXISD::Suld2DI32Zero:
00466   case NVPTXISD::Suld2DI64Zero:
00467   case NVPTXISD::Suld2DV2I8Zero:
00468   case NVPTXISD::Suld2DV2I16Zero:
00469   case NVPTXISD::Suld2DV2I32Zero:
00470   case NVPTXISD::Suld2DV2I64Zero:
00471   case NVPTXISD::Suld2DV4I8Zero:
00472   case NVPTXISD::Suld2DV4I16Zero:
00473   case NVPTXISD::Suld2DV4I32Zero:
00474   case NVPTXISD::Suld2DArrayI8Zero:
00475   case NVPTXISD::Suld2DArrayI16Zero:
00476   case NVPTXISD::Suld2DArrayI32Zero:
00477   case NVPTXISD::Suld2DArrayI64Zero:
00478   case NVPTXISD::Suld2DArrayV2I8Zero:
00479   case NVPTXISD::Suld2DArrayV2I16Zero:
00480   case NVPTXISD::Suld2DArrayV2I32Zero:
00481   case NVPTXISD::Suld2DArrayV2I64Zero:
00482   case NVPTXISD::Suld2DArrayV4I8Zero:
00483   case NVPTXISD::Suld2DArrayV4I16Zero:
00484   case NVPTXISD::Suld2DArrayV4I32Zero:
00485   case NVPTXISD::Suld3DI8Zero:
00486   case NVPTXISD::Suld3DI16Zero:
00487   case NVPTXISD::Suld3DI32Zero:
00488   case NVPTXISD::Suld3DI64Zero:
00489   case NVPTXISD::Suld3DV2I8Zero:
00490   case NVPTXISD::Suld3DV2I16Zero:
00491   case NVPTXISD::Suld3DV2I32Zero:
00492   case NVPTXISD::Suld3DV2I64Zero:
00493   case NVPTXISD::Suld3DV4I8Zero:
00494   case NVPTXISD::Suld3DV4I16Zero:
00495   case NVPTXISD::Suld3DV4I32Zero:
00496     ResNode = SelectSurfaceIntrinsic(N);
00497     break;
00498   case ISD::AND:
00499   case ISD::SRA:
00500   case ISD::SRL:
00501     // Try to select BFE
00502     ResNode = SelectBFE(N);
00503     break;
00504   case ISD::ADDRSPACECAST:
00505     ResNode = SelectAddrSpaceCast(N);
00506     break;
00507   default:
00508     break;
00509   }
00510   if (ResNode)
00511     return ResNode;
00512   return SelectCode(N);
00513 }
00514 
00515 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
00516   unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
00517   switch (IID) {
00518   default:
00519     return NULL;
00520   case Intrinsic::nvvm_ldg_global_f:
00521   case Intrinsic::nvvm_ldg_global_i:
00522   case Intrinsic::nvvm_ldg_global_p:
00523   case Intrinsic::nvvm_ldu_global_f:
00524   case Intrinsic::nvvm_ldu_global_i:
00525   case Intrinsic::nvvm_ldu_global_p:
00526     return SelectLDGLDU(N);
00527   }
00528 }
00529 
00530 static unsigned int getCodeAddrSpace(MemSDNode *N) {
00531   const Value *Src = N->getMemOperand()->getValue();
00532 
00533   if (!Src)
00534     return NVPTX::PTXLdStInstCode::GENERIC;
00535 
00536   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
00537     switch (PT->getAddressSpace()) {
00538     case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
00539     case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
00540     case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
00541     case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
00542     case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
00543     case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
00544     default: break;
00545     }
00546   }
00547   return NVPTX::PTXLdStInstCode::GENERIC;
00548 }
00549 
00550 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
00551   unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
00552   switch (IID) {
00553   default:
00554     return nullptr;
00555   case Intrinsic::nvvm_texsurf_handle_internal:
00556     return SelectTexSurfHandle(N);
00557   }
00558 }
00559 
00560 SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
00561   // Op 0 is the intrinsic ID
00562   SDValue Wrapper = N->getOperand(1);
00563   SDValue GlobalVal = Wrapper.getOperand(0);
00564   return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
00565                                 GlobalVal);
00566 }
00567 
00568 SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
00569   SDValue Src = N->getOperand(0);
00570   AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
00571   unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
00572   unsigned DstAddrSpace = CastN->getDestAddressSpace();
00573 
00574   assert(SrcAddrSpace != DstAddrSpace &&
00575          "addrspacecast must be between different address spaces");
00576 
00577   if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
00578     // Specific to generic
00579     unsigned Opc;
00580     switch (SrcAddrSpace) {
00581     default: report_fatal_error("Bad address space in addrspacecast");
00582     case ADDRESS_SPACE_GLOBAL:
00583       Opc = TM.is64Bit() ? NVPTX::cvta_global_yes_64 : NVPTX::cvta_global_yes;
00584       break;
00585     case ADDRESS_SPACE_SHARED:
00586       Opc = TM.is64Bit() ? NVPTX::cvta_shared_yes_64 : NVPTX::cvta_shared_yes;
00587       break;
00588     case ADDRESS_SPACE_CONST:
00589       Opc = TM.is64Bit() ? NVPTX::cvta_const_yes_64 : NVPTX::cvta_const_yes;
00590       break;
00591     case ADDRESS_SPACE_LOCAL:
00592       Opc = TM.is64Bit() ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes;
00593       break;
00594     }
00595     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00596   } else {
00597     // Generic to specific
00598     if (SrcAddrSpace != 0)
00599       report_fatal_error("Cannot cast between two non-generic address spaces");
00600     unsigned Opc;
00601     switch (DstAddrSpace) {
00602     default: report_fatal_error("Bad address space in addrspacecast");
00603     case ADDRESS_SPACE_GLOBAL:
00604       Opc = TM.is64Bit() ? NVPTX::cvta_to_global_yes_64
00605                          : NVPTX::cvta_to_global_yes;
00606       break;
00607     case ADDRESS_SPACE_SHARED:
00608       Opc = TM.is64Bit() ? NVPTX::cvta_to_shared_yes_64
00609                          : NVPTX::cvta_to_shared_yes;
00610       break;
00611     case ADDRESS_SPACE_CONST:
00612       Opc =
00613           TM.is64Bit() ? NVPTX::cvta_to_const_yes_64 : NVPTX::cvta_to_const_yes;
00614       break;
00615     case ADDRESS_SPACE_LOCAL:
00616       Opc =
00617           TM.is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes;
00618       break;
00619     }
00620     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00621   }
00622 }
00623 
00624 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
00625   SDLoc dl(N);
00626   LoadSDNode *LD = cast<LoadSDNode>(N);
00627   EVT LoadedVT = LD->getMemoryVT();
00628   SDNode *NVPTXLD = nullptr;
00629 
00630   // do not support pre/post inc/dec
00631   if (LD->isIndexed())
00632     return nullptr;
00633 
00634   if (!LoadedVT.isSimple())
00635     return nullptr;
00636 
00637   // Address Space Setting
00638   unsigned int codeAddrSpace = getCodeAddrSpace(LD);
00639 
00640   // Volatile Setting
00641   // - .volatile is only availalble for .global and .shared
00642   bool isVolatile = LD->isVolatile();
00643   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00644       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00645       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00646     isVolatile = false;
00647 
00648   // Vector Setting
00649   MVT SimpleVT = LoadedVT.getSimpleVT();
00650   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
00651   if (SimpleVT.isVector()) {
00652     unsigned num = SimpleVT.getVectorNumElements();
00653     if (num == 2)
00654       vecType = NVPTX::PTXLdStInstCode::V2;
00655     else if (num == 4)
00656       vecType = NVPTX::PTXLdStInstCode::V4;
00657     else
00658       return nullptr;
00659   }
00660 
00661   // Type Setting: fromType + fromTypeWidth
00662   //
00663   // Sign   : ISD::SEXTLOAD
00664   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00665   //          type is integer
00666   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00667   MVT ScalarVT = SimpleVT.getScalarType();
00668   // Read at least 8 bits (predicates are stored as 8-bit values)
00669   unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00670   unsigned int fromType;
00671   if ((LD->getExtensionType() == ISD::SEXTLOAD))
00672     fromType = NVPTX::PTXLdStInstCode::Signed;
00673   else if (ScalarVT.isFloatingPoint())
00674     fromType = NVPTX::PTXLdStInstCode::Float;
00675   else
00676     fromType = NVPTX::PTXLdStInstCode::Unsigned;
00677 
00678   // Create the machine instruction DAG
00679   SDValue Chain = N->getOperand(0);
00680   SDValue N1 = N->getOperand(1);
00681   SDValue Addr;
00682   SDValue Offset, Base;
00683   unsigned Opcode;
00684   MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
00685 
00686   if (SelectDirectAddr(N1, Addr)) {
00687     switch (TargetVT) {
00688     case MVT::i8:
00689       Opcode = NVPTX::LD_i8_avar;
00690       break;
00691     case MVT::i16:
00692       Opcode = NVPTX::LD_i16_avar;
00693       break;
00694     case MVT::i32:
00695       Opcode = NVPTX::LD_i32_avar;
00696       break;
00697     case MVT::i64:
00698       Opcode = NVPTX::LD_i64_avar;
00699       break;
00700     case MVT::f32:
00701       Opcode = NVPTX::LD_f32_avar;
00702       break;
00703     case MVT::f64:
00704       Opcode = NVPTX::LD_f64_avar;
00705       break;
00706     default:
00707       return nullptr;
00708     }
00709     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00710                       getI32Imm(vecType), getI32Imm(fromType),
00711                       getI32Imm(fromTypeWidth), Addr, Chain };
00712     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00713   } else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
00714                           : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
00715     switch (TargetVT) {
00716     case MVT::i8:
00717       Opcode = NVPTX::LD_i8_asi;
00718       break;
00719     case MVT::i16:
00720       Opcode = NVPTX::LD_i16_asi;
00721       break;
00722     case MVT::i32:
00723       Opcode = NVPTX::LD_i32_asi;
00724       break;
00725     case MVT::i64:
00726       Opcode = NVPTX::LD_i64_asi;
00727       break;
00728     case MVT::f32:
00729       Opcode = NVPTX::LD_f32_asi;
00730       break;
00731     case MVT::f64:
00732       Opcode = NVPTX::LD_f64_asi;
00733       break;
00734     default:
00735       return nullptr;
00736     }
00737     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00738                       getI32Imm(vecType), getI32Imm(fromType),
00739                       getI32Imm(fromTypeWidth), Base, Offset, Chain };
00740     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00741   } else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
00742                           : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
00743     if (TM.is64Bit()) {
00744       switch (TargetVT) {
00745       case MVT::i8:
00746         Opcode = NVPTX::LD_i8_ari_64;
00747         break;
00748       case MVT::i16:
00749         Opcode = NVPTX::LD_i16_ari_64;
00750         break;
00751       case MVT::i32:
00752         Opcode = NVPTX::LD_i32_ari_64;
00753         break;
00754       case MVT::i64:
00755         Opcode = NVPTX::LD_i64_ari_64;
00756         break;
00757       case MVT::f32:
00758         Opcode = NVPTX::LD_f32_ari_64;
00759         break;
00760       case MVT::f64:
00761         Opcode = NVPTX::LD_f64_ari_64;
00762         break;
00763       default:
00764         return nullptr;
00765       }
00766     } else {
00767       switch (TargetVT) {
00768       case MVT::i8:
00769         Opcode = NVPTX::LD_i8_ari;
00770         break;
00771       case MVT::i16:
00772         Opcode = NVPTX::LD_i16_ari;
00773         break;
00774       case MVT::i32:
00775         Opcode = NVPTX::LD_i32_ari;
00776         break;
00777       case MVT::i64:
00778         Opcode = NVPTX::LD_i64_ari;
00779         break;
00780       case MVT::f32:
00781         Opcode = NVPTX::LD_f32_ari;
00782         break;
00783       case MVT::f64:
00784         Opcode = NVPTX::LD_f64_ari;
00785         break;
00786       default:
00787         return nullptr;
00788       }
00789     }
00790     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00791                       getI32Imm(vecType), getI32Imm(fromType),
00792                       getI32Imm(fromTypeWidth), Base, Offset, Chain };
00793     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00794   } else {
00795     if (TM.is64Bit()) {
00796       switch (TargetVT) {
00797       case MVT::i8:
00798         Opcode = NVPTX::LD_i8_areg_64;
00799         break;
00800       case MVT::i16:
00801         Opcode = NVPTX::LD_i16_areg_64;
00802         break;
00803       case MVT::i32:
00804         Opcode = NVPTX::LD_i32_areg_64;
00805         break;
00806       case MVT::i64:
00807         Opcode = NVPTX::LD_i64_areg_64;
00808         break;
00809       case MVT::f32:
00810         Opcode = NVPTX::LD_f32_areg_64;
00811         break;
00812       case MVT::f64:
00813         Opcode = NVPTX::LD_f64_areg_64;
00814         break;
00815       default:
00816         return nullptr;
00817       }
00818     } else {
00819       switch (TargetVT) {
00820       case MVT::i8:
00821         Opcode = NVPTX::LD_i8_areg;
00822         break;
00823       case MVT::i16:
00824         Opcode = NVPTX::LD_i16_areg;
00825         break;
00826       case MVT::i32:
00827         Opcode = NVPTX::LD_i32_areg;
00828         break;
00829       case MVT::i64:
00830         Opcode = NVPTX::LD_i64_areg;
00831         break;
00832       case MVT::f32:
00833         Opcode = NVPTX::LD_f32_areg;
00834         break;
00835       case MVT::f64:
00836         Opcode = NVPTX::LD_f64_areg;
00837         break;
00838       default:
00839         return nullptr;
00840       }
00841     }
00842     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00843                       getI32Imm(vecType), getI32Imm(fromType),
00844                       getI32Imm(fromTypeWidth), N1, Chain };
00845     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00846   }
00847 
00848   if (NVPTXLD) {
00849     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
00850     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
00851     cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
00852   }
00853 
00854   return NVPTXLD;
00855 }
00856 
00857 SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
00858 
00859   SDValue Chain = N->getOperand(0);
00860   SDValue Op1 = N->getOperand(1);
00861   SDValue Addr, Offset, Base;
00862   unsigned Opcode;
00863   SDLoc DL(N);
00864   SDNode *LD;
00865   MemSDNode *MemSD = cast<MemSDNode>(N);
00866   EVT LoadedVT = MemSD->getMemoryVT();
00867 
00868   if (!LoadedVT.isSimple())
00869     return nullptr;
00870 
00871   // Address Space Setting
00872   unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
00873 
00874   // Volatile Setting
00875   // - .volatile is only availalble for .global and .shared
00876   bool IsVolatile = MemSD->isVolatile();
00877   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00878       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00879       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00880     IsVolatile = false;
00881 
00882   // Vector Setting
00883   MVT SimpleVT = LoadedVT.getSimpleVT();
00884 
00885   // Type Setting: fromType + fromTypeWidth
00886   //
00887   // Sign   : ISD::SEXTLOAD
00888   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00889   //          type is integer
00890   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00891   MVT ScalarVT = SimpleVT.getScalarType();
00892   // Read at least 8 bits (predicates are stored as 8-bit values)
00893   unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00894   unsigned int FromType;
00895   // The last operand holds the original LoadSDNode::getExtensionType() value
00896   unsigned ExtensionType = cast<ConstantSDNode>(
00897       N->getOperand(N->getNumOperands() - 1))->getZExtValue();
00898   if (ExtensionType == ISD::SEXTLOAD)
00899     FromType = NVPTX::PTXLdStInstCode::Signed;
00900   else if (ScalarVT.isFloatingPoint())
00901     FromType = NVPTX::PTXLdStInstCode::Float;
00902   else
00903     FromType = NVPTX::PTXLdStInstCode::Unsigned;
00904 
00905   unsigned VecType;
00906 
00907   switch (N->getOpcode()) {
00908   case NVPTXISD::LoadV2:
00909     VecType = NVPTX::PTXLdStInstCode::V2;
00910     break;
00911   case NVPTXISD::LoadV4:
00912     VecType = NVPTX::PTXLdStInstCode::V4;
00913     break;
00914   default:
00915     return nullptr;
00916   }
00917 
00918   EVT EltVT = N->getValueType(0);
00919 
00920   if (SelectDirectAddr(Op1, Addr)) {
00921     switch (N->getOpcode()) {
00922     default:
00923       return nullptr;
00924     case NVPTXISD::LoadV2:
00925       switch (EltVT.getSimpleVT().SimpleTy) {
00926       default:
00927         return nullptr;
00928       case MVT::i8:
00929         Opcode = NVPTX::LDV_i8_v2_avar;
00930         break;
00931       case MVT::i16:
00932         Opcode = NVPTX::LDV_i16_v2_avar;
00933         break;
00934       case MVT::i32:
00935         Opcode = NVPTX::LDV_i32_v2_avar;
00936         break;
00937       case MVT::i64:
00938         Opcode = NVPTX::LDV_i64_v2_avar;
00939         break;
00940       case MVT::f32:
00941         Opcode = NVPTX::LDV_f32_v2_avar;
00942         break;
00943       case MVT::f64:
00944         Opcode = NVPTX::LDV_f64_v2_avar;
00945         break;
00946       }
00947       break;
00948     case NVPTXISD::LoadV4:
00949       switch (EltVT.getSimpleVT().SimpleTy) {
00950       default:
00951         return nullptr;
00952       case MVT::i8:
00953         Opcode = NVPTX::LDV_i8_v4_avar;
00954         break;
00955       case MVT::i16:
00956         Opcode = NVPTX::LDV_i16_v4_avar;
00957         break;
00958       case MVT::i32:
00959         Opcode = NVPTX::LDV_i32_v4_avar;
00960         break;
00961       case MVT::f32:
00962         Opcode = NVPTX::LDV_f32_v4_avar;
00963         break;
00964       }
00965       break;
00966     }
00967 
00968     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
00969                       getI32Imm(VecType), getI32Imm(FromType),
00970                       getI32Imm(FromTypeWidth), Addr, Chain };
00971     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
00972   } else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
00973                           : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
00974     switch (N->getOpcode()) {
00975     default:
00976       return nullptr;
00977     case NVPTXISD::LoadV2:
00978       switch (EltVT.getSimpleVT().SimpleTy) {
00979       default:
00980         return nullptr;
00981       case MVT::i8:
00982         Opcode = NVPTX::LDV_i8_v2_asi;
00983         break;
00984       case MVT::i16:
00985         Opcode = NVPTX::LDV_i16_v2_asi;
00986         break;
00987       case MVT::i32:
00988         Opcode = NVPTX::LDV_i32_v2_asi;
00989         break;
00990       case MVT::i64:
00991         Opcode = NVPTX::LDV_i64_v2_asi;
00992         break;
00993       case MVT::f32:
00994         Opcode = NVPTX::LDV_f32_v2_asi;
00995         break;
00996       case MVT::f64:
00997         Opcode = NVPTX::LDV_f64_v2_asi;
00998         break;
00999       }
01000       break;
01001     case NVPTXISD::LoadV4:
01002       switch (EltVT.getSimpleVT().SimpleTy) {
01003       default:
01004         return nullptr;
01005       case MVT::i8:
01006         Opcode = NVPTX::LDV_i8_v4_asi;
01007         break;
01008       case MVT::i16:
01009         Opcode = NVPTX::LDV_i16_v4_asi;
01010         break;
01011       case MVT::i32:
01012         Opcode = NVPTX::LDV_i32_v4_asi;
01013         break;
01014       case MVT::f32:
01015         Opcode = NVPTX::LDV_f32_v4_asi;
01016         break;
01017       }
01018       break;
01019     }
01020 
01021     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
01022                       getI32Imm(VecType), getI32Imm(FromType),
01023                       getI32Imm(FromTypeWidth), Base, Offset, Chain };
01024     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01025   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
01026                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
01027     if (TM.is64Bit()) {
01028       switch (N->getOpcode()) {
01029       default:
01030         return nullptr;
01031       case NVPTXISD::LoadV2:
01032         switch (EltVT.getSimpleVT().SimpleTy) {
01033         default:
01034           return nullptr;
01035         case MVT::i8:
01036           Opcode = NVPTX::LDV_i8_v2_ari_64;
01037           break;
01038         case MVT::i16:
01039           Opcode = NVPTX::LDV_i16_v2_ari_64;
01040           break;
01041         case MVT::i32:
01042           Opcode = NVPTX::LDV_i32_v2_ari_64;
01043           break;
01044         case MVT::i64:
01045           Opcode = NVPTX::LDV_i64_v2_ari_64;
01046           break;
01047         case MVT::f32:
01048           Opcode = NVPTX::LDV_f32_v2_ari_64;
01049           break;
01050         case MVT::f64:
01051           Opcode = NVPTX::LDV_f64_v2_ari_64;
01052           break;
01053         }
01054         break;
01055       case NVPTXISD::LoadV4:
01056         switch (EltVT.getSimpleVT().SimpleTy) {
01057         default:
01058           return nullptr;
01059         case MVT::i8:
01060           Opcode = NVPTX::LDV_i8_v4_ari_64;
01061           break;
01062         case MVT::i16:
01063           Opcode = NVPTX::LDV_i16_v4_ari_64;
01064           break;
01065         case MVT::i32:
01066           Opcode = NVPTX::LDV_i32_v4_ari_64;
01067           break;
01068         case MVT::f32:
01069           Opcode = NVPTX::LDV_f32_v4_ari_64;
01070           break;
01071         }
01072         break;
01073       }
01074     } else {
01075       switch (N->getOpcode()) {
01076       default:
01077         return nullptr;
01078       case NVPTXISD::LoadV2:
01079         switch (EltVT.getSimpleVT().SimpleTy) {
01080         default:
01081           return nullptr;
01082         case MVT::i8:
01083           Opcode = NVPTX::LDV_i8_v2_ari;
01084           break;
01085         case MVT::i16:
01086           Opcode = NVPTX::LDV_i16_v2_ari;
01087           break;
01088         case MVT::i32:
01089           Opcode = NVPTX::LDV_i32_v2_ari;
01090           break;
01091         case MVT::i64:
01092           Opcode = NVPTX::LDV_i64_v2_ari;
01093           break;
01094         case MVT::f32:
01095           Opcode = NVPTX::LDV_f32_v2_ari;
01096           break;
01097         case MVT::f64:
01098           Opcode = NVPTX::LDV_f64_v2_ari;
01099           break;
01100         }
01101         break;
01102       case NVPTXISD::LoadV4:
01103         switch (EltVT.getSimpleVT().SimpleTy) {
01104         default:
01105           return nullptr;
01106         case MVT::i8:
01107           Opcode = NVPTX::LDV_i8_v4_ari;
01108           break;
01109         case MVT::i16:
01110           Opcode = NVPTX::LDV_i16_v4_ari;
01111           break;
01112         case MVT::i32:
01113           Opcode = NVPTX::LDV_i32_v4_ari;
01114           break;
01115         case MVT::f32:
01116           Opcode = NVPTX::LDV_f32_v4_ari;
01117           break;
01118         }
01119         break;
01120       }
01121     }
01122 
01123     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
01124                       getI32Imm(VecType), getI32Imm(FromType),
01125                       getI32Imm(FromTypeWidth), Base, Offset, Chain };
01126 
01127     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01128   } else {
01129     if (TM.is64Bit()) {
01130       switch (N->getOpcode()) {
01131       default:
01132         return nullptr;
01133       case NVPTXISD::LoadV2:
01134         switch (EltVT.getSimpleVT().SimpleTy) {
01135         default:
01136           return nullptr;
01137         case MVT::i8:
01138           Opcode = NVPTX::LDV_i8_v2_areg_64;
01139           break;
01140         case MVT::i16:
01141           Opcode = NVPTX::LDV_i16_v2_areg_64;
01142           break;
01143         case MVT::i32:
01144           Opcode = NVPTX::LDV_i32_v2_areg_64;
01145           break;
01146         case MVT::i64:
01147           Opcode = NVPTX::LDV_i64_v2_areg_64;
01148           break;
01149         case MVT::f32:
01150           Opcode = NVPTX::LDV_f32_v2_areg_64;
01151           break;
01152         case MVT::f64:
01153           Opcode = NVPTX::LDV_f64_v2_areg_64;
01154           break;
01155         }
01156         break;
01157       case NVPTXISD::LoadV4:
01158         switch (EltVT.getSimpleVT().SimpleTy) {
01159         default:
01160           return nullptr;
01161         case MVT::i8:
01162           Opcode = NVPTX::LDV_i8_v4_areg_64;
01163           break;
01164         case MVT::i16:
01165           Opcode = NVPTX::LDV_i16_v4_areg_64;
01166           break;
01167         case MVT::i32:
01168           Opcode = NVPTX::LDV_i32_v4_areg_64;
01169           break;
01170         case MVT::f32:
01171           Opcode = NVPTX::LDV_f32_v4_areg_64;
01172           break;
01173         }
01174         break;
01175       }
01176     } else {
01177       switch (N->getOpcode()) {
01178       default:
01179         return nullptr;
01180       case NVPTXISD::LoadV2:
01181         switch (EltVT.getSimpleVT().SimpleTy) {
01182         default:
01183           return nullptr;
01184         case MVT::i8:
01185           Opcode = NVPTX::LDV_i8_v2_areg;
01186           break;
01187         case MVT::i16:
01188           Opcode = NVPTX::LDV_i16_v2_areg;
01189           break;
01190         case MVT::i32:
01191           Opcode = NVPTX::LDV_i32_v2_areg;
01192           break;
01193         case MVT::i64:
01194           Opcode = NVPTX::LDV_i64_v2_areg;
01195           break;
01196         case MVT::f32:
01197           Opcode = NVPTX::LDV_f32_v2_areg;
01198           break;
01199         case MVT::f64:
01200           Opcode = NVPTX::LDV_f64_v2_areg;
01201           break;
01202         }
01203         break;
01204       case NVPTXISD::LoadV4:
01205         switch (EltVT.getSimpleVT().SimpleTy) {
01206         default:
01207           return nullptr;
01208         case MVT::i8:
01209           Opcode = NVPTX::LDV_i8_v4_areg;
01210           break;
01211         case MVT::i16:
01212           Opcode = NVPTX::LDV_i16_v4_areg;
01213           break;
01214         case MVT::i32:
01215           Opcode = NVPTX::LDV_i32_v4_areg;
01216           break;
01217         case MVT::f32:
01218           Opcode = NVPTX::LDV_f32_v4_areg;
01219           break;
01220         }
01221         break;
01222       }
01223     }
01224 
01225     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
01226                       getI32Imm(VecType), getI32Imm(FromType),
01227                       getI32Imm(FromTypeWidth), Op1, Chain };
01228     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01229   }
01230 
01231   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01232   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
01233   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
01234 
01235   return LD;
01236 }
01237 
01238 SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
01239 
01240   SDValue Chain = N->getOperand(0);
01241   SDValue Op1;
01242   MemSDNode *Mem;
01243   bool IsLDG = true;
01244 
01245   // If this is an LDG intrinsic, the address is the third operand. Its its an
01246   // LDG/LDU SD node (from custom vector handling), then its the second operand
01247   if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
01248     Op1 = N->getOperand(2);
01249     Mem = cast<MemIntrinsicSDNode>(N);
01250     unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
01251     switch (IID) {
01252     default:
01253       return NULL;
01254     case Intrinsic::nvvm_ldg_global_f:
01255     case Intrinsic::nvvm_ldg_global_i:
01256     case Intrinsic::nvvm_ldg_global_p:
01257       IsLDG = true;
01258       break;
01259     case Intrinsic::nvvm_ldu_global_f:
01260     case Intrinsic::nvvm_ldu_global_i:
01261     case Intrinsic::nvvm_ldu_global_p:
01262       IsLDG = false;
01263       break;
01264     }
01265   } else {
01266     Op1 = N->getOperand(1);
01267     Mem = cast<MemSDNode>(N);
01268   }
01269 
01270   unsigned Opcode;
01271   SDLoc DL(N);
01272   SDNode *LD;
01273   SDValue Base, Offset, Addr;
01274 
01275   EVT EltVT = Mem->getMemoryVT();
01276   if (EltVT.isVector()) {
01277     EltVT = EltVT.getVectorElementType();
01278   }
01279 
01280   if (SelectDirectAddr(Op1, Addr)) {
01281     switch (N->getOpcode()) {
01282     default:
01283       return nullptr;
01284     case ISD::INTRINSIC_W_CHAIN:
01285       if (IsLDG) {
01286         switch (EltVT.getSimpleVT().SimpleTy) {
01287         default:
01288           return nullptr;
01289         case MVT::i8:
01290           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
01291           break;
01292         case MVT::i16:
01293           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
01294           break;
01295         case MVT::i32:
01296           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
01297           break;
01298         case MVT::i64:
01299           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
01300           break;
01301         case MVT::f32:
01302           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
01303           break;
01304         case MVT::f64:
01305           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
01306           break;
01307         }
01308       } else {
01309         switch (EltVT.getSimpleVT().SimpleTy) {
01310         default:
01311           return nullptr;
01312         case MVT::i8:
01313           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
01314           break;
01315         case MVT::i16:
01316           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
01317           break;
01318         case MVT::i32:
01319           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
01320           break;
01321         case MVT::i64:
01322           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
01323           break;
01324         case MVT::f32:
01325           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
01326           break;
01327         case MVT::f64:
01328           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
01329           break;
01330         }
01331       }
01332       break;
01333     case NVPTXISD::LDGV2:
01334       switch (EltVT.getSimpleVT().SimpleTy) {
01335       default:
01336         return nullptr;
01337       case MVT::i8:
01338         Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
01339         break;
01340       case MVT::i16:
01341         Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
01342         break;
01343       case MVT::i32:
01344         Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
01345         break;
01346       case MVT::i64:
01347         Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
01348         break;
01349       case MVT::f32:
01350         Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
01351         break;
01352       case MVT::f64:
01353         Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
01354         break;
01355       }
01356       break;
01357     case NVPTXISD::LDUV2:
01358       switch (EltVT.getSimpleVT().SimpleTy) {
01359       default:
01360         return nullptr;
01361       case MVT::i8:
01362         Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
01363         break;
01364       case MVT::i16:
01365         Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
01366         break;
01367       case MVT::i32:
01368         Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
01369         break;
01370       case MVT::i64:
01371         Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
01372         break;
01373       case MVT::f32:
01374         Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
01375         break;
01376       case MVT::f64:
01377         Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
01378         break;
01379       }
01380       break;
01381     case NVPTXISD::LDGV4:
01382       switch (EltVT.getSimpleVT().SimpleTy) {
01383       default:
01384         return nullptr;
01385       case MVT::i8:
01386         Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
01387         break;
01388       case MVT::i16:
01389         Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
01390         break;
01391       case MVT::i32:
01392         Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
01393         break;
01394       case MVT::f32:
01395         Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
01396         break;
01397       }
01398       break;
01399     case NVPTXISD::LDUV4:
01400       switch (EltVT.getSimpleVT().SimpleTy) {
01401       default:
01402         return nullptr;
01403       case MVT::i8:
01404         Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
01405         break;
01406       case MVT::i16:
01407         Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
01408         break;
01409       case MVT::i32:
01410         Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
01411         break;
01412       case MVT::f32:
01413         Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
01414         break;
01415       }
01416       break;
01417     }
01418 
01419     SDValue Ops[] = { Addr, Chain };
01420     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01421   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
01422                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
01423     if (TM.is64Bit()) {
01424       switch (N->getOpcode()) {
01425       default:
01426         return nullptr;
01427       case ISD::INTRINSIC_W_CHAIN:
01428         if (IsLDG) {
01429           switch (EltVT.getSimpleVT().SimpleTy) {
01430           default:
01431             return nullptr;
01432           case MVT::i8:
01433             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
01434             break;
01435           case MVT::i16:
01436             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
01437             break;
01438           case MVT::i32:
01439             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
01440             break;
01441           case MVT::i64:
01442             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
01443             break;
01444           case MVT::f32:
01445             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
01446             break;
01447           case MVT::f64:
01448             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
01449             break;
01450           }
01451         } else {
01452           switch (EltVT.getSimpleVT().SimpleTy) {
01453           default:
01454             return nullptr;
01455           case MVT::i8:
01456             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
01457             break;
01458           case MVT::i16:
01459             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
01460             break;
01461           case MVT::i32:
01462             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
01463             break;
01464           case MVT::i64:
01465             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
01466             break;
01467           case MVT::f32:
01468             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
01469             break;
01470           case MVT::f64:
01471             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
01472             break;
01473           }
01474         }
01475         break;
01476       case NVPTXISD::LDGV2:
01477         switch (EltVT.getSimpleVT().SimpleTy) {
01478         default:
01479           return nullptr;
01480         case MVT::i8:
01481           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
01482           break;
01483         case MVT::i16:
01484           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
01485           break;
01486         case MVT::i32:
01487           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
01488           break;
01489         case MVT::i64:
01490           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
01491           break;
01492         case MVT::f32:
01493           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
01494           break;
01495         case MVT::f64:
01496           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
01497           break;
01498         }
01499         break;
01500       case NVPTXISD::LDUV2:
01501         switch (EltVT.getSimpleVT().SimpleTy) {
01502         default:
01503           return nullptr;
01504         case MVT::i8:
01505           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
01506           break;
01507         case MVT::i16:
01508           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
01509           break;
01510         case MVT::i32:
01511           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
01512           break;
01513         case MVT::i64:
01514           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
01515           break;
01516         case MVT::f32:
01517           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
01518           break;
01519         case MVT::f64:
01520           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
01521           break;
01522         }
01523         break;
01524       case NVPTXISD::LDGV4:
01525         switch (EltVT.getSimpleVT().SimpleTy) {
01526         default:
01527           return nullptr;
01528         case MVT::i8:
01529           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
01530           break;
01531         case MVT::i16:
01532           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
01533           break;
01534         case MVT::i32:
01535           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
01536           break;
01537         case MVT::f32:
01538           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
01539           break;
01540         }
01541         break;
01542       case NVPTXISD::LDUV4:
01543         switch (EltVT.getSimpleVT().SimpleTy) {
01544         default:
01545           return nullptr;
01546         case MVT::i8:
01547           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
01548           break;
01549         case MVT::i16:
01550           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
01551           break;
01552         case MVT::i32:
01553           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
01554           break;
01555         case MVT::f32:
01556           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
01557           break;
01558         }
01559         break;
01560       }
01561     } else {
01562       switch (N->getOpcode()) {
01563       default:
01564         return nullptr;
01565       case ISD::INTRINSIC_W_CHAIN:
01566         if (IsLDG) {
01567           switch (EltVT.getSimpleVT().SimpleTy) {
01568           default:
01569             return nullptr;
01570           case MVT::i8:
01571             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
01572             break;
01573           case MVT::i16:
01574             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
01575             break;
01576           case MVT::i32:
01577             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
01578             break;
01579           case MVT::i64:
01580             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
01581             break;
01582           case MVT::f32:
01583             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
01584             break;
01585           case MVT::f64:
01586             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
01587             break;
01588           }
01589         } else {
01590           switch (EltVT.getSimpleVT().SimpleTy) {
01591           default:
01592             return nullptr;
01593           case MVT::i8:
01594             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
01595             break;
01596           case MVT::i16:
01597             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
01598             break;
01599           case MVT::i32:
01600             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
01601             break;
01602           case MVT::i64:
01603             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
01604             break;
01605           case MVT::f32:
01606             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
01607             break;
01608           case MVT::f64:
01609             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
01610             break;
01611           }
01612         }
01613         break;
01614       case NVPTXISD::LDGV2:
01615         switch (EltVT.getSimpleVT().SimpleTy) {
01616         default:
01617           return nullptr;
01618         case MVT::i8:
01619           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
01620           break;
01621         case MVT::i16:
01622           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
01623           break;
01624         case MVT::i32:
01625           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
01626           break;
01627         case MVT::i64:
01628           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
01629           break;
01630         case MVT::f32:
01631           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
01632           break;
01633         case MVT::f64:
01634           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
01635           break;
01636         }
01637         break;
01638       case NVPTXISD::LDUV2:
01639         switch (EltVT.getSimpleVT().SimpleTy) {
01640         default:
01641           return nullptr;
01642         case MVT::i8:
01643           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
01644           break;
01645         case MVT::i16:
01646           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
01647           break;
01648         case MVT::i32:
01649           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
01650           break;
01651         case MVT::i64:
01652           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
01653           break;
01654         case MVT::f32:
01655           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
01656           break;
01657         case MVT::f64:
01658           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
01659           break;
01660         }
01661         break;
01662       case NVPTXISD::LDGV4:
01663         switch (EltVT.getSimpleVT().SimpleTy) {
01664         default:
01665           return nullptr;
01666         case MVT::i8:
01667           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
01668           break;
01669         case MVT::i16:
01670           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
01671           break;
01672         case MVT::i32:
01673           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
01674           break;
01675         case MVT::f32:
01676           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
01677           break;
01678         }
01679         break;
01680       case NVPTXISD::LDUV4:
01681         switch (EltVT.getSimpleVT().SimpleTy) {
01682         default:
01683           return nullptr;
01684         case MVT::i8:
01685           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
01686           break;
01687         case MVT::i16:
01688           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
01689           break;
01690         case MVT::i32:
01691           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
01692           break;
01693         case MVT::f32:
01694           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
01695           break;
01696         }
01697         break;
01698       }
01699     }
01700 
01701     SDValue Ops[] = { Base, Offset, Chain };
01702 
01703     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01704   } else {
01705     if (TM.is64Bit()) {
01706       switch (N->getOpcode()) {
01707       default:
01708         return nullptr;
01709       case ISD::INTRINSIC_W_CHAIN:
01710         if (IsLDG) {
01711           switch (EltVT.getSimpleVT().SimpleTy) {
01712           default:
01713             return nullptr;
01714           case MVT::i8:
01715             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
01716             break;
01717           case MVT::i16:
01718             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
01719             break;
01720           case MVT::i32:
01721             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
01722             break;
01723           case MVT::i64:
01724             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
01725             break;
01726           case MVT::f32:
01727             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
01728             break;
01729           case MVT::f64:
01730             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
01731             break;
01732           }
01733         } else {
01734           switch (EltVT.getSimpleVT().SimpleTy) {
01735           default:
01736             return nullptr;
01737           case MVT::i8:
01738             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
01739             break;
01740           case MVT::i16:
01741             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
01742             break;
01743           case MVT::i32:
01744             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
01745             break;
01746           case MVT::i64:
01747             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
01748             break;
01749           case MVT::f32:
01750             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
01751             break;
01752           case MVT::f64:
01753             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
01754             break;
01755           }
01756         }
01757         break;
01758       case NVPTXISD::LDGV2:
01759         switch (EltVT.getSimpleVT().SimpleTy) {
01760         default:
01761           return nullptr;
01762         case MVT::i8:
01763           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
01764           break;
01765         case MVT::i16:
01766           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
01767           break;
01768         case MVT::i32:
01769           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
01770           break;
01771         case MVT::i64:
01772           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
01773           break;
01774         case MVT::f32:
01775           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
01776           break;
01777         case MVT::f64:
01778           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
01779           break;
01780         }
01781         break;
01782       case NVPTXISD::LDUV2:
01783         switch (EltVT.getSimpleVT().SimpleTy) {
01784         default:
01785           return nullptr;
01786         case MVT::i8:
01787           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
01788           break;
01789         case MVT::i16:
01790           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
01791           break;
01792         case MVT::i32:
01793           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
01794           break;
01795         case MVT::i64:
01796           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
01797           break;
01798         case MVT::f32:
01799           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
01800           break;
01801         case MVT::f64:
01802           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
01803           break;
01804         }
01805         break;
01806       case NVPTXISD::LDGV4:
01807         switch (EltVT.getSimpleVT().SimpleTy) {
01808         default:
01809           return nullptr;
01810         case MVT::i8:
01811           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
01812           break;
01813         case MVT::i16:
01814           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
01815           break;
01816         case MVT::i32:
01817           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
01818           break;
01819         case MVT::f32:
01820           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
01821           break;
01822         }
01823         break;
01824       case NVPTXISD::LDUV4:
01825         switch (EltVT.getSimpleVT().SimpleTy) {
01826         default:
01827           return nullptr;
01828         case MVT::i8:
01829           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
01830           break;
01831         case MVT::i16:
01832           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
01833           break;
01834         case MVT::i32:
01835           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
01836           break;
01837         case MVT::f32:
01838           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
01839           break;
01840         }
01841         break;
01842       }
01843     } else {
01844       switch (N->getOpcode()) {
01845       default:
01846         return nullptr;
01847       case ISD::INTRINSIC_W_CHAIN:
01848         if (IsLDG) {
01849           switch (EltVT.getSimpleVT().SimpleTy) {
01850           default:
01851             return nullptr;
01852           case MVT::i8:
01853             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
01854             break;
01855           case MVT::i16:
01856             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
01857             break;
01858           case MVT::i32:
01859             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
01860             break;
01861           case MVT::i64:
01862             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
01863             break;
01864           case MVT::f32:
01865             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
01866             break;
01867           case MVT::f64:
01868             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
01869             break;
01870           }
01871         } else {
01872           switch (EltVT.getSimpleVT().SimpleTy) {
01873           default:
01874             return nullptr;
01875           case MVT::i8:
01876             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
01877             break;
01878           case MVT::i16:
01879             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
01880             break;
01881           case MVT::i32:
01882             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
01883             break;
01884           case MVT::i64:
01885             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
01886             break;
01887           case MVT::f32:
01888             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
01889             break;
01890           case MVT::f64:
01891             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
01892             break;
01893           }
01894         }
01895         break;
01896       case NVPTXISD::LDGV2:
01897         switch (EltVT.getSimpleVT().SimpleTy) {
01898         default:
01899           return nullptr;
01900         case MVT::i8:
01901           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
01902           break;
01903         case MVT::i16:
01904           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
01905           break;
01906         case MVT::i32:
01907           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
01908           break;
01909         case MVT::i64:
01910           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
01911           break;
01912         case MVT::f32:
01913           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
01914           break;
01915         case MVT::f64:
01916           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
01917           break;
01918         }
01919         break;
01920       case NVPTXISD::LDUV2:
01921         switch (EltVT.getSimpleVT().SimpleTy) {
01922         default:
01923           return nullptr;
01924         case MVT::i8:
01925           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
01926           break;
01927         case MVT::i16:
01928           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
01929           break;
01930         case MVT::i32:
01931           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
01932           break;
01933         case MVT::i64:
01934           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
01935           break;
01936         case MVT::f32:
01937           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
01938           break;
01939         case MVT::f64:
01940           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
01941           break;
01942         }
01943         break;
01944       case NVPTXISD::LDGV4:
01945         switch (EltVT.getSimpleVT().SimpleTy) {
01946         default:
01947           return nullptr;
01948         case MVT::i8:
01949           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
01950           break;
01951         case MVT::i16:
01952           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
01953           break;
01954         case MVT::i32:
01955           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
01956           break;
01957         case MVT::f32:
01958           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
01959           break;
01960         }
01961         break;
01962       case NVPTXISD::LDUV4:
01963         switch (EltVT.getSimpleVT().SimpleTy) {
01964         default:
01965           return nullptr;
01966         case MVT::i8:
01967           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
01968           break;
01969         case MVT::i16:
01970           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
01971           break;
01972         case MVT::i32:
01973           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
01974           break;
01975         case MVT::f32:
01976           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
01977           break;
01978         }
01979         break;
01980       }
01981     }
01982 
01983     SDValue Ops[] = { Op1, Chain };
01984     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01985   }
01986 
01987   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01988   MemRefs0[0] = Mem->getMemOperand();
01989   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
01990 
01991   return LD;
01992 }
01993 
01994 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
01995   SDLoc dl(N);
01996   StoreSDNode *ST = cast<StoreSDNode>(N);
01997   EVT StoreVT = ST->getMemoryVT();
01998   SDNode *NVPTXST = nullptr;
01999 
02000   // do not support pre/post inc/dec
02001   if (ST->isIndexed())
02002     return nullptr;
02003 
02004   if (!StoreVT.isSimple())
02005     return nullptr;
02006 
02007   // Address Space Setting
02008   unsigned int codeAddrSpace = getCodeAddrSpace(ST);
02009 
02010   // Volatile Setting
02011   // - .volatile is only availalble for .global and .shared
02012   bool isVolatile = ST->isVolatile();
02013   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
02014       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
02015       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
02016     isVolatile = false;
02017 
02018   // Vector Setting
02019   MVT SimpleVT = StoreVT.getSimpleVT();
02020   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
02021   if (SimpleVT.isVector()) {
02022     unsigned num = SimpleVT.getVectorNumElements();
02023     if (num == 2)
02024       vecType = NVPTX::PTXLdStInstCode::V2;
02025     else if (num == 4)
02026       vecType = NVPTX::PTXLdStInstCode::V4;
02027     else
02028       return nullptr;
02029   }
02030 
02031   // Type Setting: toType + toTypeWidth
02032   // - for integer type, always use 'u'
02033   //
02034   MVT ScalarVT = SimpleVT.getScalarType();
02035   unsigned toTypeWidth = ScalarVT.getSizeInBits();
02036   unsigned int toType;
02037   if (ScalarVT.isFloatingPoint())
02038     toType = NVPTX::PTXLdStInstCode::Float;
02039   else
02040     toType = NVPTX::PTXLdStInstCode::Unsigned;
02041 
02042   // Create the machine instruction DAG
02043   SDValue Chain = N->getOperand(0);
02044   SDValue N1 = N->getOperand(1);
02045   SDValue N2 = N->getOperand(2);
02046   SDValue Addr;
02047   SDValue Offset, Base;
02048   unsigned Opcode;
02049   MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
02050 
02051   if (SelectDirectAddr(N2, Addr)) {
02052     switch (SourceVT) {
02053     case MVT::i8:
02054       Opcode = NVPTX::ST_i8_avar;
02055       break;
02056     case MVT::i16:
02057       Opcode = NVPTX::ST_i16_avar;
02058       break;
02059     case MVT::i32:
02060       Opcode = NVPTX::ST_i32_avar;
02061       break;
02062     case MVT::i64:
02063       Opcode = NVPTX::ST_i64_avar;
02064       break;
02065     case MVT::f32:
02066       Opcode = NVPTX::ST_f32_avar;
02067       break;
02068     case MVT::f64:
02069       Opcode = NVPTX::ST_f64_avar;
02070       break;
02071     default:
02072       return nullptr;
02073     }
02074     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
02075                       getI32Imm(vecType), getI32Imm(toType),
02076                       getI32Imm(toTypeWidth), Addr, Chain };
02077     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02078   } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
02079                           : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
02080     switch (SourceVT) {
02081     case MVT::i8:
02082       Opcode = NVPTX::ST_i8_asi;
02083       break;
02084     case MVT::i16:
02085       Opcode = NVPTX::ST_i16_asi;
02086       break;
02087     case MVT::i32:
02088       Opcode = NVPTX::ST_i32_asi;
02089       break;
02090     case MVT::i64:
02091       Opcode = NVPTX::ST_i64_asi;
02092       break;
02093     case MVT::f32:
02094       Opcode = NVPTX::ST_f32_asi;
02095       break;
02096     case MVT::f64:
02097       Opcode = NVPTX::ST_f64_asi;
02098       break;
02099     default:
02100       return nullptr;
02101     }
02102     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
02103                       getI32Imm(vecType), getI32Imm(toType),
02104                       getI32Imm(toTypeWidth), Base, Offset, Chain };
02105     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02106   } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
02107                           : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
02108     if (TM.is64Bit()) {
02109       switch (SourceVT) {
02110       case MVT::i8:
02111         Opcode = NVPTX::ST_i8_ari_64;
02112         break;
02113       case MVT::i16:
02114         Opcode = NVPTX::ST_i16_ari_64;
02115         break;
02116       case MVT::i32:
02117         Opcode = NVPTX::ST_i32_ari_64;
02118         break;
02119       case MVT::i64:
02120         Opcode = NVPTX::ST_i64_ari_64;
02121         break;
02122       case MVT::f32:
02123         Opcode = NVPTX::ST_f32_ari_64;
02124         break;
02125       case MVT::f64:
02126         Opcode = NVPTX::ST_f64_ari_64;
02127         break;
02128       default:
02129         return nullptr;
02130       }
02131     } else {
02132       switch (SourceVT) {
02133       case MVT::i8:
02134         Opcode = NVPTX::ST_i8_ari;
02135         break;
02136       case MVT::i16:
02137         Opcode = NVPTX::ST_i16_ari;
02138         break;
02139       case MVT::i32:
02140         Opcode = NVPTX::ST_i32_ari;
02141         break;
02142       case MVT::i64:
02143         Opcode = NVPTX::ST_i64_ari;
02144         break;
02145       case MVT::f32:
02146         Opcode = NVPTX::ST_f32_ari;
02147         break;
02148       case MVT::f64:
02149         Opcode = NVPTX::ST_f64_ari;
02150         break;
02151       default:
02152         return nullptr;
02153       }
02154     }
02155     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
02156                       getI32Imm(vecType), getI32Imm(toType),
02157                       getI32Imm(toTypeWidth), Base, Offset, Chain };
02158     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02159   } else {
02160     if (TM.is64Bit()) {
02161       switch (SourceVT) {
02162       case MVT::i8:
02163         Opcode = NVPTX::ST_i8_areg_64;
02164         break;
02165       case MVT::i16:
02166         Opcode = NVPTX::ST_i16_areg_64;
02167         break;
02168       case MVT::i32:
02169         Opcode = NVPTX::ST_i32_areg_64;
02170         break;
02171       case MVT::i64:
02172         Opcode = NVPTX::ST_i64_areg_64;
02173         break;
02174       case MVT::f32:
02175         Opcode = NVPTX::ST_f32_areg_64;
02176         break;
02177       case MVT::f64:
02178         Opcode = NVPTX::ST_f64_areg_64;
02179         break;
02180       default:
02181         return nullptr;
02182       }
02183     } else {
02184       switch (SourceVT) {
02185       case MVT::i8:
02186         Opcode = NVPTX::ST_i8_areg;
02187         break;
02188       case MVT::i16:
02189         Opcode = NVPTX::ST_i16_areg;
02190         break;
02191       case MVT::i32:
02192         Opcode = NVPTX::ST_i32_areg;
02193         break;
02194       case MVT::i64:
02195         Opcode = NVPTX::ST_i64_areg;
02196         break;
02197       case MVT::f32:
02198         Opcode = NVPTX::ST_f32_areg;
02199         break;
02200       case MVT::f64:
02201         Opcode = NVPTX::ST_f64_areg;
02202         break;
02203       default:
02204         return nullptr;
02205       }
02206     }
02207     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
02208                       getI32Imm(vecType), getI32Imm(toType),
02209                       getI32Imm(toTypeWidth), N2, Chain };
02210     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02211   }
02212 
02213   if (NVPTXST) {
02214     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02215     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02216     cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
02217   }
02218 
02219   return NVPTXST;
02220 }
02221 
02222 SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
02223   SDValue Chain = N->getOperand(0);
02224   SDValue Op1 = N->getOperand(1);
02225   SDValue Addr, Offset, Base;
02226   unsigned Opcode;
02227   SDLoc DL(N);
02228   SDNode *ST;
02229   EVT EltVT = Op1.getValueType();
02230   MemSDNode *MemSD = cast<MemSDNode>(N);
02231   EVT StoreVT = MemSD->getMemoryVT();
02232 
02233   // Address Space Setting
02234   unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
02235 
02236   if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
02237     report_fatal_error("Cannot store to pointer that points to constant "
02238                        "memory space");
02239   }
02240 
02241   // Volatile Setting
02242   // - .volatile is only availalble for .global and .shared
02243   bool IsVolatile = MemSD->isVolatile();
02244   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
02245       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
02246       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
02247     IsVolatile = false;
02248 
02249   // Type Setting: toType + toTypeWidth
02250   // - for integer type, always use 'u'
02251   assert(StoreVT.isSimple() && "Store value is not simple");
02252   MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
02253   unsigned ToTypeWidth = ScalarVT.getSizeInBits();
02254   unsigned ToType;
02255   if (ScalarVT.isFloatingPoint())
02256     ToType = NVPTX::PTXLdStInstCode::Float;
02257   else
02258     ToType = NVPTX::PTXLdStInstCode::Unsigned;
02259 
02260   SmallVector<SDValue, 12> StOps;
02261   SDValue N2;
02262   unsigned VecType;
02263 
02264   switch (N->getOpcode()) {
02265   case NVPTXISD::StoreV2:
02266     VecType = NVPTX::PTXLdStInstCode::V2;
02267     StOps.push_back(N->getOperand(1));
02268     StOps.push_back(N->getOperand(2));
02269     N2 = N->getOperand(3);
02270     break;
02271   case NVPTXISD::StoreV4:
02272     VecType = NVPTX::PTXLdStInstCode::V4;
02273     StOps.push_back(N->getOperand(1));
02274     StOps.push_back(N->getOperand(2));
02275     StOps.push_back(N->getOperand(3));
02276     StOps.push_back(N->getOperand(4));
02277     N2 = N->getOperand(5);
02278     break;
02279   default:
02280     return nullptr;
02281   }
02282 
02283   StOps.push_back(getI32Imm(IsVolatile));
02284   StOps.push_back(getI32Imm(CodeAddrSpace));
02285   StOps.push_back(getI32Imm(VecType));
02286   StOps.push_back(getI32Imm(ToType));
02287   StOps.push_back(getI32Imm(ToTypeWidth));
02288 
02289   if (SelectDirectAddr(N2, Addr)) {
02290     switch (N->getOpcode()) {
02291     default:
02292       return nullptr;
02293     case NVPTXISD::StoreV2:
02294       switch (EltVT.getSimpleVT().SimpleTy) {
02295       default:
02296         return nullptr;
02297       case MVT::i8:
02298         Opcode = NVPTX::STV_i8_v2_avar;
02299         break;
02300       case MVT::i16:
02301         Opcode = NVPTX::STV_i16_v2_avar;
02302         break;
02303       case MVT::i32:
02304         Opcode = NVPTX::STV_i32_v2_avar;
02305         break;
02306       case MVT::i64:
02307         Opcode = NVPTX::STV_i64_v2_avar;
02308         break;
02309       case MVT::f32:
02310         Opcode = NVPTX::STV_f32_v2_avar;
02311         break;
02312       case MVT::f64:
02313         Opcode = NVPTX::STV_f64_v2_avar;
02314         break;
02315       }
02316       break;
02317     case NVPTXISD::StoreV4:
02318       switch (EltVT.getSimpleVT().SimpleTy) {
02319       default:
02320         return nullptr;
02321       case MVT::i8:
02322         Opcode = NVPTX::STV_i8_v4_avar;
02323         break;
02324       case MVT::i16:
02325         Opcode = NVPTX::STV_i16_v4_avar;
02326         break;
02327       case MVT::i32:
02328         Opcode = NVPTX::STV_i32_v4_avar;
02329         break;
02330       case MVT::f32:
02331         Opcode = NVPTX::STV_f32_v4_avar;
02332         break;
02333       }
02334       break;
02335     }
02336     StOps.push_back(Addr);
02337   } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
02338                           : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
02339     switch (N->getOpcode()) {
02340     default:
02341       return nullptr;
02342     case NVPTXISD::StoreV2:
02343       switch (EltVT.getSimpleVT().SimpleTy) {
02344       default:
02345         return nullptr;
02346       case MVT::i8:
02347         Opcode = NVPTX::STV_i8_v2_asi;
02348         break;
02349       case MVT::i16:
02350         Opcode = NVPTX::STV_i16_v2_asi;
02351         break;
02352       case MVT::i32:
02353         Opcode = NVPTX::STV_i32_v2_asi;
02354         break;
02355       case MVT::i64:
02356         Opcode = NVPTX::STV_i64_v2_asi;
02357         break;
02358       case MVT::f32:
02359         Opcode = NVPTX::STV_f32_v2_asi;
02360         break;
02361       case MVT::f64:
02362         Opcode = NVPTX::STV_f64_v2_asi;
02363         break;
02364       }
02365       break;
02366     case NVPTXISD::StoreV4:
02367       switch (EltVT.getSimpleVT().SimpleTy) {
02368       default:
02369         return nullptr;
02370       case MVT::i8:
02371         Opcode = NVPTX::STV_i8_v4_asi;
02372         break;
02373       case MVT::i16:
02374         Opcode = NVPTX::STV_i16_v4_asi;
02375         break;
02376       case MVT::i32:
02377         Opcode = NVPTX::STV_i32_v4_asi;
02378         break;
02379       case MVT::f32:
02380         Opcode = NVPTX::STV_f32_v4_asi;
02381         break;
02382       }
02383       break;
02384     }
02385     StOps.push_back(Base);
02386     StOps.push_back(Offset);
02387   } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
02388                           : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
02389     if (TM.is64Bit()) {
02390       switch (N->getOpcode()) {
02391       default:
02392         return nullptr;
02393       case NVPTXISD::StoreV2:
02394         switch (EltVT.getSimpleVT().SimpleTy) {
02395         default:
02396           return nullptr;
02397         case MVT::i8:
02398           Opcode = NVPTX::STV_i8_v2_ari_64;
02399           break;
02400         case MVT::i16:
02401           Opcode = NVPTX::STV_i16_v2_ari_64;
02402           break;
02403         case MVT::i32:
02404           Opcode = NVPTX::STV_i32_v2_ari_64;
02405           break;
02406         case MVT::i64:
02407           Opcode = NVPTX::STV_i64_v2_ari_64;
02408           break;
02409         case MVT::f32:
02410           Opcode = NVPTX::STV_f32_v2_ari_64;
02411           break;
02412         case MVT::f64:
02413           Opcode = NVPTX::STV_f64_v2_ari_64;
02414           break;
02415         }
02416         break;
02417       case NVPTXISD::StoreV4:
02418         switch (EltVT.getSimpleVT().SimpleTy) {
02419         default:
02420           return nullptr;
02421         case MVT::i8:
02422           Opcode = NVPTX::STV_i8_v4_ari_64;
02423           break;
02424         case MVT::i16:
02425           Opcode = NVPTX::STV_i16_v4_ari_64;
02426           break;
02427         case MVT::i32:
02428           Opcode = NVPTX::STV_i32_v4_ari_64;
02429           break;
02430         case MVT::f32:
02431           Opcode = NVPTX::STV_f32_v4_ari_64;
02432           break;
02433         }
02434         break;
02435       }
02436     } else {
02437       switch (N->getOpcode()) {
02438       default:
02439         return nullptr;
02440       case NVPTXISD::StoreV2:
02441         switch (EltVT.getSimpleVT().SimpleTy) {
02442         default:
02443           return nullptr;
02444         case MVT::i8:
02445           Opcode = NVPTX::STV_i8_v2_ari;
02446           break;
02447         case MVT::i16:
02448           Opcode = NVPTX::STV_i16_v2_ari;
02449           break;
02450         case MVT::i32:
02451           Opcode = NVPTX::STV_i32_v2_ari;
02452           break;
02453         case MVT::i64:
02454           Opcode = NVPTX::STV_i64_v2_ari;
02455           break;
02456         case MVT::f32:
02457           Opcode = NVPTX::STV_f32_v2_ari;
02458           break;
02459         case MVT::f64:
02460           Opcode = NVPTX::STV_f64_v2_ari;
02461           break;
02462         }
02463         break;
02464       case NVPTXISD::StoreV4:
02465         switch (EltVT.getSimpleVT().SimpleTy) {
02466         default:
02467           return nullptr;
02468         case MVT::i8:
02469           Opcode = NVPTX::STV_i8_v4_ari;
02470           break;
02471         case MVT::i16:
02472           Opcode = NVPTX::STV_i16_v4_ari;
02473           break;
02474         case MVT::i32:
02475           Opcode = NVPTX::STV_i32_v4_ari;
02476           break;
02477         case MVT::f32:
02478           Opcode = NVPTX::STV_f32_v4_ari;
02479           break;
02480         }
02481         break;
02482       }
02483     }
02484     StOps.push_back(Base);
02485     StOps.push_back(Offset);
02486   } else {
02487     if (TM.is64Bit()) {
02488       switch (N->getOpcode()) {
02489       default:
02490         return nullptr;
02491       case NVPTXISD::StoreV2:
02492         switch (EltVT.getSimpleVT().SimpleTy) {
02493         default:
02494           return nullptr;
02495         case MVT::i8:
02496           Opcode = NVPTX::STV_i8_v2_areg_64;
02497           break;
02498         case MVT::i16:
02499           Opcode = NVPTX::STV_i16_v2_areg_64;
02500           break;
02501         case MVT::i32:
02502           Opcode = NVPTX::STV_i32_v2_areg_64;
02503           break;
02504         case MVT::i64:
02505           Opcode = NVPTX::STV_i64_v2_areg_64;
02506           break;
02507         case MVT::f32:
02508           Opcode = NVPTX::STV_f32_v2_areg_64;
02509           break;
02510         case MVT::f64:
02511           Opcode = NVPTX::STV_f64_v2_areg_64;
02512           break;
02513         }
02514         break;
02515       case NVPTXISD::StoreV4:
02516         switch (EltVT.getSimpleVT().SimpleTy) {
02517         default:
02518           return nullptr;
02519         case MVT::i8:
02520           Opcode = NVPTX::STV_i8_v4_areg_64;
02521           break;
02522         case MVT::i16:
02523           Opcode = NVPTX::STV_i16_v4_areg_64;
02524           break;
02525         case MVT::i32:
02526           Opcode = NVPTX::STV_i32_v4_areg_64;
02527           break;
02528         case MVT::f32:
02529           Opcode = NVPTX::STV_f32_v4_areg_64;
02530           break;
02531         }
02532         break;
02533       }
02534     } else {
02535       switch (N->getOpcode()) {
02536       default:
02537         return nullptr;
02538       case NVPTXISD::StoreV2:
02539         switch (EltVT.getSimpleVT().SimpleTy) {
02540         default:
02541           return nullptr;
02542         case MVT::i8:
02543           Opcode = NVPTX::STV_i8_v2_areg;
02544           break;
02545         case MVT::i16:
02546           Opcode = NVPTX::STV_i16_v2_areg;
02547           break;
02548         case MVT::i32:
02549           Opcode = NVPTX::STV_i32_v2_areg;
02550           break;
02551         case MVT::i64:
02552           Opcode = NVPTX::STV_i64_v2_areg;
02553           break;
02554         case MVT::f32:
02555           Opcode = NVPTX::STV_f32_v2_areg;
02556           break;
02557         case MVT::f64:
02558           Opcode = NVPTX::STV_f64_v2_areg;
02559           break;
02560         }
02561         break;
02562       case NVPTXISD::StoreV4:
02563         switch (EltVT.getSimpleVT().SimpleTy) {
02564         default:
02565           return nullptr;
02566         case MVT::i8:
02567           Opcode = NVPTX::STV_i8_v4_areg;
02568           break;
02569         case MVT::i16:
02570           Opcode = NVPTX::STV_i16_v4_areg;
02571           break;
02572         case MVT::i32:
02573           Opcode = NVPTX::STV_i32_v4_areg;
02574           break;
02575         case MVT::f32:
02576           Opcode = NVPTX::STV_f32_v4_areg;
02577           break;
02578         }
02579         break;
02580       }
02581     }
02582     StOps.push_back(N2);
02583   }
02584 
02585   StOps.push_back(Chain);
02586 
02587   ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
02588 
02589   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02590   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02591   cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
02592 
02593   return ST;
02594 }
02595 
02596 SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
02597   SDValue Chain = Node->getOperand(0);
02598   SDValue Offset = Node->getOperand(2);
02599   SDValue Flag = Node->getOperand(3);
02600   SDLoc DL(Node);
02601   MemSDNode *Mem = cast<MemSDNode>(Node);
02602 
02603   unsigned VecSize;
02604   switch (Node->getOpcode()) {
02605   default:
02606     return nullptr;
02607   case NVPTXISD::LoadParam:
02608     VecSize = 1;
02609     break;
02610   case NVPTXISD::LoadParamV2:
02611     VecSize = 2;
02612     break;
02613   case NVPTXISD::LoadParamV4:
02614     VecSize = 4;
02615     break;
02616   }
02617 
02618   EVT EltVT = Node->getValueType(0);
02619   EVT MemVT = Mem->getMemoryVT();
02620 
02621   unsigned Opc = 0;
02622 
02623   switch (VecSize) {
02624   default:
02625     return nullptr;
02626   case 1:
02627     switch (MemVT.getSimpleVT().SimpleTy) {
02628     default:
02629       return nullptr;
02630     case MVT::i1:
02631       Opc = NVPTX::LoadParamMemI8;
02632       break;
02633     case MVT::i8:
02634       Opc = NVPTX::LoadParamMemI8;
02635       break;
02636     case MVT::i16:
02637       Opc = NVPTX::LoadParamMemI16;
02638       break;
02639     case MVT::i32:
02640       Opc = NVPTX::LoadParamMemI32;
02641       break;
02642     case MVT::i64:
02643       Opc = NVPTX::LoadParamMemI64;
02644       break;
02645     case MVT::f32:
02646       Opc = NVPTX::LoadParamMemF32;
02647       break;
02648     case MVT::f64:
02649       Opc = NVPTX::LoadParamMemF64;
02650       break;
02651     }
02652     break;
02653   case 2:
02654     switch (MemVT.getSimpleVT().SimpleTy) {
02655     default:
02656       return nullptr;
02657     case MVT::i1:
02658       Opc = NVPTX::LoadParamMemV2I8;
02659       break;
02660     case MVT::i8:
02661       Opc = NVPTX::LoadParamMemV2I8;
02662       break;
02663     case MVT::i16:
02664       Opc = NVPTX::LoadParamMemV2I16;
02665       break;
02666     case MVT::i32:
02667       Opc = NVPTX::LoadParamMemV2I32;
02668       break;
02669     case MVT::i64:
02670       Opc = NVPTX::LoadParamMemV2I64;
02671       break;
02672     case MVT::f32:
02673       Opc = NVPTX::LoadParamMemV2F32;
02674       break;
02675     case MVT::f64:
02676       Opc = NVPTX::LoadParamMemV2F64;
02677       break;
02678     }
02679     break;
02680   case 4:
02681     switch (MemVT.getSimpleVT().SimpleTy) {
02682     default:
02683       return nullptr;
02684     case MVT::i1:
02685       Opc = NVPTX::LoadParamMemV4I8;
02686       break;
02687     case MVT::i8:
02688       Opc = NVPTX::LoadParamMemV4I8;
02689       break;
02690     case MVT::i16:
02691       Opc = NVPTX::LoadParamMemV4I16;
02692       break;
02693     case MVT::i32:
02694       Opc = NVPTX::LoadParamMemV4I32;
02695       break;
02696     case MVT::f32:
02697       Opc = NVPTX::LoadParamMemV4F32;
02698       break;
02699     }
02700     break;
02701   }
02702 
02703   SDVTList VTs;
02704   if (VecSize == 1) {
02705     VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
02706   } else if (VecSize == 2) {
02707     VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
02708   } else {
02709     EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
02710     VTs = CurDAG->getVTList(EVTs);
02711   }
02712 
02713   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02714 
02715   SmallVector<SDValue, 2> Ops;
02716   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
02717   Ops.push_back(Chain);
02718   Ops.push_back(Flag);
02719 
02720   SDNode *Ret =
02721       CurDAG->getMachineNode(Opc, DL, VTs, Ops);
02722   return Ret;
02723 }
02724 
02725 SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
02726   SDLoc DL(N);
02727   SDValue Chain = N->getOperand(0);
02728   SDValue Offset = N->getOperand(1);
02729   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02730   MemSDNode *Mem = cast<MemSDNode>(N);
02731 
02732   // How many elements do we have?
02733   unsigned NumElts = 1;
02734   switch (N->getOpcode()) {
02735   default:
02736     return nullptr;
02737   case NVPTXISD::StoreRetval:
02738     NumElts = 1;
02739     break;
02740   case NVPTXISD::StoreRetvalV2:
02741     NumElts = 2;
02742     break;
02743   case NVPTXISD::StoreRetvalV4:
02744     NumElts = 4;
02745     break;
02746   }
02747 
02748   // Build vector of operands
02749   SmallVector<SDValue, 6> Ops;
02750   for (unsigned i = 0; i < NumElts; ++i)
02751     Ops.push_back(N->getOperand(i + 2));
02752   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
02753   Ops.push_back(Chain);
02754 
02755   // Determine target opcode
02756   // If we have an i1, use an 8-bit store. The lowering code in
02757   // NVPTXISelLowering will have already emitted an upcast.
02758   unsigned Opcode = 0;
02759   switch (NumElts) {
02760   default:
02761     return nullptr;
02762   case 1:
02763     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02764     default:
02765       return nullptr;
02766     case MVT::i1:
02767       Opcode = NVPTX::StoreRetvalI8;
02768       break;
02769     case MVT::i8:
02770       Opcode = NVPTX::StoreRetvalI8;
02771       break;
02772     case MVT::i16:
02773       Opcode = NVPTX::StoreRetvalI16;
02774       break;
02775     case MVT::i32:
02776       Opcode = NVPTX::StoreRetvalI32;
02777       break;
02778     case MVT::i64:
02779       Opcode = NVPTX::StoreRetvalI64;
02780       break;
02781     case MVT::f32:
02782       Opcode = NVPTX::StoreRetvalF32;
02783       break;
02784     case MVT::f64:
02785       Opcode = NVPTX::StoreRetvalF64;
02786       break;
02787     }
02788     break;
02789   case 2:
02790     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02791     default:
02792       return nullptr;
02793     case MVT::i1:
02794       Opcode = NVPTX::StoreRetvalV2I8;
02795       break;
02796     case MVT::i8:
02797       Opcode = NVPTX::StoreRetvalV2I8;
02798       break;
02799     case MVT::i16:
02800       Opcode = NVPTX::StoreRetvalV2I16;
02801       break;
02802     case MVT::i32:
02803       Opcode = NVPTX::StoreRetvalV2I32;
02804       break;
02805     case MVT::i64:
02806       Opcode = NVPTX::StoreRetvalV2I64;
02807       break;
02808     case MVT::f32:
02809       Opcode = NVPTX::StoreRetvalV2F32;
02810       break;
02811     case MVT::f64:
02812       Opcode = NVPTX::StoreRetvalV2F64;
02813       break;
02814     }
02815     break;
02816   case 4:
02817     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02818     default:
02819       return nullptr;
02820     case MVT::i1:
02821       Opcode = NVPTX::StoreRetvalV4I8;
02822       break;
02823     case MVT::i8:
02824       Opcode = NVPTX::StoreRetvalV4I8;
02825       break;
02826     case MVT::i16:
02827       Opcode = NVPTX::StoreRetvalV4I16;
02828       break;
02829     case MVT::i32:
02830       Opcode = NVPTX::StoreRetvalV4I32;
02831       break;
02832     case MVT::f32:
02833       Opcode = NVPTX::StoreRetvalV4F32;
02834       break;
02835     }
02836     break;
02837   }
02838 
02839   SDNode *Ret =
02840       CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
02841   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02842   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02843   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
02844 
02845   return Ret;
02846 }
02847 
02848 SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
02849   SDLoc DL(N);
02850   SDValue Chain = N->getOperand(0);
02851   SDValue Param = N->getOperand(1);
02852   unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
02853   SDValue Offset = N->getOperand(2);
02854   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02855   MemSDNode *Mem = cast<MemSDNode>(N);
02856   SDValue Flag = N->getOperand(N->getNumOperands() - 1);
02857 
02858   // How many elements do we have?
02859   unsigned NumElts = 1;
02860   switch (N->getOpcode()) {
02861   default:
02862     return nullptr;
02863   case NVPTXISD::StoreParamU32:
02864   case NVPTXISD::StoreParamS32:
02865   case NVPTXISD::StoreParam:
02866     NumElts = 1;
02867     break;
02868   case NVPTXISD::StoreParamV2:
02869     NumElts = 2;
02870     break;
02871   case NVPTXISD::StoreParamV4:
02872     NumElts = 4;
02873     break;
02874   }
02875 
02876   // Build vector of operands
02877   SmallVector<SDValue, 8> Ops;
02878   for (unsigned i = 0; i < NumElts; ++i)
02879     Ops.push_back(N->getOperand(i + 3));
02880   Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
02881   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
02882   Ops.push_back(Chain);
02883   Ops.push_back(Flag);
02884 
02885   // Determine target opcode
02886   // If we have an i1, use an 8-bit store. The lowering code in
02887   // NVPTXISelLowering will have already emitted an upcast.
02888   unsigned Opcode = 0;
02889   switch (N->getOpcode()) {
02890   default:
02891     switch (NumElts) {
02892     default:
02893       return nullptr;
02894     case 1:
02895       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02896       default:
02897         return nullptr;
02898       case MVT::i1:
02899         Opcode = NVPTX::StoreParamI8;
02900         break;
02901       case MVT::i8:
02902         Opcode = NVPTX::StoreParamI8;
02903         break;
02904       case MVT::i16:
02905         Opcode = NVPTX::StoreParamI16;
02906         break;
02907       case MVT::i32:
02908         Opcode = NVPTX::StoreParamI32;
02909         break;
02910       case MVT::i64:
02911         Opcode = NVPTX::StoreParamI64;
02912         break;
02913       case MVT::f32:
02914         Opcode = NVPTX::StoreParamF32;
02915         break;
02916       case MVT::f64:
02917         Opcode = NVPTX::StoreParamF64;
02918         break;
02919       }
02920       break;
02921     case 2:
02922       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02923       default:
02924         return nullptr;
02925       case MVT::i1:
02926         Opcode = NVPTX::StoreParamV2I8;
02927         break;
02928       case MVT::i8:
02929         Opcode = NVPTX::StoreParamV2I8;
02930         break;
02931       case MVT::i16:
02932         Opcode = NVPTX::StoreParamV2I16;
02933         break;
02934       case MVT::i32:
02935         Opcode = NVPTX::StoreParamV2I32;
02936         break;
02937       case MVT::i64:
02938         Opcode = NVPTX::StoreParamV2I64;
02939         break;
02940       case MVT::f32:
02941         Opcode = NVPTX::StoreParamV2F32;
02942         break;
02943       case MVT::f64:
02944         Opcode = NVPTX::StoreParamV2F64;
02945         break;
02946       }
02947       break;
02948     case 4:
02949       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02950       default:
02951         return nullptr;
02952       case MVT::i1:
02953         Opcode = NVPTX::StoreParamV4I8;
02954         break;
02955       case MVT::i8:
02956         Opcode = NVPTX::StoreParamV4I8;
02957         break;
02958       case MVT::i16:
02959         Opcode = NVPTX::StoreParamV4I16;
02960         break;
02961       case MVT::i32:
02962         Opcode = NVPTX::StoreParamV4I32;
02963         break;
02964       case MVT::f32:
02965         Opcode = NVPTX::StoreParamV4F32;
02966         break;
02967       }
02968       break;
02969     }
02970     break;
02971   // Special case: if we have a sign-extend/zero-extend node, insert the
02972   // conversion instruction first, and use that as the value operand to
02973   // the selected StoreParam node.
02974   case NVPTXISD::StoreParamU32: {
02975     Opcode = NVPTX::StoreParamI32;
02976     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
02977                                                 MVT::i32);
02978     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
02979                                          MVT::i32, Ops[0], CvtNone);
02980     Ops[0] = SDValue(Cvt, 0);
02981     break;
02982   }
02983   case NVPTXISD::StoreParamS32: {
02984     Opcode = NVPTX::StoreParamI32;
02985     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
02986                                                 MVT::i32);
02987     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
02988                                          MVT::i32, Ops[0], CvtNone);
02989     Ops[0] = SDValue(Cvt, 0);
02990     break;
02991   }
02992   }
02993 
02994   SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
02995   SDNode *Ret =
02996       CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
02997   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02998   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02999   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
03000 
03001   return Ret;
03002 }
03003 
03004 SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
03005   SDValue Chain = N->getOperand(0);
03006   SDNode *Ret = nullptr;
03007   unsigned Opc = 0;
03008   SmallVector<SDValue, 8> Ops;
03009 
03010   switch (N->getOpcode()) {
03011   default: return nullptr;
03012   case NVPTXISD::Tex1DFloatS32:
03013     Opc = NVPTX::TEX_1D_F32_S32;
03014     break;
03015   case NVPTXISD::Tex1DFloatFloat:
03016     Opc = NVPTX::TEX_1D_F32_F32;
03017     break;
03018   case NVPTXISD::Tex1DFloatFloatLevel:
03019     Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
03020     break;
03021   case NVPTXISD::Tex1DFloatFloatGrad:
03022     Opc = NVPTX::TEX_1D_F32_F32_GRAD;
03023     break;
03024   case NVPTXISD::Tex1DS32S32:
03025     Opc = NVPTX::TEX_1D_S32_S32;
03026     break;
03027   case NVPTXISD::Tex1DS32Float:
03028     Opc = NVPTX::TEX_1D_S32_F32;
03029     break;
03030   case NVPTXISD::Tex1DS32FloatLevel:
03031     Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
03032     break;
03033   case NVPTXISD::Tex1DS32FloatGrad:
03034     Opc = NVPTX::TEX_1D_S32_F32_GRAD;
03035     break;
03036   case NVPTXISD::Tex1DU32S32:
03037     Opc = NVPTX::TEX_1D_U32_S32;
03038     break;
03039   case NVPTXISD::Tex1DU32Float:
03040     Opc = NVPTX::TEX_1D_U32_F32;
03041     break;
03042   case NVPTXISD::Tex1DU32FloatLevel:
03043     Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
03044     break;
03045   case NVPTXISD::Tex1DU32FloatGrad:
03046     Opc = NVPTX::TEX_1D_U32_F32_GRAD;
03047     break;
03048   case NVPTXISD::Tex1DArrayFloatS32:
03049     Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
03050     break;
03051   case NVPTXISD::Tex1DArrayFloatFloat:
03052     Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
03053     break;
03054   case NVPTXISD::Tex1DArrayFloatFloatLevel:
03055     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
03056     break;
03057   case NVPTXISD::Tex1DArrayFloatFloatGrad:
03058     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
03059     break;
03060   case NVPTXISD::Tex1DArrayS32S32:
03061     Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
03062     break;
03063   case NVPTXISD::Tex1DArrayS32Float:
03064     Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
03065     break;
03066   case NVPTXISD::Tex1DArrayS32FloatLevel:
03067     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
03068     break;
03069   case NVPTXISD::Tex1DArrayS32FloatGrad:
03070     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
03071     break;
03072   case NVPTXISD::Tex1DArrayU32S32:
03073     Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
03074     break;
03075   case NVPTXISD::Tex1DArrayU32Float:
03076     Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
03077     break;
03078   case NVPTXISD::Tex1DArrayU32FloatLevel:
03079     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
03080     break;
03081   case NVPTXISD::Tex1DArrayU32FloatGrad:
03082     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
03083     break;
03084   case NVPTXISD::Tex2DFloatS32:
03085     Opc = NVPTX::TEX_2D_F32_S32;
03086     break;
03087   case NVPTXISD::Tex2DFloatFloat:
03088     Opc = NVPTX::TEX_2D_F32_F32;
03089     break;
03090   case NVPTXISD::Tex2DFloatFloatLevel:
03091     Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
03092     break;
03093   case NVPTXISD::Tex2DFloatFloatGrad:
03094     Opc = NVPTX::TEX_2D_F32_F32_GRAD;
03095     break;
03096   case NVPTXISD::Tex2DS32S32:
03097     Opc = NVPTX::TEX_2D_S32_S32;
03098     break;
03099   case NVPTXISD::Tex2DS32Float:
03100     Opc = NVPTX::TEX_2D_S32_F32;
03101     break;
03102   case NVPTXISD::Tex2DS32FloatLevel:
03103     Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
03104     break;
03105   case NVPTXISD::Tex2DS32FloatGrad:
03106     Opc = NVPTX::TEX_2D_S32_F32_GRAD;
03107     break;
03108   case NVPTXISD::Tex2DU32S32:
03109     Opc = NVPTX::TEX_2D_U32_S32;
03110     break;
03111   case NVPTXISD::Tex2DU32Float:
03112     Opc = NVPTX::TEX_2D_U32_F32;
03113     break;
03114   case NVPTXISD::Tex2DU32FloatLevel:
03115     Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
03116     break;
03117   case NVPTXISD::Tex2DU32FloatGrad:
03118     Opc = NVPTX::TEX_2D_U32_F32_GRAD;
03119     break;
03120   case NVPTXISD::Tex2DArrayFloatS32:
03121     Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
03122     break;
03123   case NVPTXISD::Tex2DArrayFloatFloat:
03124     Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
03125     break;
03126   case NVPTXISD::Tex2DArrayFloatFloatLevel:
03127     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
03128     break;
03129   case NVPTXISD::Tex2DArrayFloatFloatGrad:
03130     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
03131     break;
03132   case NVPTXISD::Tex2DArrayS32S32:
03133     Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
03134     break;
03135   case NVPTXISD::Tex2DArrayS32Float:
03136     Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
03137     break;
03138   case NVPTXISD::Tex2DArrayS32FloatLevel:
03139     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
03140     break;
03141   case NVPTXISD::Tex2DArrayS32FloatGrad:
03142     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
03143     break;
03144   case NVPTXISD::Tex2DArrayU32S32:
03145     Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
03146     break;
03147   case NVPTXISD::Tex2DArrayU32Float:
03148     Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
03149     break;
03150   case NVPTXISD::Tex2DArrayU32FloatLevel:
03151     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
03152     break;
03153   case NVPTXISD::Tex2DArrayU32FloatGrad:
03154     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
03155     break;
03156   case NVPTXISD::Tex3DFloatS32:
03157     Opc = NVPTX::TEX_3D_F32_S32;
03158     break;
03159   case NVPTXISD::Tex3DFloatFloat:
03160     Opc = NVPTX::TEX_3D_F32_F32;
03161     break;
03162   case NVPTXISD::Tex3DFloatFloatLevel:
03163     Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
03164     break;
03165   case NVPTXISD::Tex3DFloatFloatGrad:
03166     Opc = NVPTX::TEX_3D_F32_F32_GRAD;
03167     break;
03168   case NVPTXISD::Tex3DS32S32:
03169     Opc = NVPTX::TEX_3D_S32_S32;
03170     break;
03171   case NVPTXISD::Tex3DS32Float:
03172     Opc = NVPTX::TEX_3D_S32_F32;
03173     break;
03174   case NVPTXISD::Tex3DS32FloatLevel:
03175     Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
03176     break;
03177   case NVPTXISD::Tex3DS32FloatGrad:
03178     Opc = NVPTX::TEX_3D_S32_F32_GRAD;
03179     break;
03180   case NVPTXISD::Tex3DU32S32:
03181     Opc = NVPTX::TEX_3D_U32_S32;
03182     break;
03183   case NVPTXISD::Tex3DU32Float:
03184     Opc = NVPTX::TEX_3D_U32_F32;
03185     break;
03186   case NVPTXISD::Tex3DU32FloatLevel:
03187     Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
03188     break;
03189   case NVPTXISD::Tex3DU32FloatGrad:
03190     Opc = NVPTX::TEX_3D_U32_F32_GRAD;
03191     break;
03192   case NVPTXISD::TexCubeFloatFloat:
03193     Opc = NVPTX::TEX_CUBE_F32_F32;
03194     break;
03195   case NVPTXISD::TexCubeFloatFloatLevel:
03196     Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
03197     break;
03198   case NVPTXISD::TexCubeS32Float:
03199     Opc = NVPTX::TEX_CUBE_S32_F32;
03200     break;
03201   case NVPTXISD::TexCubeS32FloatLevel:
03202     Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
03203     break;
03204   case NVPTXISD::TexCubeU32Float:
03205     Opc = NVPTX::TEX_CUBE_U32_F32;
03206     break;
03207   case NVPTXISD::TexCubeU32FloatLevel:
03208     Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
03209     break;
03210   case NVPTXISD::TexCubeArrayFloatFloat:
03211     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
03212     break;
03213   case NVPTXISD::TexCubeArrayFloatFloatLevel:
03214     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
03215     break;
03216   case NVPTXISD::TexCubeArrayS32Float:
03217     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
03218     break;
03219   case NVPTXISD::TexCubeArrayS32FloatLevel:
03220     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
03221     break;
03222   case NVPTXISD::TexCubeArrayU32Float:
03223     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
03224     break;
03225   case NVPTXISD::TexCubeArrayU32FloatLevel:
03226     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
03227     break;
03228   case NVPTXISD::Tld4R2DFloatFloat:
03229     Opc = NVPTX::TLD4_R_2D_F32_F32;
03230     break;
03231   case NVPTXISD::Tld4G2DFloatFloat:
03232     Opc = NVPTX::TLD4_G_2D_F32_F32;
03233     break;
03234   case NVPTXISD::Tld4B2DFloatFloat:
03235     Opc = NVPTX::TLD4_B_2D_F32_F32;
03236     break;
03237   case NVPTXISD::Tld4A2DFloatFloat:
03238     Opc = NVPTX::TLD4_A_2D_F32_F32;
03239     break;
03240   case NVPTXISD::Tld4R2DS64Float:
03241     Opc = NVPTX::TLD4_R_2D_S32_F32;
03242     break;
03243   case NVPTXISD::Tld4G2DS64Float:
03244     Opc = NVPTX::TLD4_G_2D_S32_F32;
03245     break;
03246   case NVPTXISD::Tld4B2DS64Float:
03247     Opc = NVPTX::TLD4_B_2D_S32_F32;
03248     break;
03249   case NVPTXISD::Tld4A2DS64Float:
03250     Opc = NVPTX::TLD4_A_2D_S32_F32;
03251     break;
03252   case NVPTXISD::Tld4R2DU64Float:
03253     Opc = NVPTX::TLD4_R_2D_U32_F32;
03254     break;
03255   case NVPTXISD::Tld4G2DU64Float:
03256     Opc = NVPTX::TLD4_G_2D_U32_F32;
03257     break;
03258   case NVPTXISD::Tld4B2DU64Float:
03259     Opc = NVPTX::TLD4_B_2D_U32_F32;
03260     break;
03261   case NVPTXISD::Tld4A2DU64Float:
03262     Opc = NVPTX::TLD4_A_2D_U32_F32;
03263     break;
03264   case NVPTXISD::TexUnified1DFloatS32:
03265     Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
03266     break;
03267   case NVPTXISD::TexUnified1DFloatFloat:
03268     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
03269     break;
03270   case NVPTXISD::TexUnified1DFloatFloatLevel:
03271     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
03272     break;
03273   case NVPTXISD::TexUnified1DFloatFloatGrad:
03274     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
03275     break;
03276   case NVPTXISD::TexUnified1DS32S32:
03277     Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
03278     break;
03279   case NVPTXISD::TexUnified1DS32Float:
03280     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
03281     break;
03282   case NVPTXISD::TexUnified1DS32FloatLevel:
03283     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
03284     break;
03285   case NVPTXISD::TexUnified1DS32FloatGrad:
03286     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
03287     break;
03288   case NVPTXISD::TexUnified1DU32S32:
03289     Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
03290     break;
03291   case NVPTXISD::TexUnified1DU32Float:
03292     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
03293     break;
03294   case NVPTXISD::TexUnified1DU32FloatLevel:
03295     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
03296     break;
03297   case NVPTXISD::TexUnified1DU32FloatGrad:
03298     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
03299     break;
03300   case NVPTXISD::TexUnified1DArrayFloatS32:
03301     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
03302     break;
03303   case NVPTXISD::TexUnified1DArrayFloatFloat:
03304     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
03305     break;
03306   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
03307     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
03308     break;
03309   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
03310     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
03311     break;
03312   case NVPTXISD::TexUnified1DArrayS32S32:
03313     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
03314     break;
03315   case NVPTXISD::TexUnified1DArrayS32Float:
03316     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
03317     break;
03318   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
03319     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
03320     break;
03321   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
03322     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
03323     break;
03324   case NVPTXISD::TexUnified1DArrayU32S32:
03325     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
03326     break;
03327   case NVPTXISD::TexUnified1DArrayU32Float:
03328     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
03329     break;
03330   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
03331     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
03332     break;
03333   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
03334     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
03335     break;
03336   case NVPTXISD::TexUnified2DFloatS32:
03337     Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
03338     break;
03339   case NVPTXISD::TexUnified2DFloatFloat:
03340     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
03341     break;
03342   case NVPTXISD::TexUnified2DFloatFloatLevel:
03343     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
03344     break;
03345   case NVPTXISD::TexUnified2DFloatFloatGrad:
03346     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
03347     break;
03348   case NVPTXISD::TexUnified2DS32S32:
03349     Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
03350     break;
03351   case NVPTXISD::TexUnified2DS32Float:
03352     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
03353     break;
03354   case NVPTXISD::TexUnified2DS32FloatLevel:
03355     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
03356     break;
03357   case NVPTXISD::TexUnified2DS32FloatGrad:
03358     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
03359     break;
03360   case NVPTXISD::TexUnified2DU32S32:
03361     Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
03362     break;
03363   case NVPTXISD::TexUnified2DU32Float:
03364     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
03365     break;
03366   case NVPTXISD::TexUnified2DU32FloatLevel:
03367     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
03368     break;
03369   case NVPTXISD::TexUnified2DU32FloatGrad:
03370     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
03371     break;
03372   case NVPTXISD::TexUnified2DArrayFloatS32:
03373     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
03374     break;
03375   case NVPTXISD::TexUnified2DArrayFloatFloat:
03376     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
03377     break;
03378   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
03379     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
03380     break;
03381   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
03382     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
03383     break;
03384   case NVPTXISD::TexUnified2DArrayS32S32:
03385     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
03386     break;
03387   case NVPTXISD::TexUnified2DArrayS32Float:
03388     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
03389     break;
03390   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
03391     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
03392     break;
03393   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
03394     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
03395     break;
03396   case NVPTXISD::TexUnified2DArrayU32S32:
03397     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
03398     break;
03399   case NVPTXISD::TexUnified2DArrayU32Float:
03400     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
03401     break;
03402   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
03403     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
03404     break;
03405   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
03406     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
03407     break;
03408   case NVPTXISD::TexUnified3DFloatS32:
03409     Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
03410     break;
03411   case NVPTXISD::TexUnified3DFloatFloat:
03412     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
03413     break;
03414   case NVPTXISD::TexUnified3DFloatFloatLevel:
03415     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
03416     break;
03417   case NVPTXISD::TexUnified3DFloatFloatGrad:
03418     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
03419     break;
03420   case NVPTXISD::TexUnified3DS32S32:
03421     Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
03422     break;
03423   case NVPTXISD::TexUnified3DS32Float:
03424     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
03425     break;
03426   case NVPTXISD::TexUnified3DS32FloatLevel:
03427     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
03428     break;
03429   case NVPTXISD::TexUnified3DS32FloatGrad:
03430     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
03431     break;
03432   case NVPTXISD::TexUnified3DU32S32:
03433     Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
03434     break;
03435   case NVPTXISD::TexUnified3DU32Float:
03436     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
03437     break;
03438   case NVPTXISD::TexUnified3DU32FloatLevel:
03439     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
03440     break;
03441   case NVPTXISD::TexUnified3DU32FloatGrad:
03442     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
03443     break;
03444   case NVPTXISD::TexUnifiedCubeFloatFloat:
03445     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
03446     break;
03447   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
03448     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
03449     break;
03450   case NVPTXISD::TexUnifiedCubeS32Float:
03451     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
03452     break;
03453   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
03454     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
03455     break;
03456   case NVPTXISD::TexUnifiedCubeU32Float:
03457     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
03458     break;
03459   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
03460     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
03461     break;
03462   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
03463     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
03464     break;
03465   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
03466     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
03467     break;
03468   case NVPTXISD::TexUnifiedCubeArrayS32Float:
03469     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
03470     break;
03471   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
03472     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
03473     break;
03474   case NVPTXISD::TexUnifiedCubeArrayU32Float:
03475     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
03476     break;
03477   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
03478     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
03479     break;
03480   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
03481     Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
03482     break;
03483   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
03484     Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
03485     break;
03486   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
03487     Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
03488     break;
03489   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
03490     Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
03491     break;
03492   case NVPTXISD::Tld4UnifiedR2DS64Float:
03493     Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
03494     break;
03495   case NVPTXISD::Tld4UnifiedG2DS64Float:
03496     Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
03497     break;
03498   case NVPTXISD::Tld4UnifiedB2DS64Float:
03499     Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
03500     break;
03501   case NVPTXISD::Tld4UnifiedA2DS64Float:
03502     Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
03503     break;
03504   case NVPTXISD::Tld4UnifiedR2DU64Float:
03505     Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
03506     break;
03507   case NVPTXISD::Tld4UnifiedG2DU64Float:
03508     Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
03509     break;
03510   case NVPTXISD::Tld4UnifiedB2DU64Float:
03511     Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
03512     break;
03513   case NVPTXISD::Tld4UnifiedA2DU64Float:
03514     Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
03515     break;
03516   }
03517 
03518   // Copy over operands
03519   for (unsigned i = 1; i < N->getNumOperands(); ++i) {
03520     Ops.push_back(N->getOperand(i));
03521   }
03522 
03523   Ops.push_back(Chain);
03524   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
03525   return Ret;
03526 }
03527 
03528 SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
03529   SDValue Chain = N->getOperand(0);
03530   SDValue TexHandle = N->getOperand(1);
03531   SDNode *Ret = nullptr;
03532   unsigned Opc = 0;
03533   SmallVector<SDValue, 8> Ops;
03534   switch (N->getOpcode()) {
03535   default: return nullptr;
03536   case NVPTXISD::Suld1DI8Clamp:
03537     Opc = NVPTX::SULD_1D_I8_CLAMP;
03538     Ops.push_back(TexHandle);
03539     Ops.push_back(N->getOperand(2));
03540     Ops.push_back(Chain);
03541     break;
03542   case NVPTXISD::Suld1DI16Clamp:
03543     Opc = NVPTX::SULD_1D_I16_CLAMP;
03544     Ops.push_back(TexHandle);
03545     Ops.push_back(N->getOperand(2));
03546     Ops.push_back(Chain);
03547     break;
03548   case NVPTXISD::Suld1DI32Clamp:
03549     Opc = NVPTX::SULD_1D_I32_CLAMP;
03550     Ops.push_back(TexHandle);
03551     Ops.push_back(N->getOperand(2));
03552     Ops.push_back(Chain);
03553     break;
03554   case NVPTXISD::Suld1DI64Clamp:
03555     Opc = NVPTX::SULD_1D_I64_CLAMP;
03556     Ops.push_back(TexHandle);
03557     Ops.push_back(N->getOperand(2));
03558     Ops.push_back(Chain);
03559     break;
03560   case NVPTXISD::Suld1DV2I8Clamp:
03561     Opc = NVPTX::SULD_1D_V2I8_CLAMP;
03562     Ops.push_back(TexHandle);
03563     Ops.push_back(N->getOperand(2));
03564     Ops.push_back(Chain);
03565     break;
03566   case NVPTXISD::Suld1DV2I16Clamp:
03567     Opc = NVPTX::SULD_1D_V2I16_CLAMP;
03568     Ops.push_back(TexHandle);
03569     Ops.push_back(N->getOperand(2));
03570     Ops.push_back(Chain);
03571     break;
03572   case NVPTXISD::Suld1DV2I32Clamp:
03573     Opc = NVPTX::SULD_1D_V2I32_CLAMP;
03574     Ops.push_back(TexHandle);
03575     Ops.push_back(N->getOperand(2));
03576     Ops.push_back(Chain);
03577     break;
03578   case NVPTXISD::Suld1DV2I64Clamp:
03579     Opc = NVPTX::SULD_1D_V2I64_CLAMP;
03580     Ops.push_back(TexHandle);
03581     Ops.push_back(N->getOperand(2));
03582     Ops.push_back(Chain);
03583     break;
03584   case NVPTXISD::Suld1DV4I8Clamp:
03585     Opc = NVPTX::SULD_1D_V4I8_CLAMP;
03586     Ops.push_back(TexHandle);
03587     Ops.push_back(N->getOperand(2));
03588     Ops.push_back(Chain);
03589     break;
03590   case NVPTXISD::Suld1DV4I16Clamp:
03591     Opc = NVPTX::SULD_1D_V4I16_CLAMP;
03592     Ops.push_back(TexHandle);
03593     Ops.push_back(N->getOperand(2));
03594     Ops.push_back(Chain);
03595     break;
03596   case NVPTXISD::Suld1DV4I32Clamp:
03597     Opc = NVPTX::SULD_1D_V4I32_CLAMP;
03598     Ops.push_back(TexHandle);
03599     Ops.push_back(N->getOperand(2));
03600     Ops.push_back(Chain);
03601     break;
03602   case NVPTXISD::Suld1DArrayI8Clamp:
03603     Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
03604     Ops.push_back(TexHandle);
03605     Ops.push_back(N->getOperand(2));
03606     Ops.push_back(N->getOperand(3));
03607     Ops.push_back(Chain);
03608     break;
03609   case NVPTXISD::Suld1DArrayI16Clamp:
03610     Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
03611     Ops.push_back(TexHandle);
03612     Ops.push_back(N->getOperand(2));
03613     Ops.push_back(N->getOperand(3));
03614     Ops.push_back(Chain);
03615     break;
03616   case NVPTXISD::Suld1DArrayI32Clamp:
03617     Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
03618     Ops.push_back(TexHandle);
03619     Ops.push_back(N->getOperand(2));
03620     Ops.push_back(N->getOperand(3));
03621     Ops.push_back(Chain);
03622     break;
03623   case NVPTXISD::Suld1DArrayI64Clamp:
03624     Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
03625     Ops.push_back(TexHandle);
03626     Ops.push_back(N->getOperand(2));
03627     Ops.push_back(N->getOperand(3));
03628     Ops.push_back(Chain);
03629     break;
03630   case NVPTXISD::Suld1DArrayV2I8Clamp:
03631     Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
03632     Ops.push_back(TexHandle);
03633     Ops.push_back(N->getOperand(2));
03634     Ops.push_back(N->getOperand(3));
03635     Ops.push_back(Chain);
03636     break;
03637   case NVPTXISD::Suld1DArrayV2I16Clamp:
03638     Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
03639     Ops.push_back(TexHandle);
03640     Ops.push_back(N->getOperand(2));
03641     Ops.push_back(N->getOperand(3));
03642     Ops.push_back(Chain);
03643     break;
03644   case NVPTXISD::Suld1DArrayV2I32Clamp:
03645     Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
03646     Ops.push_back(TexHandle);
03647     Ops.push_back(N->getOperand(2));
03648     Ops.push_back(N->getOperand(3));
03649     Ops.push_back(Chain);
03650     break;
03651   case NVPTXISD::Suld1DArrayV2I64Clamp:
03652     Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
03653     Ops.push_back(TexHandle);
03654     Ops.push_back(N->getOperand(2));
03655     Ops.push_back(N->getOperand(3));
03656     Ops.push_back(Chain);
03657     break;
03658   case NVPTXISD::Suld1DArrayV4I8Clamp:
03659     Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
03660     Ops.push_back(TexHandle);
03661     Ops.push_back(N->getOperand(2));
03662     Ops.push_back(N->getOperand(3));
03663     Ops.push_back(Chain);
03664     break;
03665   case NVPTXISD::Suld1DArrayV4I16Clamp:
03666     Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
03667     Ops.push_back(TexHandle);
03668     Ops.push_back(N->getOperand(2));
03669     Ops.push_back(N->getOperand(3));
03670     Ops.push_back(Chain);
03671     break;
03672   case NVPTXISD::Suld1DArrayV4I32Clamp:
03673     Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
03674     Ops.push_back(TexHandle);
03675     Ops.push_back(N->getOperand(2));
03676     Ops.push_back(N->getOperand(3));
03677     Ops.push_back(Chain);
03678     break;
03679   case NVPTXISD::Suld2DI8Clamp:
03680     Opc = NVPTX::SULD_2D_I8_CLAMP;
03681     Ops.push_back(TexHandle);
03682     Ops.push_back(N->getOperand(2));
03683     Ops.push_back(N->getOperand(3));
03684     Ops.push_back(Chain);
03685     break;
03686   case NVPTXISD::Suld2DI16Clamp:
03687     Opc = NVPTX::SULD_2D_I16_CLAMP;
03688     Ops.push_back(TexHandle);
03689     Ops.push_back(N->getOperand(2));
03690     Ops.push_back(N->getOperand(3));
03691     Ops.push_back(Chain);
03692     break;
03693   case NVPTXISD::Suld2DI32Clamp:
03694     Opc = NVPTX::SULD_2D_I32_CLAMP;
03695     Ops.push_back(TexHandle);
03696     Ops.push_back(N->getOperand(2));
03697     Ops.push_back(N->getOperand(3));
03698     Ops.push_back(Chain);
03699     break;
03700   case NVPTXISD::Suld2DI64Clamp:
03701     Opc = NVPTX::SULD_2D_I64_CLAMP;
03702     Ops.push_back(TexHandle);
03703     Ops.push_back(N->getOperand(2));
03704     Ops.push_back(N->getOperand(3));
03705     Ops.push_back(Chain);
03706     break;
03707   case NVPTXISD::Suld2DV2I8Clamp:
03708     Opc = NVPTX::SULD_2D_V2I8_CLAMP;
03709     Ops.push_back(TexHandle);
03710     Ops.push_back(N->getOperand(2));
03711     Ops.push_back(N->getOperand(3));
03712     Ops.push_back(Chain);
03713     break;
03714   case NVPTXISD::Suld2DV2I16Clamp:
03715     Opc = NVPTX::SULD_2D_V2I16_CLAMP;
03716     Ops.push_back(TexHandle);
03717     Ops.push_back(N->getOperand(2));
03718     Ops.push_back(N->getOperand(3));
03719     Ops.push_back(Chain);
03720     break;
03721   case NVPTXISD::Suld2DV2I32Clamp:
03722     Opc = NVPTX::SULD_2D_V2I32_CLAMP;
03723     Ops.push_back(TexHandle);
03724     Ops.push_back(N->getOperand(2));
03725     Ops.push_back(N->getOperand(3));
03726     Ops.push_back(Chain);
03727     break;
03728   case NVPTXISD::Suld2DV2I64Clamp:
03729     Opc = NVPTX::SULD_2D_V2I64_CLAMP;
03730     Ops.push_back(TexHandle);
03731     Ops.push_back(N->getOperand(2));
03732     Ops.push_back(N->getOperand(3));
03733     Ops.push_back(Chain);
03734     break;
03735   case NVPTXISD::Suld2DV4I8Clamp:
03736     Opc = NVPTX::SULD_2D_V4I8_CLAMP;
03737     Ops.push_back(TexHandle);
03738     Ops.push_back(N->getOperand(2));
03739     Ops.push_back(N->getOperand(3));
03740     Ops.push_back(Chain);
03741     break;
03742   case NVPTXISD::Suld2DV4I16Clamp:
03743     Opc = NVPTX::SULD_2D_V4I16_CLAMP;
03744     Ops.push_back(TexHandle);
03745     Ops.push_back(N->getOperand(2));
03746     Ops.push_back(N->getOperand(3));
03747     Ops.push_back(Chain);
03748     break;
03749   case NVPTXISD::Suld2DV4I32Clamp:
03750     Opc = NVPTX::SULD_2D_V4I32_CLAMP;
03751     Ops.push_back(TexHandle);
03752     Ops.push_back(N->getOperand(2));
03753     Ops.push_back(N->getOperand(3));
03754     Ops.push_back(Chain);
03755     break;
03756   case NVPTXISD::Suld2DArrayI8Clamp:
03757     Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
03758     Ops.push_back(TexHandle);
03759     Ops.push_back(N->getOperand(2));
03760     Ops.push_back(N->getOperand(3));
03761     Ops.push_back(N->getOperand(4));
03762     Ops.push_back(Chain);
03763     break;
03764   case NVPTXISD::Suld2DArrayI16Clamp:
03765     Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
03766     Ops.push_back(TexHandle);
03767     Ops.push_back(N->getOperand(2));
03768     Ops.push_back(N->getOperand(3));
03769     Ops.push_back(N->getOperand(4));
03770     Ops.push_back(Chain);
03771     break;
03772   case NVPTXISD::Suld2DArrayI32Clamp:
03773     Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
03774     Ops.push_back(TexHandle);
03775     Ops.push_back(N->getOperand(2));
03776     Ops.push_back(N->getOperand(3));
03777     Ops.push_back(N->getOperand(4));
03778     Ops.push_back(Chain);
03779     break;
03780   case NVPTXISD::Suld2DArrayI64Clamp:
03781     Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
03782     Ops.push_back(TexHandle);
03783     Ops.push_back(N->getOperand(2));
03784     Ops.push_back(N->getOperand(3));
03785     Ops.push_back(N->getOperand(4));
03786     Ops.push_back(Chain);
03787     break;
03788   case NVPTXISD::Suld2DArrayV2I8Clamp:
03789     Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
03790     Ops.push_back(TexHandle);
03791     Ops.push_back(N->getOperand(2));
03792     Ops.push_back(N->getOperand(3));
03793     Ops.push_back(N->getOperand(4));
03794     Ops.push_back(Chain);
03795     break;
03796   case NVPTXISD::Suld2DArrayV2I16Clamp:
03797     Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
03798     Ops.push_back(TexHandle);
03799     Ops.push_back(N->getOperand(2));
03800     Ops.push_back(N->getOperand(3));
03801     Ops.push_back(N->getOperand(4));
03802     Ops.push_back(Chain);
03803     break;
03804   case NVPTXISD::Suld2DArrayV2I32Clamp:
03805     Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
03806     Ops.push_back(TexHandle);
03807     Ops.push_back(N->getOperand(2));
03808     Ops.push_back(N->getOperand(3));
03809     Ops.push_back(N->getOperand(4));
03810     Ops.push_back(Chain);
03811     break;
03812   case NVPTXISD::Suld2DArrayV2I64Clamp:
03813     Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
03814     Ops.push_back(TexHandle);
03815     Ops.push_back(N->getOperand(2));
03816     Ops.push_back(N->getOperand(3));
03817     Ops.push_back(N->getOperand(4));
03818     Ops.push_back(Chain);
03819     break;
03820   case NVPTXISD::Suld2DArrayV4I8Clamp:
03821     Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
03822     Ops.push_back(TexHandle);
03823     Ops.push_back(N->getOperand(2));
03824     Ops.push_back(N->getOperand(3));
03825     Ops.push_back(N->getOperand(4));
03826     Ops.push_back(Chain);
03827     break;
03828   case NVPTXISD::Suld2DArrayV4I16Clamp:
03829     Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
03830     Ops.push_back(TexHandle);
03831     Ops.push_back(N->getOperand(2));
03832     Ops.push_back(N->getOperand(3));
03833     Ops.push_back(N->getOperand(4));
03834     Ops.push_back(Chain);
03835     break;
03836   case NVPTXISD::Suld2DArrayV4I32Clamp:
03837     Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
03838     Ops.push_back(TexHandle);
03839     Ops.push_back(N->getOperand(2));
03840     Ops.push_back(N->getOperand(3));
03841     Ops.push_back(N->getOperand(4));
03842     Ops.push_back(Chain);
03843     break;
03844   case NVPTXISD::Suld3DI8Clamp:
03845     Opc = NVPTX::SULD_3D_I8_CLAMP;
03846     Ops.push_back(TexHandle);
03847     Ops.push_back(N->getOperand(2));
03848     Ops.push_back(N->getOperand(3));
03849     Ops.push_back(N->getOperand(4));
03850     Ops.push_back(Chain);
03851     break;
03852   case NVPTXISD::Suld3DI16Clamp:
03853     Opc = NVPTX::SULD_3D_I16_CLAMP;
03854     Ops.push_back(TexHandle);
03855     Ops.push_back(N->getOperand(2));
03856     Ops.push_back(N->getOperand(3));
03857     Ops.push_back(N->getOperand(4));
03858     Ops.push_back(Chain);
03859     break;
03860   case NVPTXISD::Suld3DI32Clamp:
03861     Opc = NVPTX::SULD_3D_I32_CLAMP;
03862     Ops.push_back(TexHandle);
03863     Ops.push_back(N->getOperand(2));
03864     Ops.push_back(N->getOperand(3));
03865     Ops.push_back(N->getOperand(4));
03866     Ops.push_back(Chain);
03867     break;
03868   case NVPTXISD::Suld3DI64Clamp:
03869     Opc = NVPTX::SULD_3D_I64_CLAMP;
03870     Ops.push_back(TexHandle);
03871     Ops.push_back(N->getOperand(2));
03872     Ops.push_back(N->getOperand(3));
03873     Ops.push_back(N->getOperand(4));
03874     Ops.push_back(Chain);
03875     break;
03876   case NVPTXISD::Suld3DV2I8Clamp:
03877     Opc = NVPTX::SULD_3D_V2I8_CLAMP;
03878     Ops.push_back(TexHandle);
03879     Ops.push_back(N->getOperand(2));
03880     Ops.push_back(N->getOperand(3));
03881     Ops.push_back(N->getOperand(4));
03882     Ops.push_back(Chain);
03883     break;
03884   case NVPTXISD::Suld3DV2I16Clamp:
03885     Opc = NVPTX::SULD_3D_V2I16_CLAMP;
03886     Ops.push_back(TexHandle);
03887     Ops.push_back(N->getOperand(2));
03888     Ops.push_back(N->getOperand(3));
03889     Ops.push_back(N->getOperand(4));
03890     Ops.push_back(Chain);
03891     break;
03892   case NVPTXISD::Suld3DV2I32Clamp:
03893     Opc = NVPTX::SULD_3D_V2I32_CLAMP;
03894     Ops.push_back(TexHandle);
03895     Ops.push_back(N->getOperand(2));
03896     Ops.push_back(N->getOperand(3));
03897     Ops.push_back(N->getOperand(4));
03898     Ops.push_back(Chain);
03899     break;
03900   case NVPTXISD::Suld3DV2I64Clamp:
03901     Opc = NVPTX::SULD_3D_V2I64_CLAMP;
03902     Ops.push_back(TexHandle);
03903     Ops.push_back(N->getOperand(2));
03904     Ops.push_back(N->getOperand(3));
03905     Ops.push_back(N->getOperand(4));
03906     Ops.push_back(Chain);
03907     break;
03908   case NVPTXISD::Suld3DV4I8Clamp:
03909     Opc = NVPTX::SULD_3D_V4I8_CLAMP;
03910     Ops.push_back(TexHandle);
03911     Ops.push_back(N->getOperand(2));
03912     Ops.push_back(N->getOperand(3));
03913     Ops.push_back(N->getOperand(4));
03914     Ops.push_back(Chain);
03915     break;
03916   case NVPTXISD::Suld3DV4I16Clamp:
03917     Opc = NVPTX::SULD_3D_V4I16_CLAMP;
03918     Ops.push_back(TexHandle);
03919     Ops.push_back(N->getOperand(2));
03920     Ops.push_back(N->getOperand(3));
03921     Ops.push_back(N->getOperand(4));
03922     Ops.push_back(Chain);
03923     break;
03924   case NVPTXISD::Suld3DV4I32Clamp:
03925     Opc = NVPTX::SULD_3D_V4I32_CLAMP;
03926     Ops.push_back(TexHandle);
03927     Ops.push_back(N->getOperand(2));
03928     Ops.push_back(N->getOperand(3));
03929     Ops.push_back(N->getOperand(4));
03930     Ops.push_back(Chain);
03931     break;
03932   case NVPTXISD::Suld1DI8Trap:
03933     Opc = NVPTX::SULD_1D_I8_TRAP;
03934     Ops.push_back(TexHandle);
03935     Ops.push_back(N->getOperand(2));
03936     Ops.push_back(Chain);
03937     break;
03938   case NVPTXISD::Suld1DI16Trap:
03939     Opc = NVPTX::SULD_1D_I16_TRAP;
03940     Ops.push_back(TexHandle);
03941     Ops.push_back(N->getOperand(2));
03942     Ops.push_back(Chain);
03943     break;
03944   case NVPTXISD::Suld1DI32Trap:
03945     Opc = NVPTX::SULD_1D_I32_TRAP;
03946     Ops.push_back(TexHandle);
03947     Ops.push_back(N->getOperand(2));
03948     Ops.push_back(Chain);
03949     break;
03950   case NVPTXISD::Suld1DI64Trap:
03951     Opc = NVPTX::SULD_1D_I64_TRAP;
03952     Ops.push_back(TexHandle);
03953     Ops.push_back(N->getOperand(2));
03954     Ops.push_back(Chain);
03955     break;
03956   case NVPTXISD::Suld1DV2I8Trap:
03957     Opc = NVPTX::SULD_1D_V2I8_TRAP;
03958     Ops.push_back(TexHandle);
03959     Ops.push_back(N->getOperand(2));
03960     Ops.push_back(Chain);
03961     break;
03962   case NVPTXISD::Suld1DV2I16Trap:
03963     Opc = NVPTX::SULD_1D_V2I16_TRAP;
03964     Ops.push_back(TexHandle);
03965     Ops.push_back(N->getOperand(2));
03966     Ops.push_back(Chain);
03967     break;
03968   case NVPTXISD::Suld1DV2I32Trap:
03969     Opc = NVPTX::SULD_1D_V2I32_TRAP;
03970     Ops.push_back(TexHandle);
03971     Ops.push_back(N->getOperand(2));
03972     Ops.push_back(Chain);
03973     break;
03974   case NVPTXISD::Suld1DV2I64Trap:
03975     Opc = NVPTX::SULD_1D_V2I64_TRAP;
03976     Ops.push_back(TexHandle);
03977     Ops.push_back(N->getOperand(2));
03978     Ops.push_back(Chain);
03979     break;
03980   case NVPTXISD::Suld1DV4I8Trap:
03981     Opc = NVPTX::SULD_1D_V4I8_TRAP;
03982     Ops.push_back(TexHandle);
03983     Ops.push_back(N->getOperand(2));
03984     Ops.push_back(Chain);
03985     break;
03986   case NVPTXISD::Suld1DV4I16Trap:
03987     Opc = NVPTX::SULD_1D_V4I16_TRAP;
03988     Ops.push_back(TexHandle);
03989     Ops.push_back(N->getOperand(2));
03990     Ops.push_back(Chain);
03991     break;
03992   case NVPTXISD::Suld1DV4I32Trap:
03993     Opc = NVPTX::SULD_1D_V4I32_TRAP;
03994     Ops.push_back(TexHandle);
03995     Ops.push_back(N->getOperand(2));
03996     Ops.push_back(Chain);
03997     break;
03998   case NVPTXISD::Suld1DArrayI8Trap:
03999     Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
04000     Ops.push_back(TexHandle);
04001     Ops.push_back(N->getOperand(2));
04002     Ops.push_back(N->getOperand(3));
04003     Ops.push_back(Chain);
04004     break;
04005   case NVPTXISD::Suld1DArrayI16Trap:
04006     Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
04007     Ops.push_back(TexHandle);
04008     Ops.push_back(N->getOperand(2));
04009     Ops.push_back(N->getOperand(3));
04010     Ops.push_back(Chain);
04011     break;
04012   case NVPTXISD::Suld1DArrayI32Trap:
04013     Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
04014     Ops.push_back(TexHandle);
04015     Ops.push_back(N->getOperand(2));
04016     Ops.push_back(N->getOperand(3));
04017     Ops.push_back(Chain);
04018     break;
04019   case NVPTXISD::Suld1DArrayI64Trap:
04020     Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
04021     Ops.push_back(TexHandle);
04022     Ops.push_back(N->getOperand(2));
04023     Ops.push_back(N->getOperand(3));
04024     Ops.push_back(Chain);
04025     break;
04026   case NVPTXISD::Suld1DArrayV2I8Trap:
04027     Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
04028     Ops.push_back(TexHandle);
04029     Ops.push_back(N->getOperand(2));
04030     Ops.push_back(N->getOperand(3));
04031     Ops.push_back(Chain);
04032     break;
04033   case NVPTXISD::Suld1DArrayV2I16Trap:
04034     Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
04035     Ops.push_back(TexHandle);
04036     Ops.push_back(N->getOperand(2));
04037     Ops.push_back(N->getOperand(3));
04038     Ops.push_back(Chain);
04039     break;
04040   case NVPTXISD::Suld1DArrayV2I32Trap:
04041     Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
04042     Ops.push_back(TexHandle);
04043     Ops.push_back(N->getOperand(2));
04044     Ops.push_back(N->getOperand(3));
04045     Ops.push_back(Chain);
04046     break;
04047   case NVPTXISD::Suld1DArrayV2I64Trap:
04048     Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
04049     Ops.push_back(TexHandle);
04050     Ops.push_back(N->getOperand(2));
04051     Ops.push_back(N->getOperand(3));
04052     Ops.push_back(Chain);
04053     break;
04054   case NVPTXISD::Suld1DArrayV4I8Trap:
04055     Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
04056     Ops.push_back(TexHandle);
04057     Ops.push_back(N->getOperand(2));
04058     Ops.push_back(N->getOperand(3));
04059     Ops.push_back(Chain);
04060     break;
04061   case NVPTXISD::Suld1DArrayV4I16Trap:
04062     Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
04063     Ops.push_back(TexHandle);
04064     Ops.push_back(N->getOperand(2));
04065     Ops.push_back(N->getOperand(3));
04066     Ops.push_back(Chain);
04067     break;
04068   case NVPTXISD::Suld1DArrayV4I32Trap:
04069     Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
04070     Ops.push_back(TexHandle);
04071     Ops.push_back(N->getOperand(2));
04072     Ops.push_back(N->getOperand(3));
04073     Ops.push_back(Chain);
04074     break;
04075   case NVPTXISD::Suld2DI8Trap:
04076     Opc = NVPTX::SULD_2D_I8_TRAP;
04077     Ops.push_back(TexHandle);
04078     Ops.push_back(N->getOperand(2));
04079     Ops.push_back(N->getOperand(3));
04080     Ops.push_back(Chain);
04081     break;
04082   case NVPTXISD::Suld2DI16Trap:
04083     Opc = NVPTX::SULD_2D_I16_TRAP;
04084     Ops.push_back(TexHandle);
04085     Ops.push_back(N->getOperand(2));
04086     Ops.push_back(N->getOperand(3));
04087     Ops.push_back(Chain);
04088     break;
04089   case NVPTXISD::Suld2DI32Trap:
04090     Opc = NVPTX::SULD_2D_I32_TRAP;
04091     Ops.push_back(TexHandle);
04092     Ops.push_back(N->getOperand(2));
04093     Ops.push_back(N->getOperand(3));
04094     Ops.push_back(Chain);
04095     break;
04096   case NVPTXISD::Suld2DI64Trap:
04097     Opc = NVPTX::SULD_2D_I64_TRAP;
04098     Ops.push_back(TexHandle);
04099     Ops.push_back(N->getOperand(2));
04100     Ops.push_back(N->getOperand(3));
04101     Ops.push_back(Chain);
04102     break;
04103   case NVPTXISD::Suld2DV2I8Trap:
04104     Opc = NVPTX::SULD_2D_V2I8_TRAP;
04105     Ops.push_back(TexHandle);
04106     Ops.push_back(N->getOperand(2));
04107     Ops.push_back(N->getOperand(3));
04108     Ops.push_back(Chain);
04109     break;
04110   case NVPTXISD::Suld2DV2I16Trap:
04111     Opc = NVPTX::SULD_2D_V2I16_TRAP;
04112     Ops.push_back(TexHandle);
04113     Ops.push_back(N->getOperand(2));
04114     Ops.push_back(N->getOperand(3));
04115     Ops.push_back(Chain);
04116     break;
04117   case NVPTXISD::Suld2DV2I32Trap:
04118     Opc = NVPTX::SULD_2D_V2I32_TRAP;
04119     Ops.push_back(TexHandle);
04120     Ops.push_back(N->getOperand(2));
04121     Ops.push_back(N->getOperand(3));
04122     Ops.push_back(Chain);
04123     break;
04124   case NVPTXISD::Suld2DV2I64Trap:
04125     Opc = NVPTX::SULD_2D_V2I64_TRAP;
04126     Ops.push_back(TexHandle);
04127     Ops.push_back(N->getOperand(2));
04128     Ops.push_back(N->getOperand(3));
04129     Ops.push_back(Chain);
04130     break;
04131   case NVPTXISD::Suld2DV4I8Trap:
04132     Opc = NVPTX::SULD_2D_V4I8_TRAP;
04133     Ops.push_back(TexHandle);
04134     Ops.push_back(N->getOperand(2));
04135     Ops.push_back(N->getOperand(3));
04136     Ops.push_back(Chain);
04137     break;
04138   case NVPTXISD::Suld2DV4I16Trap:
04139     Opc = NVPTX::SULD_2D_V4I16_TRAP;
04140     Ops.push_back(TexHandle);
04141     Ops.push_back(N->getOperand(2));
04142     Ops.push_back(N->getOperand(3));
04143     Ops.push_back(Chain);
04144     break;
04145   case NVPTXISD::Suld2DV4I32Trap:
04146     Opc = NVPTX::SULD_2D_V4I32_TRAP;
04147     Ops.push_back(TexHandle);
04148     Ops.push_back(N->getOperand(2));
04149     Ops.push_back(N->getOperand(3));
04150     Ops.push_back(Chain);
04151     break;
04152   case NVPTXISD::Suld2DArrayI8Trap:
04153     Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
04154     Ops.push_back(TexHandle);
04155     Ops.push_back(N->getOperand(2));
04156     Ops.push_back(N->getOperand(3));
04157     Ops.push_back(N->getOperand(4));
04158     Ops.push_back(Chain);
04159     break;
04160   case NVPTXISD::Suld2DArrayI16Trap:
04161     Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
04162     Ops.push_back(TexHandle);
04163     Ops.push_back(N->getOperand(2));
04164     Ops.push_back(N->getOperand(3));
04165     Ops.push_back(N->getOperand(4));
04166     Ops.push_back(Chain);
04167     break;
04168   case NVPTXISD::Suld2DArrayI32Trap:
04169     Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
04170     Ops.push_back(TexHandle);
04171     Ops.push_back(N->getOperand(2));
04172     Ops.push_back(N->getOperand(3));
04173     Ops.push_back(N->getOperand(4));
04174     Ops.push_back(Chain);
04175     break;
04176   case NVPTXISD::Suld2DArrayI64Trap:
04177     Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
04178     Ops.push_back(TexHandle);
04179     Ops.push_back(N->getOperand(2));
04180     Ops.push_back(N->getOperand(3));
04181     Ops.push_back(N->getOperand(4));
04182     Ops.push_back(Chain);
04183     break;
04184   case NVPTXISD::Suld2DArrayV2I8Trap:
04185     Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
04186     Ops.push_back(TexHandle);
04187     Ops.push_back(N->getOperand(2));
04188     Ops.push_back(N->getOperand(3));
04189     Ops.push_back(N->getOperand(4));
04190     Ops.push_back(Chain);
04191     break;
04192   case NVPTXISD::Suld2DArrayV2I16Trap:
04193     Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
04194     Ops.push_back(TexHandle);
04195     Ops.push_back(N->getOperand(2));
04196     Ops.push_back(N->getOperand(3));
04197     Ops.push_back(N->getOperand(4));
04198     Ops.push_back(Chain);
04199     break;
04200   case NVPTXISD::Suld2DArrayV2I32Trap:
04201     Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
04202     Ops.push_back(TexHandle);
04203     Ops.push_back(N->getOperand(2));
04204     Ops.push_back(N->getOperand(3));
04205     Ops.push_back(N->getOperand(4));
04206     Ops.push_back(Chain);
04207     break;
04208   case NVPTXISD::Suld2DArrayV2I64Trap:
04209     Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
04210     Ops.push_back(TexHandle);
04211     Ops.push_back(N->getOperand(2));
04212     Ops.push_back(N->getOperand(3));
04213     Ops.push_back(N->getOperand(4));
04214     Ops.push_back(Chain);
04215     break;
04216   case NVPTXISD::Suld2DArrayV4I8Trap:
04217     Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
04218     Ops.push_back(TexHandle);
04219     Ops.push_back(N->getOperand(2));
04220     Ops.push_back(N->getOperand(3));
04221     Ops.push_back(N->getOperand(4));
04222     Ops.push_back(Chain);
04223     break;
04224   case NVPTXISD::Suld2DArrayV4I16Trap:
04225     Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
04226     Ops.push_back(TexHandle);
04227     Ops.push_back(N->getOperand(2));
04228     Ops.push_back(N->getOperand(3));
04229     Ops.push_back(N->getOperand(4));
04230     Ops.push_back(Chain);
04231     break;
04232   case NVPTXISD::Suld2DArrayV4I32Trap:
04233     Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
04234     Ops.push_back(TexHandle);
04235     Ops.push_back(N->getOperand(2));
04236     Ops.push_back(N->getOperand(3));
04237     Ops.push_back(N->getOperand(4));
04238     Ops.push_back(Chain);
04239     break;
04240   case NVPTXISD::Suld3DI8Trap:
04241     Opc = NVPTX::SULD_3D_I8_TRAP;
04242     Ops.push_back(TexHandle);
04243     Ops.push_back(N->getOperand(2));
04244     Ops.push_back(N->getOperand(3));
04245     Ops.push_back(N->getOperand(4));
04246     Ops.push_back(Chain);
04247     break;
04248   case NVPTXISD::Suld3DI16Trap:
04249     Opc = NVPTX::SULD_3D_I16_TRAP;
04250     Ops.push_back(TexHandle);
04251     Ops.push_back(N->getOperand(2));
04252     Ops.push_back(N->getOperand(3));
04253     Ops.push_back(N->getOperand(4));
04254     Ops.push_back(Chain);
04255     break;
04256   case NVPTXISD::Suld3DI32Trap:
04257     Opc = NVPTX::SULD_3D_I32_TRAP;
04258     Ops.push_back(TexHandle);
04259     Ops.push_back(N->getOperand(2));
04260     Ops.push_back(N->getOperand(3));
04261     Ops.push_back(N->getOperand(4));
04262     Ops.push_back(Chain);
04263     break;
04264   case NVPTXISD::Suld3DI64Trap:
04265     Opc = NVPTX::SULD_3D_I64_TRAP;
04266     Ops.push_back(TexHandle);
04267     Ops.push_back(N->getOperand(2));
04268     Ops.push_back(N->getOperand(3));
04269     Ops.push_back(N->getOperand(4));
04270     Ops.push_back(Chain);
04271     break;
04272   case NVPTXISD::Suld3DV2I8Trap:
04273     Opc = NVPTX::SULD_3D_V2I8_TRAP;
04274     Ops.push_back(TexHandle);
04275     Ops.push_back(N->getOperand(2));
04276     Ops.push_back(N->getOperand(3));
04277     Ops.push_back(N->getOperand(4));
04278     Ops.push_back(Chain);
04279     break;
04280   case NVPTXISD::Suld3DV2I16Trap:
04281     Opc = NVPTX::SULD_3D_V2I16_TRAP;
04282     Ops.push_back(TexHandle);
04283     Ops.push_back(N->getOperand(2));
04284     Ops.push_back(N->getOperand(3));
04285     Ops.push_back(N->getOperand(4));
04286     Ops.push_back(Chain);
04287     break;
04288   case NVPTXISD::Suld3DV2I32Trap:
04289     Opc = NVPTX::SULD_3D_V2I32_TRAP;
04290     Ops.push_back(TexHandle);
04291     Ops.push_back(N->getOperand(2));
04292     Ops.push_back(N->getOperand(3));
04293     Ops.push_back(N->getOperand(4));
04294     Ops.push_back(Chain);
04295     break;
04296   case NVPTXISD::Suld3DV2I64Trap:
04297     Opc = NVPTX::SULD_3D_V2I64_TRAP;
04298     Ops.push_back(TexHandle);
04299     Ops.push_back(N->getOperand(2));
04300     Ops.push_back(N->getOperand(3));
04301     Ops.push_back(N->getOperand(4));
04302     Ops.push_back(Chain);
04303     break;
04304   case NVPTXISD::Suld3DV4I8Trap:
04305     Opc = NVPTX::SULD_3D_V4I8_TRAP;
04306     Ops.push_back(TexHandle);
04307     Ops.push_back(N->getOperand(2));
04308     Ops.push_back(N->getOperand(3));
04309     Ops.push_back(N->getOperand(4));
04310     Ops.push_back(Chain);
04311     break;
04312   case NVPTXISD::Suld3DV4I16Trap:
04313     Opc = NVPTX::SULD_3D_V4I16_TRAP;
04314     Ops.push_back(TexHandle);
04315     Ops.push_back(N->getOperand(2));
04316     Ops.push_back(N->getOperand(3));
04317     Ops.push_back(N->getOperand(4));
04318     Ops.push_back(Chain);
04319     break;
04320   case NVPTXISD::Suld3DV4I32Trap:
04321     Opc = NVPTX::SULD_3D_V4I32_TRAP;
04322     Ops.push_back(TexHandle);
04323     Ops.push_back(N->getOperand(2));
04324     Ops.push_back(N->getOperand(3));
04325     Ops.push_back(N->getOperand(4));
04326     Ops.push_back(Chain);
04327     break;
04328   case NVPTXISD::Suld1DI8Zero:
04329     Opc = NVPTX::SULD_1D_I8_ZERO;
04330     Ops.push_back(TexHandle);
04331     Ops.push_back(N->getOperand(2));
04332     Ops.push_back(Chain);
04333     break;
04334   case NVPTXISD::Suld1DI16Zero:
04335     Opc = NVPTX::SULD_1D_I16_ZERO;
04336     Ops.push_back(TexHandle);
04337     Ops.push_back(N->getOperand(2));
04338     Ops.push_back(Chain);
04339     break;
04340   case NVPTXISD::Suld1DI32Zero:
04341     Opc = NVPTX::SULD_1D_I32_ZERO;
04342     Ops.push_back(TexHandle);
04343     Ops.push_back(N->getOperand(2));
04344     Ops.push_back(Chain);
04345     break;
04346   case NVPTXISD::Suld1DI64Zero:
04347     Opc = NVPTX::SULD_1D_I64_ZERO;
04348     Ops.push_back(TexHandle);
04349     Ops.push_back(N->getOperand(2));
04350     Ops.push_back(Chain);
04351     break;
04352   case NVPTXISD::Suld1DV2I8Zero:
04353     Opc = NVPTX::SULD_1D_V2I8_ZERO;
04354     Ops.push_back(TexHandle);
04355     Ops.push_back(N->getOperand(2));
04356     Ops.push_back(Chain);
04357     break;
04358   case NVPTXISD::Suld1DV2I16Zero:
04359     Opc = NVPTX::SULD_1D_V2I16_ZERO;
04360     Ops.push_back(TexHandle);
04361     Ops.push_back(N->getOperand(2));
04362     Ops.push_back(Chain);
04363     break;
04364   case NVPTXISD::Suld1DV2I32Zero:
04365     Opc = NVPTX::SULD_1D_V2I32_ZERO;
04366     Ops.push_back(TexHandle);
04367     Ops.push_back(N->getOperand(2));
04368     Ops.push_back(Chain);
04369     break;
04370   case NVPTXISD::Suld1DV2I64Zero:
04371     Opc = NVPTX::SULD_1D_V2I64_ZERO;
04372     Ops.push_back(TexHandle);
04373     Ops.push_back(N->getOperand(2));
04374     Ops.push_back(Chain);
04375     break;
04376   case NVPTXISD::Suld1DV4I8Zero:
04377     Opc = NVPTX::SULD_1D_V4I8_ZERO;
04378     Ops.push_back(TexHandle);
04379     Ops.push_back(N->getOperand(2));
04380     Ops.push_back(Chain);
04381     break;
04382   case NVPTXISD::Suld1DV4I16Zero:
04383     Opc = NVPTX::SULD_1D_V4I16_ZERO;
04384     Ops.push_back(TexHandle);
04385     Ops.push_back(N->getOperand(2));
04386     Ops.push_back(Chain);
04387     break;
04388   case NVPTXISD::Suld1DV4I32Zero:
04389     Opc = NVPTX::SULD_1D_V4I32_ZERO;
04390     Ops.push_back(TexHandle);
04391     Ops.push_back(N->getOperand(2));
04392     Ops.push_back(Chain);
04393     break;
04394   case NVPTXISD::Suld1DArrayI8Zero:
04395     Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
04396     Ops.push_back(TexHandle);
04397     Ops.push_back(N->getOperand(2));
04398     Ops.push_back(N->getOperand(3));
04399     Ops.push_back(Chain);
04400     break;
04401   case NVPTXISD::Suld1DArrayI16Zero:
04402     Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
04403     Ops.push_back(TexHandle);
04404     Ops.push_back(N->getOperand(2));
04405     Ops.push_back(N->getOperand(3));
04406     Ops.push_back(Chain);
04407     break;
04408   case NVPTXISD::Suld1DArrayI32Zero:
04409     Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
04410     Ops.push_back(TexHandle);
04411     Ops.push_back(N->getOperand(2));
04412     Ops.push_back(N->getOperand(3));
04413     Ops.push_back(Chain);
04414     break;
04415   case NVPTXISD::Suld1DArrayI64Zero:
04416     Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
04417     Ops.push_back(TexHandle);
04418     Ops.push_back(N->getOperand(2));
04419     Ops.push_back(N->getOperand(3));
04420     Ops.push_back(Chain);
04421     break;
04422   case NVPTXISD::Suld1DArrayV2I8Zero:
04423     Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
04424     Ops.push_back(TexHandle);
04425     Ops.push_back(N->getOperand(2));
04426     Ops.push_back(N->getOperand(3));
04427     Ops.push_back(Chain);
04428     break;
04429   case NVPTXISD::Suld1DArrayV2I16Zero:
04430     Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
04431     Ops.push_back(TexHandle);
04432     Ops.push_back(N->getOperand(2));
04433     Ops.push_back(N->getOperand(3));
04434     Ops.push_back(Chain);
04435     break;
04436   case NVPTXISD::Suld1DArrayV2I32Zero:
04437     Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
04438     Ops.push_back(TexHandle);
04439     Ops.push_back(N->getOperand(2));
04440     Ops.push_back(N->getOperand(3));
04441     Ops.push_back(Chain);
04442     break;
04443   case NVPTXISD::Suld1DArrayV2I64Zero:
04444     Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
04445     Ops.push_back(TexHandle);
04446     Ops.push_back(N->getOperand(2));
04447     Ops.push_back(N->getOperand(3));
04448     Ops.push_back(Chain);
04449     break;
04450   case NVPTXISD::Suld1DArrayV4I8Zero:
04451     Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
04452     Ops.push_back(TexHandle);
04453     Ops.push_back(N->getOperand(2));
04454     Ops.push_back(N->getOperand(3));
04455     Ops.push_back(Chain);
04456     break;
04457   case NVPTXISD::Suld1DArrayV4I16Zero:
04458     Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
04459     Ops.push_back(TexHandle);
04460     Ops.push_back(N->getOperand(2));
04461     Ops.push_back(N->getOperand(3));
04462     Ops.push_back(Chain);
04463     break;
04464   case NVPTXISD::Suld1DArrayV4I32Zero:
04465     Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
04466     Ops.push_back(TexHandle);
04467     Ops.push_back(N->getOperand(2));
04468     Ops.push_back(N->getOperand(3));
04469     Ops.push_back(Chain);
04470     break;
04471   case NVPTXISD::Suld2DI8Zero:
04472     Opc = NVPTX::SULD_2D_I8_ZERO;
04473     Ops.push_back(TexHandle);
04474     Ops.push_back(N->getOperand(2));
04475     Ops.push_back(N->getOperand(3));
04476     Ops.push_back(Chain);
04477     break;
04478   case NVPTXISD::Suld2DI16Zero:
04479     Opc = NVPTX::SULD_2D_I16_ZERO;
04480     Ops.push_back(TexHandle);
04481     Ops.push_back(N->getOperand(2));
04482     Ops.push_back(N->getOperand(3));
04483     Ops.push_back(Chain);
04484     break;
04485   case NVPTXISD::Suld2DI32Zero:
04486     Opc = NVPTX::SULD_2D_I32_ZERO;
04487     Ops.push_back(TexHandle);
04488     Ops.push_back(N->getOperand(2));
04489     Ops.push_back(N->getOperand(3));
04490     Ops.push_back(Chain);
04491     break;
04492   case NVPTXISD::Suld2DI64Zero:
04493     Opc = NVPTX::SULD_2D_I64_ZERO;
04494     Ops.push_back(TexHandle);
04495     Ops.push_back(N->getOperand(2));
04496     Ops.push_back(N->getOperand(3));
04497     Ops.push_back(Chain);
04498     break;
04499   case NVPTXISD::Suld2DV2I8Zero:
04500     Opc = NVPTX::SULD_2D_V2I8_ZERO;
04501     Ops.push_back(TexHandle);
04502     Ops.push_back(N->getOperand(2));
04503     Ops.push_back(N->getOperand(3));
04504     Ops.push_back(Chain);
04505     break;
04506   case NVPTXISD::Suld2DV2I16Zero:
04507     Opc = NVPTX::SULD_2D_V2I16_ZERO;
04508     Ops.push_back(TexHandle);
04509     Ops.push_back(N->getOperand(2));
04510     Ops.push_back(N->getOperand(3));
04511     Ops.push_back(Chain);
04512     break;
04513   case NVPTXISD::Suld2DV2I32Zero:
04514     Opc = NVPTX::SULD_2D_V2I32_ZERO;
04515     Ops.push_back(TexHandle);
04516     Ops.push_back(N->getOperand(2));
04517     Ops.push_back(N->getOperand(3));
04518     Ops.push_back(Chain);
04519     break;
04520   case NVPTXISD::Suld2DV2I64Zero:
04521     Opc = NVPTX::SULD_2D_V2I64_ZERO;
04522     Ops.push_back(TexHandle);
04523     Ops.push_back(N->getOperand(2));
04524     Ops.push_back(N->getOperand(3));
04525     Ops.push_back(Chain);
04526     break;
04527   case NVPTXISD::Suld2DV4I8Zero:
04528     Opc = NVPTX::SULD_2D_V4I8_ZERO;
04529     Ops.push_back(TexHandle);
04530     Ops.push_back(N->getOperand(2));
04531     Ops.push_back(N->getOperand(3));
04532     Ops.push_back(Chain);
04533     break;
04534   case NVPTXISD::Suld2DV4I16Zero:
04535     Opc = NVPTX::SULD_2D_V4I16_ZERO;
04536     Ops.push_back(TexHandle);
04537     Ops.push_back(N->getOperand(2));
04538     Ops.push_back(N->getOperand(3));
04539     Ops.push_back(Chain);
04540     break;
04541   case NVPTXISD::Suld2DV4I32Zero:
04542     Opc = NVPTX::SULD_2D_V4I32_ZERO;
04543     Ops.push_back(TexHandle);
04544     Ops.push_back(N->getOperand(2));
04545     Ops.push_back(N->getOperand(3));
04546     Ops.push_back(Chain);
04547     break;
04548   case NVPTXISD::Suld2DArrayI8Zero:
04549     Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
04550     Ops.push_back(TexHandle);
04551     Ops.push_back(N->getOperand(2));
04552     Ops.push_back(N->getOperand(3));
04553     Ops.push_back(N->getOperand(4));
04554     Ops.push_back(Chain);
04555     break;
04556   case NVPTXISD::Suld2DArrayI16Zero:
04557     Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
04558     Ops.push_back(TexHandle);
04559     Ops.push_back(N->getOperand(2));
04560     Ops.push_back(N->getOperand(3));
04561     Ops.push_back(N->getOperand(4));
04562     Ops.push_back(Chain);
04563     break;
04564   case NVPTXISD::Suld2DArrayI32Zero:
04565     Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
04566     Ops.push_back(TexHandle);
04567     Ops.push_back(N->getOperand(2));
04568     Ops.push_back(N->getOperand(3));
04569     Ops.push_back(N->getOperand(4));
04570     Ops.push_back(Chain);
04571     break;
04572   case NVPTXISD::Suld2DArrayI64Zero:
04573     Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
04574     Ops.push_back(TexHandle);
04575     Ops.push_back(N->getOperand(2));
04576     Ops.push_back(N->getOperand(3));
04577     Ops.push_back(N->getOperand(4));
04578     Ops.push_back(Chain);
04579     break;
04580   case NVPTXISD::Suld2DArrayV2I8Zero:
04581     Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
04582     Ops.push_back(TexHandle);
04583     Ops.push_back(N->getOperand(2));
04584     Ops.push_back(N->getOperand(3));
04585     Ops.push_back(N->getOperand(4));
04586     Ops.push_back(Chain);
04587     break;
04588   case NVPTXISD::Suld2DArrayV2I16Zero:
04589     Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
04590     Ops.push_back(TexHandle);
04591     Ops.push_back(N->getOperand(2));
04592     Ops.push_back(N->getOperand(3));
04593     Ops.push_back(N->getOperand(4));
04594     Ops.push_back(Chain);
04595     break;
04596   case NVPTXISD::Suld2DArrayV2I32Zero:
04597     Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
04598     Ops.push_back(TexHandle);
04599     Ops.push_back(N->getOperand(2));
04600     Ops.push_back(N->getOperand(3));
04601     Ops.push_back(N->getOperand(4));
04602     Ops.push_back(Chain);
04603     break;
04604   case NVPTXISD::Suld2DArrayV2I64Zero:
04605     Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
04606     Ops.push_back(TexHandle);
04607     Ops.push_back(N->getOperand(2));
04608     Ops.push_back(N->getOperand(3));
04609     Ops.push_back(N->getOperand(4));
04610     Ops.push_back(Chain);
04611     break;
04612   case NVPTXISD::Suld2DArrayV4I8Zero:
04613     Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
04614     Ops.push_back(TexHandle);
04615     Ops.push_back(N->getOperand(2));
04616     Ops.push_back(N->getOperand(3));
04617     Ops.push_back(N->getOperand(4));
04618     Ops.push_back(Chain);
04619     break;
04620   case NVPTXISD::Suld2DArrayV4I16Zero:
04621     Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
04622     Ops.push_back(TexHandle);
04623     Ops.push_back(N->getOperand(2));
04624     Ops.push_back(N->getOperand(3));
04625     Ops.push_back(N->getOperand(4));
04626     Ops.push_back(Chain);
04627     break;
04628   case NVPTXISD::Suld2DArrayV4I32Zero:
04629     Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
04630     Ops.push_back(TexHandle);
04631     Ops.push_back(N->getOperand(2));
04632     Ops.push_back(N->getOperand(3));
04633     Ops.push_back(N->getOperand(4));
04634     Ops.push_back(Chain);
04635     break;
04636   case NVPTXISD::Suld3DI8Zero:
04637     Opc = NVPTX::SULD_3D_I8_ZERO;
04638     Ops.push_back(TexHandle);
04639     Ops.push_back(N->getOperand(2));
04640     Ops.push_back(N->getOperand(3));
04641     Ops.push_back(N->getOperand(4));
04642     Ops.push_back(Chain);
04643     break;
04644   case NVPTXISD::Suld3DI16Zero:
04645     Opc = NVPTX::SULD_3D_I16_ZERO;
04646     Ops.push_back(TexHandle);
04647     Ops.push_back(N->getOperand(2));
04648     Ops.push_back(N->getOperand(3));
04649     Ops.push_back(N->getOperand(4));
04650     Ops.push_back(Chain);
04651     break;
04652   case NVPTXISD::Suld3DI32Zero:
04653     Opc = NVPTX::SULD_3D_I32_ZERO;
04654     Ops.push_back(TexHandle);
04655     Ops.push_back(N->getOperand(2));
04656     Ops.push_back(N->getOperand(3));
04657     Ops.push_back(N->getOperand(4));
04658     Ops.push_back(Chain);
04659     break;
04660   case NVPTXISD::Suld3DI64Zero:
04661     Opc = NVPTX::SULD_3D_I64_ZERO;
04662     Ops.push_back(TexHandle);
04663     Ops.push_back(N->getOperand(2));
04664     Ops.push_back(N->getOperand(3));
04665     Ops.push_back(N->getOperand(4));
04666     Ops.push_back(Chain);
04667     break;
04668   case NVPTXISD::Suld3DV2I8Zero:
04669     Opc = NVPTX::SULD_3D_V2I8_ZERO;
04670     Ops.push_back(TexHandle);
04671     Ops.push_back(N->getOperand(2));
04672     Ops.push_back(N->getOperand(3));
04673     Ops.push_back(N->getOperand(4));
04674     Ops.push_back(Chain);
04675     break;
04676   case NVPTXISD::Suld3DV2I16Zero:
04677     Opc = NVPTX::SULD_3D_V2I16_ZERO;
04678     Ops.push_back(TexHandle);
04679     Ops.push_back(N->getOperand(2));
04680     Ops.push_back(N->getOperand(3));
04681     Ops.push_back(N->getOperand(4));
04682     Ops.push_back(Chain);
04683     break;
04684   case NVPTXISD::Suld3DV2I32Zero:
04685     Opc = NVPTX::SULD_3D_V2I32_ZERO;
04686     Ops.push_back(TexHandle);
04687     Ops.push_back(N->getOperand(2));
04688     Ops.push_back(N->getOperand(3));
04689     Ops.push_back(N->getOperand(4));
04690     Ops.push_back(Chain);
04691     break;
04692   case NVPTXISD::Suld3DV2I64Zero:
04693     Opc = NVPTX::SULD_3D_V2I64_ZERO;
04694     Ops.push_back(TexHandle);
04695     Ops.push_back(N->getOperand(2));
04696     Ops.push_back(N->getOperand(3));
04697     Ops.push_back(N->getOperand(4));
04698     Ops.push_back(Chain);
04699     break;
04700   case NVPTXISD::Suld3DV4I8Zero:
04701     Opc = NVPTX::SULD_3D_V4I8_ZERO;
04702     Ops.push_back(TexHandle);
04703     Ops.push_back(N->getOperand(2));
04704     Ops.push_back(N->getOperand(3));
04705     Ops.push_back(N->getOperand(4));
04706     Ops.push_back(Chain);
04707     break;
04708   case NVPTXISD::Suld3DV4I16Zero:
04709     Opc = NVPTX::SULD_3D_V4I16_ZERO;
04710     Ops.push_back(TexHandle);
04711     Ops.push_back(N->getOperand(2));
04712     Ops.push_back(N->getOperand(3));
04713     Ops.push_back(N->getOperand(4));
04714     Ops.push_back(Chain);
04715     break;
04716   case NVPTXISD::Suld3DV4I32Zero:
04717     Opc = NVPTX::SULD_3D_V4I32_ZERO;
04718     Ops.push_back(TexHandle);
04719     Ops.push_back(N->getOperand(2));
04720     Ops.push_back(N->getOperand(3));
04721     Ops.push_back(N->getOperand(4));
04722     Ops.push_back(Chain);
04723     break;
04724   }
04725   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
04726   return Ret;
04727 }
04728 
04729 
04730 /// SelectBFE - Look for instruction sequences that can be made more efficient
04731 /// by using the 'bfe' (bit-field extract) PTX instruction
04732 SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
04733   SDValue LHS = N->getOperand(0);
04734   SDValue RHS = N->getOperand(1);
04735   SDValue Len;
04736   SDValue Start;
04737   SDValue Val;
04738   bool IsSigned = false;
04739 
04740   if (N->getOpcode() == ISD::AND) {
04741     // Canonicalize the operands
04742     // We want 'and %val, %mask'
04743     if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
04744       std::swap(LHS, RHS);
04745     }
04746 
04747     ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
04748     if (!Mask) {
04749       // We need a constant mask on the RHS of the AND
04750       return NULL;
04751     }
04752 
04753     // Extract the mask bits
04754     uint64_t MaskVal = Mask->getZExtValue();
04755     if (!isMask_64(MaskVal)) {
04756       // We *could* handle shifted masks here, but doing so would require an
04757       // 'and' operation to fix up the low-order bits so we would trade
04758       // shr+and for bfe+and, which has the same throughput
04759       return NULL;
04760     }
04761 
04762     // How many bits are in our mask?
04763     uint64_t NumBits = countTrailingOnes(MaskVal);
04764     Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
04765 
04766     if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
04767       // We have a 'srl/and' pair, extract the effective start bit and length
04768       Val = LHS.getNode()->getOperand(0);
04769       Start = LHS.getNode()->getOperand(1);
04770       ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
04771       if (StartConst) {
04772         uint64_t StartVal = StartConst->getZExtValue();
04773         // How many "good" bits do we have left?  "good" is defined here as bits
04774         // that exist in the original value, not shifted in.
04775         uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
04776         if (NumBits > GoodBits) {
04777           // Do not handle the case where bits have been shifted in. In theory
04778           // we could handle this, but the cost is likely higher than just
04779           // emitting the srl/and pair.
04780           return NULL;
04781         }
04782         Start = CurDAG->getTargetConstant(StartVal, MVT::i32);
04783       } else {
04784         // Do not handle the case where the shift amount (can be zero if no srl
04785         // was found) is not constant. We could handle this case, but it would
04786         // require run-time logic that would be more expensive than just
04787         // emitting the srl/and pair.
04788         return NULL;
04789       }
04790     } else {
04791       // Do not handle the case where the LHS of the and is not a shift. While
04792       // it would be trivial to handle this case, it would just transform
04793       // 'and' -> 'bfe', but 'and' has higher-throughput.
04794       return NULL;
04795     }
04796   } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
04797     if (LHS->getOpcode() == ISD::AND) {
04798       ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
04799       if (!ShiftCnst) {
04800         // Shift amount must be constant
04801         return NULL;
04802       }
04803 
04804       uint64_t ShiftAmt = ShiftCnst->getZExtValue();
04805 
04806       SDValue AndLHS = LHS->getOperand(0);
04807       SDValue AndRHS = LHS->getOperand(1);
04808 
04809       // Canonicalize the AND to have the mask on the RHS
04810       if (isa<ConstantSDNode>(AndLHS)) {
04811         std::swap(AndLHS, AndRHS);
04812       }
04813 
04814       ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
04815       if (!MaskCnst) {
04816         // Mask must be constant
04817         return NULL;
04818       }
04819 
04820       uint64_t MaskVal = MaskCnst->getZExtValue();
04821       uint64_t NumZeros;
04822       uint64_t NumBits;
04823       if (isMask_64(MaskVal)) {
04824         NumZeros = 0;
04825         // The number of bits in the result bitfield will be the number of
04826         // trailing ones (the AND) minus the number of bits we shift off
04827         NumBits = countTrailingOnes(MaskVal) - ShiftAmt;
04828       } else if (isShiftedMask_64(MaskVal)) {
04829         NumZeros = countTrailingZeros(MaskVal);
04830         unsigned NumOnes = countTrailingOnes(MaskVal >> NumZeros);
04831         // The number of bits in the result bitfield will be the number of
04832         // trailing zeros plus the number of set bits in the mask minus the
04833         // number of bits we shift off
04834         NumBits = NumZeros + NumOnes - ShiftAmt;
04835       } else {
04836         // This is not a mask we can handle
04837         return NULL;
04838       }
04839 
04840       if (ShiftAmt < NumZeros) {
04841         // Handling this case would require extra logic that would make this
04842         // transformation non-profitable
04843         return NULL;
04844       }
04845 
04846       Val = AndLHS;
04847       Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32);
04848       Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
04849     } else if (LHS->getOpcode() == ISD::SHL) {
04850       // Here, we have a pattern like:
04851       //
04852       // (sra (shl val, NN), MM)
04853       // or
04854       // (srl (shl val, NN), MM)
04855       //
04856       // If MM >= NN, we can efficiently optimize this with bfe
04857       Val = LHS->getOperand(0);
04858 
04859       SDValue ShlRHS = LHS->getOperand(1);
04860       ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
04861       if (!ShlCnst) {
04862         // Shift amount must be constant
04863         return NULL;
04864       }
04865       uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
04866 
04867       SDValue ShrRHS = RHS;
04868       ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
04869       if (!ShrCnst) {
04870         // Shift amount must be constant
04871         return NULL;
04872       }
04873       uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
04874 
04875       // To avoid extra codegen and be profitable, we need Outer >= Inner
04876       if (OuterShiftAmt < InnerShiftAmt) {
04877         return NULL;
04878       }
04879 
04880       // If the outer shift is more than the type size, we have no bitfield to
04881       // extract (since we also check that the inner shift is <= the outer shift
04882       // then this also implies that the inner shift is < the type size)
04883       if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
04884         return NULL;
04885       }
04886 
04887       Start =
04888         CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32);
04889       Len =
04890         CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
04891                                   OuterShiftAmt, MVT::i32);
04892 
04893       if (N->getOpcode() == ISD::SRA) {
04894         // If we have a arithmetic right shift, we need to use the signed bfe
04895         // variant
04896         IsSigned = true;
04897       }
04898     } else {
04899       // No can do...
04900       return NULL;
04901     }
04902   } else {
04903     // No can do...
04904     return NULL;
04905   }
04906 
04907 
04908   unsigned Opc;
04909   // For the BFE operations we form here from "and" and "srl", always use the
04910   // unsigned variants.
04911   if (Val.getValueType() == MVT::i32) {
04912     if (IsSigned) {
04913       Opc = NVPTX::BFE_S32rii;
04914     } else {
04915       Opc = NVPTX::BFE_U32rii;
04916     }
04917   } else if (Val.getValueType() == MVT::i64) {
04918     if (IsSigned) {
04919       Opc = NVPTX::BFE_S64rii;
04920     } else {
04921       Opc = NVPTX::BFE_U64rii;
04922     }
04923   } else {
04924     // We cannot handle this type
04925     return NULL;
04926   }
04927 
04928   SDValue Ops[] = {
04929     Val, Start, Len
04930   };
04931 
04932   SDNode *Ret =
04933     CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
04934 
04935   return Ret;
04936 }
04937 
04938 // SelectDirectAddr - Match a direct address for DAG.
04939 // A direct address could be a globaladdress or externalsymbol.
04940 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
04941   // Return true if TGA or ES.
04942   if (N.getOpcode() == ISD::TargetGlobalAddress ||
04943       N.getOpcode() == ISD::TargetExternalSymbol) {
04944     Address = N;
04945     return true;
04946   }
04947   if (N.getOpcode() == NVPTXISD::Wrapper) {
04948     Address = N.getOperand(0);
04949     return true;
04950   }
04951   if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
04952     unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
04953     if (IID == Intrinsic::nvvm_ptr_gen_to_param)
04954       if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
04955         return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
04956   }
04957   return false;
04958 }
04959 
04960 // symbol+offset
04961 bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
04962     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
04963   if (Addr.getOpcode() == ISD::ADD) {
04964     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
04965       SDValue base = Addr.getOperand(0);
04966       if (SelectDirectAddr(base, Base)) {
04967         Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
04968         return true;
04969       }
04970     }
04971   }
04972   return false;
04973 }
04974 
04975 // symbol+offset
04976 bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
04977                                      SDValue &Base, SDValue &Offset) {
04978   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
04979 }
04980 
04981 // symbol+offset
04982 bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
04983                                        SDValue &Base, SDValue &Offset) {
04984   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
04985 }
04986 
04987 // register+offset
04988 bool NVPTXDAGToDAGISel::SelectADDRri_imp(
04989     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
04990   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
04991     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
04992     Offset = CurDAG->getTargetConstant(0, mvt);
04993     return true;
04994   }
04995   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
04996       Addr.getOpcode() == ISD::TargetGlobalAddress)
04997     return false; // direct calls.
04998 
04999   if (Addr.getOpcode() == ISD::ADD) {
05000     if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
05001       return false;
05002     }
05003     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
05004       if (FrameIndexSDNode *FIN =
05005               dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
05006         // Constant offset from frame ref.
05007         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
05008       else
05009         Base = Addr.getOperand(0);
05010       Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
05011       return true;
05012     }
05013   }
05014   return false;
05015 }
05016 
05017 // register+offset
05018 bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
05019                                      SDValue &Base, SDValue &Offset) {
05020   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
05021 }
05022 
05023 // register+offset
05024 bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
05025                                        SDValue &Base, SDValue &Offset) {
05026   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
05027 }
05028 
05029 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
05030                                                  unsigned int spN) const {
05031   const Value *Src = nullptr;
05032   if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
05033     if (spN == 0 && mN->getMemOperand()->getPseudoValue())
05034       return true;
05035     Src = mN->getMemOperand()->getValue();
05036   }
05037   if (!Src)
05038     return false;
05039   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
05040     return (PT->getAddressSpace() == spN);
05041   return false;
05042 }
05043 
05044 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
05045 /// inline asm expressions.
05046 bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
05047     const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) {
05048   SDValue Op0, Op1;
05049   switch (ConstraintCode) {
05050   default:
05051     return true;
05052   case 'm': // memory
05053     if (SelectDirectAddr(Op, Op0)) {
05054       OutOps.push_back(Op0);
05055       OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
05056       return false;
05057     }
05058     if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
05059       OutOps.push_back(Op0);
05060       OutOps.push_back(Op1);
05061       return false;
05062     }
05063     break;
05064   }
05065   return true;
05066 }