LLVM  mainline
NVPTXISelDAGToDAG.cpp
Go to the documentation of this file.
00001 //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file defines an instruction selector for the NVPTX target.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "NVPTXISelDAGToDAG.h"
00015 #include "llvm/IR/GlobalValue.h"
00016 #include "llvm/IR/Instructions.h"
00017 #include "llvm/Support/CommandLine.h"
00018 #include "llvm/Support/Debug.h"
00019 #include "llvm/Support/ErrorHandling.h"
00020 #include "llvm/Support/raw_ostream.h"
00021 #include "llvm/Target/TargetIntrinsicInfo.h"
00022 
00023 using namespace llvm;
00024 
00025 #define DEBUG_TYPE "nvptx-isel"
00026 
00027 static cl::opt<int> UsePrecDivF32(
00028     "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
00029     cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
00030              " IEEE Compliant F32 div.rnd if available."),
00031     cl::init(2));
00032 
00033 static cl::opt<bool>
00034 UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
00035           cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
00036           cl::init(true));
00037 
00038 static cl::opt<bool>
00039 FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
00040            cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
00041            cl::init(false));
00042 
00043 
00044 /// createNVPTXISelDag - This pass converts a legalized DAG into a
00045 /// NVPTX-specific DAG, ready for instruction scheduling.
00046 FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
00047                                        llvm::CodeGenOpt::Level OptLevel) {
00048   return new NVPTXDAGToDAGISel(TM, OptLevel);
00049 }
00050 
00051 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
00052                                      CodeGenOpt::Level OptLevel)
00053     : SelectionDAGISel(tm, OptLevel), TM(tm) {
00054   doMulWide = (OptLevel > 0);
00055 }
00056 
00057 bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
00058     Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget());
00059     return SelectionDAGISel::runOnMachineFunction(MF);
00060 }
00061 
00062 int NVPTXDAGToDAGISel::getDivF32Level() const {
00063   if (UsePrecDivF32.getNumOccurrences() > 0) {
00064     // If nvptx-prec-div32=N is used on the command-line, always honor it
00065     return UsePrecDivF32;
00066   } else {
00067     // Otherwise, use div.approx if fast math is enabled
00068     if (TM.Options.UnsafeFPMath)
00069       return 0;
00070     else
00071       return 2;
00072   }
00073 }
00074 
00075 bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
00076   if (UsePrecSqrtF32.getNumOccurrences() > 0) {
00077     // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
00078     return UsePrecSqrtF32;
00079   } else {
00080     // Otherwise, use sqrt.approx if fast math is enabled
00081     return !TM.Options.UnsafeFPMath;
00082   }
00083 }
00084 
00085 bool NVPTXDAGToDAGISel::useF32FTZ() const {
00086   if (FtzEnabled.getNumOccurrences() > 0) {
00087     // If nvptx-f32ftz is used on the command-line, always honor it
00088     return FtzEnabled;
00089   } else {
00090     const Function *F = MF->getFunction();
00091     // Otherwise, check for an nvptx-f32ftz attribute on the function
00092     if (F->hasFnAttribute("nvptx-f32ftz"))
00093       return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true";
00094     else
00095       return false;
00096   }
00097 }
00098 
00099 bool NVPTXDAGToDAGISel::allowFMA() const {
00100   const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
00101   return TL->allowFMA(*MF, OptLevel);
00102 }
00103 
00104 /// Select - Select instructions not customized! Used for
00105 /// expanded, promoted and normal instructions.
00106 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
00107 
00108   if (N->isMachineOpcode()) {
00109     N->setNodeId(-1);
00110     return nullptr; // Already selected.
00111   }
00112 
00113   SDNode *ResNode = nullptr;
00114   switch (N->getOpcode()) {
00115   case ISD::LOAD:
00116     ResNode = SelectLoad(N);
00117     break;
00118   case ISD::STORE:
00119     ResNode = SelectStore(N);
00120     break;
00121   case NVPTXISD::LoadV2:
00122   case NVPTXISD::LoadV4:
00123     ResNode = SelectLoadVector(N);
00124     break;
00125   case NVPTXISD::LDGV2:
00126   case NVPTXISD::LDGV4:
00127   case NVPTXISD::LDUV2:
00128   case NVPTXISD::LDUV4:
00129     ResNode = SelectLDGLDU(N);
00130     break;
00131   case NVPTXISD::StoreV2:
00132   case NVPTXISD::StoreV4:
00133     ResNode = SelectStoreVector(N);
00134     break;
00135   case NVPTXISD::LoadParam:
00136   case NVPTXISD::LoadParamV2:
00137   case NVPTXISD::LoadParamV4:
00138     ResNode = SelectLoadParam(N);
00139     break;
00140   case NVPTXISD::StoreRetval:
00141   case NVPTXISD::StoreRetvalV2:
00142   case NVPTXISD::StoreRetvalV4:
00143     ResNode = SelectStoreRetval(N);
00144     break;
00145   case NVPTXISD::StoreParam:
00146   case NVPTXISD::StoreParamV2:
00147   case NVPTXISD::StoreParamV4:
00148   case NVPTXISD::StoreParamS32:
00149   case NVPTXISD::StoreParamU32:
00150     ResNode = SelectStoreParam(N);
00151     break;
00152   case ISD::INTRINSIC_WO_CHAIN:
00153     ResNode = SelectIntrinsicNoChain(N);
00154     break;
00155   case ISD::INTRINSIC_W_CHAIN:
00156     ResNode = SelectIntrinsicChain(N);
00157     break;
00158   case NVPTXISD::Tex1DFloatS32:
00159   case NVPTXISD::Tex1DFloatFloat:
00160   case NVPTXISD::Tex1DFloatFloatLevel:
00161   case NVPTXISD::Tex1DFloatFloatGrad:
00162   case NVPTXISD::Tex1DS32S32:
00163   case NVPTXISD::Tex1DS32Float:
00164   case NVPTXISD::Tex1DS32FloatLevel:
00165   case NVPTXISD::Tex1DS32FloatGrad:
00166   case NVPTXISD::Tex1DU32S32:
00167   case NVPTXISD::Tex1DU32Float:
00168   case NVPTXISD::Tex1DU32FloatLevel:
00169   case NVPTXISD::Tex1DU32FloatGrad:
00170   case NVPTXISD::Tex1DArrayFloatS32:
00171   case NVPTXISD::Tex1DArrayFloatFloat:
00172   case NVPTXISD::Tex1DArrayFloatFloatLevel:
00173   case NVPTXISD::Tex1DArrayFloatFloatGrad:
00174   case NVPTXISD::Tex1DArrayS32S32:
00175   case NVPTXISD::Tex1DArrayS32Float:
00176   case NVPTXISD::Tex1DArrayS32FloatLevel:
00177   case NVPTXISD::Tex1DArrayS32FloatGrad:
00178   case NVPTXISD::Tex1DArrayU32S32:
00179   case NVPTXISD::Tex1DArrayU32Float:
00180   case NVPTXISD::Tex1DArrayU32FloatLevel:
00181   case NVPTXISD::Tex1DArrayU32FloatGrad:
00182   case NVPTXISD::Tex2DFloatS32:
00183   case NVPTXISD::Tex2DFloatFloat:
00184   case NVPTXISD::Tex2DFloatFloatLevel:
00185   case NVPTXISD::Tex2DFloatFloatGrad:
00186   case NVPTXISD::Tex2DS32S32:
00187   case NVPTXISD::Tex2DS32Float:
00188   case NVPTXISD::Tex2DS32FloatLevel:
00189   case NVPTXISD::Tex2DS32FloatGrad:
00190   case NVPTXISD::Tex2DU32S32:
00191   case NVPTXISD::Tex2DU32Float:
00192   case NVPTXISD::Tex2DU32FloatLevel:
00193   case NVPTXISD::Tex2DU32FloatGrad:
00194   case NVPTXISD::Tex2DArrayFloatS32:
00195   case NVPTXISD::Tex2DArrayFloatFloat:
00196   case NVPTXISD::Tex2DArrayFloatFloatLevel:
00197   case NVPTXISD::Tex2DArrayFloatFloatGrad:
00198   case NVPTXISD::Tex2DArrayS32S32:
00199   case NVPTXISD::Tex2DArrayS32Float:
00200   case NVPTXISD::Tex2DArrayS32FloatLevel:
00201   case NVPTXISD::Tex2DArrayS32FloatGrad:
00202   case NVPTXISD::Tex2DArrayU32S32:
00203   case NVPTXISD::Tex2DArrayU32Float:
00204   case NVPTXISD::Tex2DArrayU32FloatLevel:
00205   case NVPTXISD::Tex2DArrayU32FloatGrad:
00206   case NVPTXISD::Tex3DFloatS32:
00207   case NVPTXISD::Tex3DFloatFloat:
00208   case NVPTXISD::Tex3DFloatFloatLevel:
00209   case NVPTXISD::Tex3DFloatFloatGrad:
00210   case NVPTXISD::Tex3DS32S32:
00211   case NVPTXISD::Tex3DS32Float:
00212   case NVPTXISD::Tex3DS32FloatLevel:
00213   case NVPTXISD::Tex3DS32FloatGrad:
00214   case NVPTXISD::Tex3DU32S32:
00215   case NVPTXISD::Tex3DU32Float:
00216   case NVPTXISD::Tex3DU32FloatLevel:
00217   case NVPTXISD::Tex3DU32FloatGrad:
00218   case NVPTXISD::TexCubeFloatFloat:
00219   case NVPTXISD::TexCubeFloatFloatLevel:
00220   case NVPTXISD::TexCubeS32Float:
00221   case NVPTXISD::TexCubeS32FloatLevel:
00222   case NVPTXISD::TexCubeU32Float:
00223   case NVPTXISD::TexCubeU32FloatLevel:
00224   case NVPTXISD::TexCubeArrayFloatFloat:
00225   case NVPTXISD::TexCubeArrayFloatFloatLevel:
00226   case NVPTXISD::TexCubeArrayS32Float:
00227   case NVPTXISD::TexCubeArrayS32FloatLevel:
00228   case NVPTXISD::TexCubeArrayU32Float:
00229   case NVPTXISD::TexCubeArrayU32FloatLevel:
00230   case NVPTXISD::Tld4R2DFloatFloat:
00231   case NVPTXISD::Tld4G2DFloatFloat:
00232   case NVPTXISD::Tld4B2DFloatFloat:
00233   case NVPTXISD::Tld4A2DFloatFloat:
00234   case NVPTXISD::Tld4R2DS64Float:
00235   case NVPTXISD::Tld4G2DS64Float:
00236   case NVPTXISD::Tld4B2DS64Float:
00237   case NVPTXISD::Tld4A2DS64Float:
00238   case NVPTXISD::Tld4R2DU64Float:
00239   case NVPTXISD::Tld4G2DU64Float:
00240   case NVPTXISD::Tld4B2DU64Float:
00241   case NVPTXISD::Tld4A2DU64Float:
00242   case NVPTXISD::TexUnified1DFloatS32:
00243   case NVPTXISD::TexUnified1DFloatFloat:
00244   case NVPTXISD::TexUnified1DFloatFloatLevel:
00245   case NVPTXISD::TexUnified1DFloatFloatGrad:
00246   case NVPTXISD::TexUnified1DS32S32:
00247   case NVPTXISD::TexUnified1DS32Float:
00248   case NVPTXISD::TexUnified1DS32FloatLevel:
00249   case NVPTXISD::TexUnified1DS32FloatGrad:
00250   case NVPTXISD::TexUnified1DU32S32:
00251   case NVPTXISD::TexUnified1DU32Float:
00252   case NVPTXISD::TexUnified1DU32FloatLevel:
00253   case NVPTXISD::TexUnified1DU32FloatGrad:
00254   case NVPTXISD::TexUnified1DArrayFloatS32:
00255   case NVPTXISD::TexUnified1DArrayFloatFloat:
00256   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
00257   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
00258   case NVPTXISD::TexUnified1DArrayS32S32:
00259   case NVPTXISD::TexUnified1DArrayS32Float:
00260   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
00261   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
00262   case NVPTXISD::TexUnified1DArrayU32S32:
00263   case NVPTXISD::TexUnified1DArrayU32Float:
00264   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
00265   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
00266   case NVPTXISD::TexUnified2DFloatS32:
00267   case NVPTXISD::TexUnified2DFloatFloat:
00268   case NVPTXISD::TexUnified2DFloatFloatLevel:
00269   case NVPTXISD::TexUnified2DFloatFloatGrad:
00270   case NVPTXISD::TexUnified2DS32S32:
00271   case NVPTXISD::TexUnified2DS32Float:
00272   case NVPTXISD::TexUnified2DS32FloatLevel:
00273   case NVPTXISD::TexUnified2DS32FloatGrad:
00274   case NVPTXISD::TexUnified2DU32S32:
00275   case NVPTXISD::TexUnified2DU32Float:
00276   case NVPTXISD::TexUnified2DU32FloatLevel:
00277   case NVPTXISD::TexUnified2DU32FloatGrad:
00278   case NVPTXISD::TexUnified2DArrayFloatS32:
00279   case NVPTXISD::TexUnified2DArrayFloatFloat:
00280   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
00281   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
00282   case NVPTXISD::TexUnified2DArrayS32S32:
00283   case NVPTXISD::TexUnified2DArrayS32Float:
00284   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
00285   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
00286   case NVPTXISD::TexUnified2DArrayU32S32:
00287   case NVPTXISD::TexUnified2DArrayU32Float:
00288   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
00289   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
00290   case NVPTXISD::TexUnified3DFloatS32:
00291   case NVPTXISD::TexUnified3DFloatFloat:
00292   case NVPTXISD::TexUnified3DFloatFloatLevel:
00293   case NVPTXISD::TexUnified3DFloatFloatGrad:
00294   case NVPTXISD::TexUnified3DS32S32:
00295   case NVPTXISD::TexUnified3DS32Float:
00296   case NVPTXISD::TexUnified3DS32FloatLevel:
00297   case NVPTXISD::TexUnified3DS32FloatGrad:
00298   case NVPTXISD::TexUnified3DU32S32:
00299   case NVPTXISD::TexUnified3DU32Float:
00300   case NVPTXISD::TexUnified3DU32FloatLevel:
00301   case NVPTXISD::TexUnified3DU32FloatGrad:
00302   case NVPTXISD::TexUnifiedCubeFloatFloat:
00303   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
00304   case NVPTXISD::TexUnifiedCubeS32Float:
00305   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
00306   case NVPTXISD::TexUnifiedCubeU32Float:
00307   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
00308   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
00309   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
00310   case NVPTXISD::TexUnifiedCubeArrayS32Float:
00311   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
00312   case NVPTXISD::TexUnifiedCubeArrayU32Float:
00313   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
00314   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
00315   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
00316   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
00317   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
00318   case NVPTXISD::Tld4UnifiedR2DS64Float:
00319   case NVPTXISD::Tld4UnifiedG2DS64Float:
00320   case NVPTXISD::Tld4UnifiedB2DS64Float:
00321   case NVPTXISD::Tld4UnifiedA2DS64Float:
00322   case NVPTXISD::Tld4UnifiedR2DU64Float:
00323   case NVPTXISD::Tld4UnifiedG2DU64Float:
00324   case NVPTXISD::Tld4UnifiedB2DU64Float:
00325   case NVPTXISD::Tld4UnifiedA2DU64Float:
00326     ResNode = SelectTextureIntrinsic(N);
00327     break;
00328   case NVPTXISD::Suld1DI8Clamp:
00329   case NVPTXISD::Suld1DI16Clamp:
00330   case NVPTXISD::Suld1DI32Clamp:
00331   case NVPTXISD::Suld1DI64Clamp:
00332   case NVPTXISD::Suld1DV2I8Clamp:
00333   case NVPTXISD::Suld1DV2I16Clamp:
00334   case NVPTXISD::Suld1DV2I32Clamp:
00335   case NVPTXISD::Suld1DV2I64Clamp:
00336   case NVPTXISD::Suld1DV4I8Clamp:
00337   case NVPTXISD::Suld1DV4I16Clamp:
00338   case NVPTXISD::Suld1DV4I32Clamp:
00339   case NVPTXISD::Suld1DArrayI8Clamp:
00340   case NVPTXISD::Suld1DArrayI16Clamp:
00341   case NVPTXISD::Suld1DArrayI32Clamp:
00342   case NVPTXISD::Suld1DArrayI64Clamp:
00343   case NVPTXISD::Suld1DArrayV2I8Clamp:
00344   case NVPTXISD::Suld1DArrayV2I16Clamp:
00345   case NVPTXISD::Suld1DArrayV2I32Clamp:
00346   case NVPTXISD::Suld1DArrayV2I64Clamp:
00347   case NVPTXISD::Suld1DArrayV4I8Clamp:
00348   case NVPTXISD::Suld1DArrayV4I16Clamp:
00349   case NVPTXISD::Suld1DArrayV4I32Clamp:
00350   case NVPTXISD::Suld2DI8Clamp:
00351   case NVPTXISD::Suld2DI16Clamp:
00352   case NVPTXISD::Suld2DI32Clamp:
00353   case NVPTXISD::Suld2DI64Clamp:
00354   case NVPTXISD::Suld2DV2I8Clamp:
00355   case NVPTXISD::Suld2DV2I16Clamp:
00356   case NVPTXISD::Suld2DV2I32Clamp:
00357   case NVPTXISD::Suld2DV2I64Clamp:
00358   case NVPTXISD::Suld2DV4I8Clamp:
00359   case NVPTXISD::Suld2DV4I16Clamp:
00360   case NVPTXISD::Suld2DV4I32Clamp:
00361   case NVPTXISD::Suld2DArrayI8Clamp:
00362   case NVPTXISD::Suld2DArrayI16Clamp:
00363   case NVPTXISD::Suld2DArrayI32Clamp:
00364   case NVPTXISD::Suld2DArrayI64Clamp:
00365   case NVPTXISD::Suld2DArrayV2I8Clamp:
00366   case NVPTXISD::Suld2DArrayV2I16Clamp:
00367   case NVPTXISD::Suld2DArrayV2I32Clamp:
00368   case NVPTXISD::Suld2DArrayV2I64Clamp:
00369   case NVPTXISD::Suld2DArrayV4I8Clamp:
00370   case NVPTXISD::Suld2DArrayV4I16Clamp:
00371   case NVPTXISD::Suld2DArrayV4I32Clamp:
00372   case NVPTXISD::Suld3DI8Clamp:
00373   case NVPTXISD::Suld3DI16Clamp:
00374   case NVPTXISD::Suld3DI32Clamp:
00375   case NVPTXISD::Suld3DI64Clamp:
00376   case NVPTXISD::Suld3DV2I8Clamp:
00377   case NVPTXISD::Suld3DV2I16Clamp:
00378   case NVPTXISD::Suld3DV2I32Clamp:
00379   case NVPTXISD::Suld3DV2I64Clamp:
00380   case NVPTXISD::Suld3DV4I8Clamp:
00381   case NVPTXISD::Suld3DV4I16Clamp:
00382   case NVPTXISD::Suld3DV4I32Clamp:
00383   case NVPTXISD::Suld1DI8Trap:
00384   case NVPTXISD::Suld1DI16Trap:
00385   case NVPTXISD::Suld1DI32Trap:
00386   case NVPTXISD::Suld1DI64Trap:
00387   case NVPTXISD::Suld1DV2I8Trap:
00388   case NVPTXISD::Suld1DV2I16Trap:
00389   case NVPTXISD::Suld1DV2I32Trap:
00390   case NVPTXISD::Suld1DV2I64Trap:
00391   case NVPTXISD::Suld1DV4I8Trap:
00392   case NVPTXISD::Suld1DV4I16Trap:
00393   case NVPTXISD::Suld1DV4I32Trap:
00394   case NVPTXISD::Suld1DArrayI8Trap:
00395   case NVPTXISD::Suld1DArrayI16Trap:
00396   case NVPTXISD::Suld1DArrayI32Trap:
00397   case NVPTXISD::Suld1DArrayI64Trap:
00398   case NVPTXISD::Suld1DArrayV2I8Trap:
00399   case NVPTXISD::Suld1DArrayV2I16Trap:
00400   case NVPTXISD::Suld1DArrayV2I32Trap:
00401   case NVPTXISD::Suld1DArrayV2I64Trap:
00402   case NVPTXISD::Suld1DArrayV4I8Trap:
00403   case NVPTXISD::Suld1DArrayV4I16Trap:
00404   case NVPTXISD::Suld1DArrayV4I32Trap:
00405   case NVPTXISD::Suld2DI8Trap:
00406   case NVPTXISD::Suld2DI16Trap:
00407   case NVPTXISD::Suld2DI32Trap:
00408   case NVPTXISD::Suld2DI64Trap:
00409   case NVPTXISD::Suld2DV2I8Trap:
00410   case NVPTXISD::Suld2DV2I16Trap:
00411   case NVPTXISD::Suld2DV2I32Trap:
00412   case NVPTXISD::Suld2DV2I64Trap:
00413   case NVPTXISD::Suld2DV4I8Trap:
00414   case NVPTXISD::Suld2DV4I16Trap:
00415   case NVPTXISD::Suld2DV4I32Trap:
00416   case NVPTXISD::Suld2DArrayI8Trap:
00417   case NVPTXISD::Suld2DArrayI16Trap:
00418   case NVPTXISD::Suld2DArrayI32Trap:
00419   case NVPTXISD::Suld2DArrayI64Trap:
00420   case NVPTXISD::Suld2DArrayV2I8Trap:
00421   case NVPTXISD::Suld2DArrayV2I16Trap:
00422   case NVPTXISD::Suld2DArrayV2I32Trap:
00423   case NVPTXISD::Suld2DArrayV2I64Trap:
00424   case NVPTXISD::Suld2DArrayV4I8Trap:
00425   case NVPTXISD::Suld2DArrayV4I16Trap:
00426   case NVPTXISD::Suld2DArrayV4I32Trap:
00427   case NVPTXISD::Suld3DI8Trap:
00428   case NVPTXISD::Suld3DI16Trap:
00429   case NVPTXISD::Suld3DI32Trap:
00430   case NVPTXISD::Suld3DI64Trap:
00431   case NVPTXISD::Suld3DV2I8Trap:
00432   case NVPTXISD::Suld3DV2I16Trap:
00433   case NVPTXISD::Suld3DV2I32Trap:
00434   case NVPTXISD::Suld3DV2I64Trap:
00435   case NVPTXISD::Suld3DV4I8Trap:
00436   case NVPTXISD::Suld3DV4I16Trap:
00437   case NVPTXISD::Suld3DV4I32Trap:
00438   case NVPTXISD::Suld1DI8Zero:
00439   case NVPTXISD::Suld1DI16Zero:
00440   case NVPTXISD::Suld1DI32Zero:
00441   case NVPTXISD::Suld1DI64Zero:
00442   case NVPTXISD::Suld1DV2I8Zero:
00443   case NVPTXISD::Suld1DV2I16Zero:
00444   case NVPTXISD::Suld1DV2I32Zero:
00445   case NVPTXISD::Suld1DV2I64Zero:
00446   case NVPTXISD::Suld1DV4I8Zero:
00447   case NVPTXISD::Suld1DV4I16Zero:
00448   case NVPTXISD::Suld1DV4I32Zero:
00449   case NVPTXISD::Suld1DArrayI8Zero:
00450   case NVPTXISD::Suld1DArrayI16Zero:
00451   case NVPTXISD::Suld1DArrayI32Zero:
00452   case NVPTXISD::Suld1DArrayI64Zero:
00453   case NVPTXISD::Suld1DArrayV2I8Zero:
00454   case NVPTXISD::Suld1DArrayV2I16Zero:
00455   case NVPTXISD::Suld1DArrayV2I32Zero:
00456   case NVPTXISD::Suld1DArrayV2I64Zero:
00457   case NVPTXISD::Suld1DArrayV4I8Zero:
00458   case NVPTXISD::Suld1DArrayV4I16Zero:
00459   case NVPTXISD::Suld1DArrayV4I32Zero:
00460   case NVPTXISD::Suld2DI8Zero:
00461   case NVPTXISD::Suld2DI16Zero:
00462   case NVPTXISD::Suld2DI32Zero:
00463   case NVPTXISD::Suld2DI64Zero:
00464   case NVPTXISD::Suld2DV2I8Zero:
00465   case NVPTXISD::Suld2DV2I16Zero:
00466   case NVPTXISD::Suld2DV2I32Zero:
00467   case NVPTXISD::Suld2DV2I64Zero:
00468   case NVPTXISD::Suld2DV4I8Zero:
00469   case NVPTXISD::Suld2DV4I16Zero:
00470   case NVPTXISD::Suld2DV4I32Zero:
00471   case NVPTXISD::Suld2DArrayI8Zero:
00472   case NVPTXISD::Suld2DArrayI16Zero:
00473   case NVPTXISD::Suld2DArrayI32Zero:
00474   case NVPTXISD::Suld2DArrayI64Zero:
00475   case NVPTXISD::Suld2DArrayV2I8Zero:
00476   case NVPTXISD::Suld2DArrayV2I16Zero:
00477   case NVPTXISD::Suld2DArrayV2I32Zero:
00478   case NVPTXISD::Suld2DArrayV2I64Zero:
00479   case NVPTXISD::Suld2DArrayV4I8Zero:
00480   case NVPTXISD::Suld2DArrayV4I16Zero:
00481   case NVPTXISD::Suld2DArrayV4I32Zero:
00482   case NVPTXISD::Suld3DI8Zero:
00483   case NVPTXISD::Suld3DI16Zero:
00484   case NVPTXISD::Suld3DI32Zero:
00485   case NVPTXISD::Suld3DI64Zero:
00486   case NVPTXISD::Suld3DV2I8Zero:
00487   case NVPTXISD::Suld3DV2I16Zero:
00488   case NVPTXISD::Suld3DV2I32Zero:
00489   case NVPTXISD::Suld3DV2I64Zero:
00490   case NVPTXISD::Suld3DV4I8Zero:
00491   case NVPTXISD::Suld3DV4I16Zero:
00492   case NVPTXISD::Suld3DV4I32Zero:
00493     ResNode = SelectSurfaceIntrinsic(N);
00494     break;
00495   case ISD::AND:
00496   case ISD::SRA:
00497   case ISD::SRL:
00498     // Try to select BFE
00499     ResNode = SelectBFE(N);
00500     break;
00501   case ISD::ADDRSPACECAST:
00502     ResNode = SelectAddrSpaceCast(N);
00503     break;
00504   default:
00505     break;
00506   }
00507   if (ResNode)
00508     return ResNode;
00509   return SelectCode(N);
00510 }
00511 
00512 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
00513   unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
00514   switch (IID) {
00515   default:
00516     return NULL;
00517   case Intrinsic::nvvm_ldg_global_f:
00518   case Intrinsic::nvvm_ldg_global_i:
00519   case Intrinsic::nvvm_ldg_global_p:
00520   case Intrinsic::nvvm_ldu_global_f:
00521   case Intrinsic::nvvm_ldu_global_i:
00522   case Intrinsic::nvvm_ldu_global_p:
00523     return SelectLDGLDU(N);
00524   }
00525 }
00526 
00527 static unsigned int getCodeAddrSpace(MemSDNode *N) {
00528   const Value *Src = N->getMemOperand()->getValue();
00529 
00530   if (!Src)
00531     return NVPTX::PTXLdStInstCode::GENERIC;
00532 
00533   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
00534     switch (PT->getAddressSpace()) {
00535     case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
00536     case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
00537     case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
00538     case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
00539     case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
00540     case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
00541     default: break;
00542     }
00543   }
00544   return NVPTX::PTXLdStInstCode::GENERIC;
00545 }
00546 
00547 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
00548   unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
00549   switch (IID) {
00550   default:
00551     return nullptr;
00552   case Intrinsic::nvvm_texsurf_handle_internal:
00553     return SelectTexSurfHandle(N);
00554   }
00555 }
00556 
00557 SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
00558   // Op 0 is the intrinsic ID
00559   SDValue Wrapper = N->getOperand(1);
00560   SDValue GlobalVal = Wrapper.getOperand(0);
00561   return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
00562                                 GlobalVal);
00563 }
00564 
00565 SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
00566   SDValue Src = N->getOperand(0);
00567   AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
00568   unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
00569   unsigned DstAddrSpace = CastN->getDestAddressSpace();
00570 
00571   assert(SrcAddrSpace != DstAddrSpace &&
00572          "addrspacecast must be between different address spaces");
00573 
00574   if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
00575     // Specific to generic
00576     unsigned Opc;
00577     switch (SrcAddrSpace) {
00578     default: report_fatal_error("Bad address space in addrspacecast");
00579     case ADDRESS_SPACE_GLOBAL:
00580       Opc = TM.is64Bit() ? NVPTX::cvta_global_yes_64 : NVPTX::cvta_global_yes;
00581       break;
00582     case ADDRESS_SPACE_SHARED:
00583       Opc = TM.is64Bit() ? NVPTX::cvta_shared_yes_64 : NVPTX::cvta_shared_yes;
00584       break;
00585     case ADDRESS_SPACE_CONST:
00586       Opc = TM.is64Bit() ? NVPTX::cvta_const_yes_64 : NVPTX::cvta_const_yes;
00587       break;
00588     case ADDRESS_SPACE_LOCAL:
00589       Opc = TM.is64Bit() ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes;
00590       break;
00591     }
00592     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00593   } else {
00594     // Generic to specific
00595     if (SrcAddrSpace != 0)
00596       report_fatal_error("Cannot cast between two non-generic address spaces");
00597     unsigned Opc;
00598     switch (DstAddrSpace) {
00599     default: report_fatal_error("Bad address space in addrspacecast");
00600     case ADDRESS_SPACE_GLOBAL:
00601       Opc = TM.is64Bit() ? NVPTX::cvta_to_global_yes_64
00602                          : NVPTX::cvta_to_global_yes;
00603       break;
00604     case ADDRESS_SPACE_SHARED:
00605       Opc = TM.is64Bit() ? NVPTX::cvta_to_shared_yes_64
00606                          : NVPTX::cvta_to_shared_yes;
00607       break;
00608     case ADDRESS_SPACE_CONST:
00609       Opc =
00610           TM.is64Bit() ? NVPTX::cvta_to_const_yes_64 : NVPTX::cvta_to_const_yes;
00611       break;
00612     case ADDRESS_SPACE_LOCAL:
00613       Opc =
00614           TM.is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes;
00615       break;
00616     }
00617     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00618   }
00619 }
00620 
00621 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
00622   SDLoc dl(N);
00623   LoadSDNode *LD = cast<LoadSDNode>(N);
00624   EVT LoadedVT = LD->getMemoryVT();
00625   SDNode *NVPTXLD = nullptr;
00626 
00627   // do not support pre/post inc/dec
00628   if (LD->isIndexed())
00629     return nullptr;
00630 
00631   if (!LoadedVT.isSimple())
00632     return nullptr;
00633 
00634   // Address Space Setting
00635   unsigned int codeAddrSpace = getCodeAddrSpace(LD);
00636 
00637   // Volatile Setting
00638   // - .volatile is only availalble for .global and .shared
00639   bool isVolatile = LD->isVolatile();
00640   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00641       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00642       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00643     isVolatile = false;
00644 
00645   // Vector Setting
00646   MVT SimpleVT = LoadedVT.getSimpleVT();
00647   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
00648   if (SimpleVT.isVector()) {
00649     unsigned num = SimpleVT.getVectorNumElements();
00650     if (num == 2)
00651       vecType = NVPTX::PTXLdStInstCode::V2;
00652     else if (num == 4)
00653       vecType = NVPTX::PTXLdStInstCode::V4;
00654     else
00655       return nullptr;
00656   }
00657 
00658   // Type Setting: fromType + fromTypeWidth
00659   //
00660   // Sign   : ISD::SEXTLOAD
00661   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00662   //          type is integer
00663   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00664   MVT ScalarVT = SimpleVT.getScalarType();
00665   // Read at least 8 bits (predicates are stored as 8-bit values)
00666   unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00667   unsigned int fromType;
00668   if ((LD->getExtensionType() == ISD::SEXTLOAD))
00669     fromType = NVPTX::PTXLdStInstCode::Signed;
00670   else if (ScalarVT.isFloatingPoint())
00671     fromType = NVPTX::PTXLdStInstCode::Float;
00672   else
00673     fromType = NVPTX::PTXLdStInstCode::Unsigned;
00674 
00675   // Create the machine instruction DAG
00676   SDValue Chain = N->getOperand(0);
00677   SDValue N1 = N->getOperand(1);
00678   SDValue Addr;
00679   SDValue Offset, Base;
00680   unsigned Opcode;
00681   MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
00682 
00683   if (SelectDirectAddr(N1, Addr)) {
00684     switch (TargetVT) {
00685     case MVT::i8:
00686       Opcode = NVPTX::LD_i8_avar;
00687       break;
00688     case MVT::i16:
00689       Opcode = NVPTX::LD_i16_avar;
00690       break;
00691     case MVT::i32:
00692       Opcode = NVPTX::LD_i32_avar;
00693       break;
00694     case MVT::i64:
00695       Opcode = NVPTX::LD_i64_avar;
00696       break;
00697     case MVT::f32:
00698       Opcode = NVPTX::LD_f32_avar;
00699       break;
00700     case MVT::f64:
00701       Opcode = NVPTX::LD_f64_avar;
00702       break;
00703     default:
00704       return nullptr;
00705     }
00706     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00707                       getI32Imm(vecType), getI32Imm(fromType),
00708                       getI32Imm(fromTypeWidth), Addr, Chain };
00709     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00710   } else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
00711                           : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
00712     switch (TargetVT) {
00713     case MVT::i8:
00714       Opcode = NVPTX::LD_i8_asi;
00715       break;
00716     case MVT::i16:
00717       Opcode = NVPTX::LD_i16_asi;
00718       break;
00719     case MVT::i32:
00720       Opcode = NVPTX::LD_i32_asi;
00721       break;
00722     case MVT::i64:
00723       Opcode = NVPTX::LD_i64_asi;
00724       break;
00725     case MVT::f32:
00726       Opcode = NVPTX::LD_f32_asi;
00727       break;
00728     case MVT::f64:
00729       Opcode = NVPTX::LD_f64_asi;
00730       break;
00731     default:
00732       return nullptr;
00733     }
00734     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00735                       getI32Imm(vecType), getI32Imm(fromType),
00736                       getI32Imm(fromTypeWidth), Base, Offset, Chain };
00737     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00738   } else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
00739                           : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
00740     if (TM.is64Bit()) {
00741       switch (TargetVT) {
00742       case MVT::i8:
00743         Opcode = NVPTX::LD_i8_ari_64;
00744         break;
00745       case MVT::i16:
00746         Opcode = NVPTX::LD_i16_ari_64;
00747         break;
00748       case MVT::i32:
00749         Opcode = NVPTX::LD_i32_ari_64;
00750         break;
00751       case MVT::i64:
00752         Opcode = NVPTX::LD_i64_ari_64;
00753         break;
00754       case MVT::f32:
00755         Opcode = NVPTX::LD_f32_ari_64;
00756         break;
00757       case MVT::f64:
00758         Opcode = NVPTX::LD_f64_ari_64;
00759         break;
00760       default:
00761         return nullptr;
00762       }
00763     } else {
00764       switch (TargetVT) {
00765       case MVT::i8:
00766         Opcode = NVPTX::LD_i8_ari;
00767         break;
00768       case MVT::i16:
00769         Opcode = NVPTX::LD_i16_ari;
00770         break;
00771       case MVT::i32:
00772         Opcode = NVPTX::LD_i32_ari;
00773         break;
00774       case MVT::i64:
00775         Opcode = NVPTX::LD_i64_ari;
00776         break;
00777       case MVT::f32:
00778         Opcode = NVPTX::LD_f32_ari;
00779         break;
00780       case MVT::f64:
00781         Opcode = NVPTX::LD_f64_ari;
00782         break;
00783       default:
00784         return nullptr;
00785       }
00786     }
00787     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00788                       getI32Imm(vecType), getI32Imm(fromType),
00789                       getI32Imm(fromTypeWidth), Base, Offset, Chain };
00790     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00791   } else {
00792     if (TM.is64Bit()) {
00793       switch (TargetVT) {
00794       case MVT::i8:
00795         Opcode = NVPTX::LD_i8_areg_64;
00796         break;
00797       case MVT::i16:
00798         Opcode = NVPTX::LD_i16_areg_64;
00799         break;
00800       case MVT::i32:
00801         Opcode = NVPTX::LD_i32_areg_64;
00802         break;
00803       case MVT::i64:
00804         Opcode = NVPTX::LD_i64_areg_64;
00805         break;
00806       case MVT::f32:
00807         Opcode = NVPTX::LD_f32_areg_64;
00808         break;
00809       case MVT::f64:
00810         Opcode = NVPTX::LD_f64_areg_64;
00811         break;
00812       default:
00813         return nullptr;
00814       }
00815     } else {
00816       switch (TargetVT) {
00817       case MVT::i8:
00818         Opcode = NVPTX::LD_i8_areg;
00819         break;
00820       case MVT::i16:
00821         Opcode = NVPTX::LD_i16_areg;
00822         break;
00823       case MVT::i32:
00824         Opcode = NVPTX::LD_i32_areg;
00825         break;
00826       case MVT::i64:
00827         Opcode = NVPTX::LD_i64_areg;
00828         break;
00829       case MVT::f32:
00830         Opcode = NVPTX::LD_f32_areg;
00831         break;
00832       case MVT::f64:
00833         Opcode = NVPTX::LD_f64_areg;
00834         break;
00835       default:
00836         return nullptr;
00837       }
00838     }
00839     SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
00840                       getI32Imm(vecType), getI32Imm(fromType),
00841                       getI32Imm(fromTypeWidth), N1, Chain };
00842     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00843   }
00844 
00845   if (NVPTXLD) {
00846     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
00847     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
00848     cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
00849   }
00850 
00851   return NVPTXLD;
00852 }
00853 
00854 SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
00855 
00856   SDValue Chain = N->getOperand(0);
00857   SDValue Op1 = N->getOperand(1);
00858   SDValue Addr, Offset, Base;
00859   unsigned Opcode;
00860   SDLoc DL(N);
00861   SDNode *LD;
00862   MemSDNode *MemSD = cast<MemSDNode>(N);
00863   EVT LoadedVT = MemSD->getMemoryVT();
00864 
00865   if (!LoadedVT.isSimple())
00866     return nullptr;
00867 
00868   // Address Space Setting
00869   unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
00870 
00871   // Volatile Setting
00872   // - .volatile is only availalble for .global and .shared
00873   bool IsVolatile = MemSD->isVolatile();
00874   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00875       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00876       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00877     IsVolatile = false;
00878 
00879   // Vector Setting
00880   MVT SimpleVT = LoadedVT.getSimpleVT();
00881 
00882   // Type Setting: fromType + fromTypeWidth
00883   //
00884   // Sign   : ISD::SEXTLOAD
00885   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00886   //          type is integer
00887   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00888   MVT ScalarVT = SimpleVT.getScalarType();
00889   // Read at least 8 bits (predicates are stored as 8-bit values)
00890   unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00891   unsigned int FromType;
00892   // The last operand holds the original LoadSDNode::getExtensionType() value
00893   unsigned ExtensionType = cast<ConstantSDNode>(
00894       N->getOperand(N->getNumOperands() - 1))->getZExtValue();
00895   if (ExtensionType == ISD::SEXTLOAD)
00896     FromType = NVPTX::PTXLdStInstCode::Signed;
00897   else if (ScalarVT.isFloatingPoint())
00898     FromType = NVPTX::PTXLdStInstCode::Float;
00899   else
00900     FromType = NVPTX::PTXLdStInstCode::Unsigned;
00901 
00902   unsigned VecType;
00903 
00904   switch (N->getOpcode()) {
00905   case NVPTXISD::LoadV2:
00906     VecType = NVPTX::PTXLdStInstCode::V2;
00907     break;
00908   case NVPTXISD::LoadV4:
00909     VecType = NVPTX::PTXLdStInstCode::V4;
00910     break;
00911   default:
00912     return nullptr;
00913   }
00914 
00915   EVT EltVT = N->getValueType(0);
00916 
00917   if (SelectDirectAddr(Op1, Addr)) {
00918     switch (N->getOpcode()) {
00919     default:
00920       return nullptr;
00921     case NVPTXISD::LoadV2:
00922       switch (EltVT.getSimpleVT().SimpleTy) {
00923       default:
00924         return nullptr;
00925       case MVT::i8:
00926         Opcode = NVPTX::LDV_i8_v2_avar;
00927         break;
00928       case MVT::i16:
00929         Opcode = NVPTX::LDV_i16_v2_avar;
00930         break;
00931       case MVT::i32:
00932         Opcode = NVPTX::LDV_i32_v2_avar;
00933         break;
00934       case MVT::i64:
00935         Opcode = NVPTX::LDV_i64_v2_avar;
00936         break;
00937       case MVT::f32:
00938         Opcode = NVPTX::LDV_f32_v2_avar;
00939         break;
00940       case MVT::f64:
00941         Opcode = NVPTX::LDV_f64_v2_avar;
00942         break;
00943       }
00944       break;
00945     case NVPTXISD::LoadV4:
00946       switch (EltVT.getSimpleVT().SimpleTy) {
00947       default:
00948         return nullptr;
00949       case MVT::i8:
00950         Opcode = NVPTX::LDV_i8_v4_avar;
00951         break;
00952       case MVT::i16:
00953         Opcode = NVPTX::LDV_i16_v4_avar;
00954         break;
00955       case MVT::i32:
00956         Opcode = NVPTX::LDV_i32_v4_avar;
00957         break;
00958       case MVT::f32:
00959         Opcode = NVPTX::LDV_f32_v4_avar;
00960         break;
00961       }
00962       break;
00963     }
00964 
00965     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
00966                       getI32Imm(VecType), getI32Imm(FromType),
00967                       getI32Imm(FromTypeWidth), Addr, Chain };
00968     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
00969   } else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
00970                           : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
00971     switch (N->getOpcode()) {
00972     default:
00973       return nullptr;
00974     case NVPTXISD::LoadV2:
00975       switch (EltVT.getSimpleVT().SimpleTy) {
00976       default:
00977         return nullptr;
00978       case MVT::i8:
00979         Opcode = NVPTX::LDV_i8_v2_asi;
00980         break;
00981       case MVT::i16:
00982         Opcode = NVPTX::LDV_i16_v2_asi;
00983         break;
00984       case MVT::i32:
00985         Opcode = NVPTX::LDV_i32_v2_asi;
00986         break;
00987       case MVT::i64:
00988         Opcode = NVPTX::LDV_i64_v2_asi;
00989         break;
00990       case MVT::f32:
00991         Opcode = NVPTX::LDV_f32_v2_asi;
00992         break;
00993       case MVT::f64:
00994         Opcode = NVPTX::LDV_f64_v2_asi;
00995         break;
00996       }
00997       break;
00998     case NVPTXISD::LoadV4:
00999       switch (EltVT.getSimpleVT().SimpleTy) {
01000       default:
01001         return nullptr;
01002       case MVT::i8:
01003         Opcode = NVPTX::LDV_i8_v4_asi;
01004         break;
01005       case MVT::i16:
01006         Opcode = NVPTX::LDV_i16_v4_asi;
01007         break;
01008       case MVT::i32:
01009         Opcode = NVPTX::LDV_i32_v4_asi;
01010         break;
01011       case MVT::f32:
01012         Opcode = NVPTX::LDV_f32_v4_asi;
01013         break;
01014       }
01015       break;
01016     }
01017 
01018     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
01019                       getI32Imm(VecType), getI32Imm(FromType),
01020                       getI32Imm(FromTypeWidth), Base, Offset, Chain };
01021     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01022   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
01023                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
01024     if (TM.is64Bit()) {
01025       switch (N->getOpcode()) {
01026       default:
01027         return nullptr;
01028       case NVPTXISD::LoadV2:
01029         switch (EltVT.getSimpleVT().SimpleTy) {
01030         default:
01031           return nullptr;
01032         case MVT::i8:
01033           Opcode = NVPTX::LDV_i8_v2_ari_64;
01034           break;
01035         case MVT::i16:
01036           Opcode = NVPTX::LDV_i16_v2_ari_64;
01037           break;
01038         case MVT::i32:
01039           Opcode = NVPTX::LDV_i32_v2_ari_64;
01040           break;
01041         case MVT::i64:
01042           Opcode = NVPTX::LDV_i64_v2_ari_64;
01043           break;
01044         case MVT::f32:
01045           Opcode = NVPTX::LDV_f32_v2_ari_64;
01046           break;
01047         case MVT::f64:
01048           Opcode = NVPTX::LDV_f64_v2_ari_64;
01049           break;
01050         }
01051         break;
01052       case NVPTXISD::LoadV4:
01053         switch (EltVT.getSimpleVT().SimpleTy) {
01054         default:
01055           return nullptr;
01056         case MVT::i8:
01057           Opcode = NVPTX::LDV_i8_v4_ari_64;
01058           break;
01059         case MVT::i16:
01060           Opcode = NVPTX::LDV_i16_v4_ari_64;
01061           break;
01062         case MVT::i32:
01063           Opcode = NVPTX::LDV_i32_v4_ari_64;
01064           break;
01065         case MVT::f32:
01066           Opcode = NVPTX::LDV_f32_v4_ari_64;
01067           break;
01068         }
01069         break;
01070       }
01071     } else {
01072       switch (N->getOpcode()) {
01073       default:
01074         return nullptr;
01075       case NVPTXISD::LoadV2:
01076         switch (EltVT.getSimpleVT().SimpleTy) {
01077         default:
01078           return nullptr;
01079         case MVT::i8:
01080           Opcode = NVPTX::LDV_i8_v2_ari;
01081           break;
01082         case MVT::i16:
01083           Opcode = NVPTX::LDV_i16_v2_ari;
01084           break;
01085         case MVT::i32:
01086           Opcode = NVPTX::LDV_i32_v2_ari;
01087           break;
01088         case MVT::i64:
01089           Opcode = NVPTX::LDV_i64_v2_ari;
01090           break;
01091         case MVT::f32:
01092           Opcode = NVPTX::LDV_f32_v2_ari;
01093           break;
01094         case MVT::f64:
01095           Opcode = NVPTX::LDV_f64_v2_ari;
01096           break;
01097         }
01098         break;
01099       case NVPTXISD::LoadV4:
01100         switch (EltVT.getSimpleVT().SimpleTy) {
01101         default:
01102           return nullptr;
01103         case MVT::i8:
01104           Opcode = NVPTX::LDV_i8_v4_ari;
01105           break;
01106         case MVT::i16:
01107           Opcode = NVPTX::LDV_i16_v4_ari;
01108           break;
01109         case MVT::i32:
01110           Opcode = NVPTX::LDV_i32_v4_ari;
01111           break;
01112         case MVT::f32:
01113           Opcode = NVPTX::LDV_f32_v4_ari;
01114           break;
01115         }
01116         break;
01117       }
01118     }
01119 
01120     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
01121                       getI32Imm(VecType), getI32Imm(FromType),
01122                       getI32Imm(FromTypeWidth), Base, Offset, Chain };
01123 
01124     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01125   } else {
01126     if (TM.is64Bit()) {
01127       switch (N->getOpcode()) {
01128       default:
01129         return nullptr;
01130       case NVPTXISD::LoadV2:
01131         switch (EltVT.getSimpleVT().SimpleTy) {
01132         default:
01133           return nullptr;
01134         case MVT::i8:
01135           Opcode = NVPTX::LDV_i8_v2_areg_64;
01136           break;
01137         case MVT::i16:
01138           Opcode = NVPTX::LDV_i16_v2_areg_64;
01139           break;
01140         case MVT::i32:
01141           Opcode = NVPTX::LDV_i32_v2_areg_64;
01142           break;
01143         case MVT::i64:
01144           Opcode = NVPTX::LDV_i64_v2_areg_64;
01145           break;
01146         case MVT::f32:
01147           Opcode = NVPTX::LDV_f32_v2_areg_64;
01148           break;
01149         case MVT::f64:
01150           Opcode = NVPTX::LDV_f64_v2_areg_64;
01151           break;
01152         }
01153         break;
01154       case NVPTXISD::LoadV4:
01155         switch (EltVT.getSimpleVT().SimpleTy) {
01156         default:
01157           return nullptr;
01158         case MVT::i8:
01159           Opcode = NVPTX::LDV_i8_v4_areg_64;
01160           break;
01161         case MVT::i16:
01162           Opcode = NVPTX::LDV_i16_v4_areg_64;
01163           break;
01164         case MVT::i32:
01165           Opcode = NVPTX::LDV_i32_v4_areg_64;
01166           break;
01167         case MVT::f32:
01168           Opcode = NVPTX::LDV_f32_v4_areg_64;
01169           break;
01170         }
01171         break;
01172       }
01173     } else {
01174       switch (N->getOpcode()) {
01175       default:
01176         return nullptr;
01177       case NVPTXISD::LoadV2:
01178         switch (EltVT.getSimpleVT().SimpleTy) {
01179         default:
01180           return nullptr;
01181         case MVT::i8:
01182           Opcode = NVPTX::LDV_i8_v2_areg;
01183           break;
01184         case MVT::i16:
01185           Opcode = NVPTX::LDV_i16_v2_areg;
01186           break;
01187         case MVT::i32:
01188           Opcode = NVPTX::LDV_i32_v2_areg;
01189           break;
01190         case MVT::i64:
01191           Opcode = NVPTX::LDV_i64_v2_areg;
01192           break;
01193         case MVT::f32:
01194           Opcode = NVPTX::LDV_f32_v2_areg;
01195           break;
01196         case MVT::f64:
01197           Opcode = NVPTX::LDV_f64_v2_areg;
01198           break;
01199         }
01200         break;
01201       case NVPTXISD::LoadV4:
01202         switch (EltVT.getSimpleVT().SimpleTy) {
01203         default:
01204           return nullptr;
01205         case MVT::i8:
01206           Opcode = NVPTX::LDV_i8_v4_areg;
01207           break;
01208         case MVT::i16:
01209           Opcode = NVPTX::LDV_i16_v4_areg;
01210           break;
01211         case MVT::i32:
01212           Opcode = NVPTX::LDV_i32_v4_areg;
01213           break;
01214         case MVT::f32:
01215           Opcode = NVPTX::LDV_f32_v4_areg;
01216           break;
01217         }
01218         break;
01219       }
01220     }
01221 
01222     SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
01223                       getI32Imm(VecType), getI32Imm(FromType),
01224                       getI32Imm(FromTypeWidth), Op1, Chain };
01225     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01226   }
01227 
01228   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01229   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
01230   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
01231 
01232   return LD;
01233 }
01234 
01235 SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
01236 
01237   SDValue Chain = N->getOperand(0);
01238   SDValue Op1;
01239   MemSDNode *Mem;
01240   bool IsLDG = true;
01241 
01242   // If this is an LDG intrinsic, the address is the third operand. Its its an
01243   // LDG/LDU SD node (from custom vector handling), then its the second operand
01244   if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
01245     Op1 = N->getOperand(2);
01246     Mem = cast<MemIntrinsicSDNode>(N);
01247     unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
01248     switch (IID) {
01249     default:
01250       return NULL;
01251     case Intrinsic::nvvm_ldg_global_f:
01252     case Intrinsic::nvvm_ldg_global_i:
01253     case Intrinsic::nvvm_ldg_global_p:
01254       IsLDG = true;
01255       break;
01256     case Intrinsic::nvvm_ldu_global_f:
01257     case Intrinsic::nvvm_ldu_global_i:
01258     case Intrinsic::nvvm_ldu_global_p:
01259       IsLDG = false;
01260       break;
01261     }
01262   } else {
01263     Op1 = N->getOperand(1);
01264     Mem = cast<MemSDNode>(N);
01265   }
01266 
01267   unsigned Opcode;
01268   SDLoc DL(N);
01269   SDNode *LD;
01270   SDValue Base, Offset, Addr;
01271 
01272   EVT EltVT = Mem->getMemoryVT();
01273   if (EltVT.isVector()) {
01274     EltVT = EltVT.getVectorElementType();
01275   }
01276 
01277   if (SelectDirectAddr(Op1, Addr)) {
01278     switch (N->getOpcode()) {
01279     default:
01280       return nullptr;
01281     case ISD::INTRINSIC_W_CHAIN:
01282       if (IsLDG) {
01283         switch (EltVT.getSimpleVT().SimpleTy) {
01284         default:
01285           return nullptr;
01286         case MVT::i8:
01287           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
01288           break;
01289         case MVT::i16:
01290           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
01291           break;
01292         case MVT::i32:
01293           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
01294           break;
01295         case MVT::i64:
01296           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
01297           break;
01298         case MVT::f32:
01299           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
01300           break;
01301         case MVT::f64:
01302           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
01303           break;
01304         }
01305       } else {
01306         switch (EltVT.getSimpleVT().SimpleTy) {
01307         default:
01308           return nullptr;
01309         case MVT::i8:
01310           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
01311           break;
01312         case MVT::i16:
01313           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
01314           break;
01315         case MVT::i32:
01316           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
01317           break;
01318         case MVT::i64:
01319           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
01320           break;
01321         case MVT::f32:
01322           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
01323           break;
01324         case MVT::f64:
01325           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
01326           break;
01327         }
01328       }
01329       break;
01330     case NVPTXISD::LDGV2:
01331       switch (EltVT.getSimpleVT().SimpleTy) {
01332       default:
01333         return nullptr;
01334       case MVT::i8:
01335         Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
01336         break;
01337       case MVT::i16:
01338         Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
01339         break;
01340       case MVT::i32:
01341         Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
01342         break;
01343       case MVT::i64:
01344         Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
01345         break;
01346       case MVT::f32:
01347         Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
01348         break;
01349       case MVT::f64:
01350         Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
01351         break;
01352       }
01353       break;
01354     case NVPTXISD::LDUV2:
01355       switch (EltVT.getSimpleVT().SimpleTy) {
01356       default:
01357         return nullptr;
01358       case MVT::i8:
01359         Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
01360         break;
01361       case MVT::i16:
01362         Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
01363         break;
01364       case MVT::i32:
01365         Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
01366         break;
01367       case MVT::i64:
01368         Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
01369         break;
01370       case MVT::f32:
01371         Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
01372         break;
01373       case MVT::f64:
01374         Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
01375         break;
01376       }
01377       break;
01378     case NVPTXISD::LDGV4:
01379       switch (EltVT.getSimpleVT().SimpleTy) {
01380       default:
01381         return nullptr;
01382       case MVT::i8:
01383         Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
01384         break;
01385       case MVT::i16:
01386         Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
01387         break;
01388       case MVT::i32:
01389         Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
01390         break;
01391       case MVT::f32:
01392         Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
01393         break;
01394       }
01395       break;
01396     case NVPTXISD::LDUV4:
01397       switch (EltVT.getSimpleVT().SimpleTy) {
01398       default:
01399         return nullptr;
01400       case MVT::i8:
01401         Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
01402         break;
01403       case MVT::i16:
01404         Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
01405         break;
01406       case MVT::i32:
01407         Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
01408         break;
01409       case MVT::f32:
01410         Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
01411         break;
01412       }
01413       break;
01414     }
01415 
01416     SDValue Ops[] = { Addr, Chain };
01417     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01418   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
01419                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
01420     if (TM.is64Bit()) {
01421       switch (N->getOpcode()) {
01422       default:
01423         return nullptr;
01424       case ISD::INTRINSIC_W_CHAIN:
01425         if (IsLDG) {
01426           switch (EltVT.getSimpleVT().SimpleTy) {
01427           default:
01428             return nullptr;
01429           case MVT::i8:
01430             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
01431             break;
01432           case MVT::i16:
01433             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
01434             break;
01435           case MVT::i32:
01436             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
01437             break;
01438           case MVT::i64:
01439             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
01440             break;
01441           case MVT::f32:
01442             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
01443             break;
01444           case MVT::f64:
01445             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
01446             break;
01447           }
01448         } else {
01449           switch (EltVT.getSimpleVT().SimpleTy) {
01450           default:
01451             return nullptr;
01452           case MVT::i8:
01453             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
01454             break;
01455           case MVT::i16:
01456             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
01457             break;
01458           case MVT::i32:
01459             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
01460             break;
01461           case MVT::i64:
01462             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
01463             break;
01464           case MVT::f32:
01465             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
01466             break;
01467           case MVT::f64:
01468             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
01469             break;
01470           }
01471         }
01472         break;
01473       case NVPTXISD::LDGV2:
01474         switch (EltVT.getSimpleVT().SimpleTy) {
01475         default:
01476           return nullptr;
01477         case MVT::i8:
01478           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
01479           break;
01480         case MVT::i16:
01481           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
01482           break;
01483         case MVT::i32:
01484           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
01485           break;
01486         case MVT::i64:
01487           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
01488           break;
01489         case MVT::f32:
01490           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
01491           break;
01492         case MVT::f64:
01493           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
01494           break;
01495         }
01496         break;
01497       case NVPTXISD::LDUV2:
01498         switch (EltVT.getSimpleVT().SimpleTy) {
01499         default:
01500           return nullptr;
01501         case MVT::i8:
01502           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
01503           break;
01504         case MVT::i16:
01505           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
01506           break;
01507         case MVT::i32:
01508           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
01509           break;
01510         case MVT::i64:
01511           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
01512           break;
01513         case MVT::f32:
01514           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
01515           break;
01516         case MVT::f64:
01517           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
01518           break;
01519         }
01520         break;
01521       case NVPTXISD::LDGV4:
01522         switch (EltVT.getSimpleVT().SimpleTy) {
01523         default:
01524           return nullptr;
01525         case MVT::i8:
01526           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
01527           break;
01528         case MVT::i16:
01529           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
01530           break;
01531         case MVT::i32:
01532           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
01533           break;
01534         case MVT::f32:
01535           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
01536           break;
01537         }
01538         break;
01539       case NVPTXISD::LDUV4:
01540         switch (EltVT.getSimpleVT().SimpleTy) {
01541         default:
01542           return nullptr;
01543         case MVT::i8:
01544           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
01545           break;
01546         case MVT::i16:
01547           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
01548           break;
01549         case MVT::i32:
01550           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
01551           break;
01552         case MVT::f32:
01553           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
01554           break;
01555         }
01556         break;
01557       }
01558     } else {
01559       switch (N->getOpcode()) {
01560       default:
01561         return nullptr;
01562       case ISD::INTRINSIC_W_CHAIN:
01563         if (IsLDG) {
01564           switch (EltVT.getSimpleVT().SimpleTy) {
01565           default:
01566             return nullptr;
01567           case MVT::i8:
01568             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
01569             break;
01570           case MVT::i16:
01571             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
01572             break;
01573           case MVT::i32:
01574             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
01575             break;
01576           case MVT::i64:
01577             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
01578             break;
01579           case MVT::f32:
01580             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
01581             break;
01582           case MVT::f64:
01583             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
01584             break;
01585           }
01586         } else {
01587           switch (EltVT.getSimpleVT().SimpleTy) {
01588           default:
01589             return nullptr;
01590           case MVT::i8:
01591             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
01592             break;
01593           case MVT::i16:
01594             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
01595             break;
01596           case MVT::i32:
01597             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
01598             break;
01599           case MVT::i64:
01600             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
01601             break;
01602           case MVT::f32:
01603             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
01604             break;
01605           case MVT::f64:
01606             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
01607             break;
01608           }
01609         }
01610         break;
01611       case NVPTXISD::LDGV2:
01612         switch (EltVT.getSimpleVT().SimpleTy) {
01613         default:
01614           return nullptr;
01615         case MVT::i8:
01616           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
01617           break;
01618         case MVT::i16:
01619           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
01620           break;
01621         case MVT::i32:
01622           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
01623           break;
01624         case MVT::i64:
01625           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
01626           break;
01627         case MVT::f32:
01628           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
01629           break;
01630         case MVT::f64:
01631           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
01632           break;
01633         }
01634         break;
01635       case NVPTXISD::LDUV2:
01636         switch (EltVT.getSimpleVT().SimpleTy) {
01637         default:
01638           return nullptr;
01639         case MVT::i8:
01640           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
01641           break;
01642         case MVT::i16:
01643           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
01644           break;
01645         case MVT::i32:
01646           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
01647           break;
01648         case MVT::i64:
01649           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
01650           break;
01651         case MVT::f32:
01652           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
01653           break;
01654         case MVT::f64:
01655           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
01656           break;
01657         }
01658         break;
01659       case NVPTXISD::LDGV4:
01660         switch (EltVT.getSimpleVT().SimpleTy) {
01661         default:
01662           return nullptr;
01663         case MVT::i8:
01664           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
01665           break;
01666         case MVT::i16:
01667           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
01668           break;
01669         case MVT::i32:
01670           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
01671           break;
01672         case MVT::f32:
01673           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
01674           break;
01675         }
01676         break;
01677       case NVPTXISD::LDUV4:
01678         switch (EltVT.getSimpleVT().SimpleTy) {
01679         default:
01680           return nullptr;
01681         case MVT::i8:
01682           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
01683           break;
01684         case MVT::i16:
01685           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
01686           break;
01687         case MVT::i32:
01688           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
01689           break;
01690         case MVT::f32:
01691           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
01692           break;
01693         }
01694         break;
01695       }
01696     }
01697 
01698     SDValue Ops[] = { Base, Offset, Chain };
01699 
01700     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01701   } else {
01702     if (TM.is64Bit()) {
01703       switch (N->getOpcode()) {
01704       default:
01705         return nullptr;
01706       case ISD::INTRINSIC_W_CHAIN:
01707         if (IsLDG) {
01708           switch (EltVT.getSimpleVT().SimpleTy) {
01709           default:
01710             return nullptr;
01711           case MVT::i8:
01712             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
01713             break;
01714           case MVT::i16:
01715             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
01716             break;
01717           case MVT::i32:
01718             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
01719             break;
01720           case MVT::i64:
01721             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
01722             break;
01723           case MVT::f32:
01724             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
01725             break;
01726           case MVT::f64:
01727             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
01728             break;
01729           }
01730         } else {
01731           switch (EltVT.getSimpleVT().SimpleTy) {
01732           default:
01733             return nullptr;
01734           case MVT::i8:
01735             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
01736             break;
01737           case MVT::i16:
01738             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
01739             break;
01740           case MVT::i32:
01741             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
01742             break;
01743           case MVT::i64:
01744             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
01745             break;
01746           case MVT::f32:
01747             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
01748             break;
01749           case MVT::f64:
01750             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
01751             break;
01752           }
01753         }
01754         break;
01755       case NVPTXISD::LDGV2:
01756         switch (EltVT.getSimpleVT().SimpleTy) {
01757         default:
01758           return nullptr;
01759         case MVT::i8:
01760           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
01761           break;
01762         case MVT::i16:
01763           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
01764           break;
01765         case MVT::i32:
01766           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
01767           break;
01768         case MVT::i64:
01769           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
01770           break;
01771         case MVT::f32:
01772           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
01773           break;
01774         case MVT::f64:
01775           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
01776           break;
01777         }
01778         break;
01779       case NVPTXISD::LDUV2:
01780         switch (EltVT.getSimpleVT().SimpleTy) {
01781         default:
01782           return nullptr;
01783         case MVT::i8:
01784           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
01785           break;
01786         case MVT::i16:
01787           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
01788           break;
01789         case MVT::i32:
01790           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
01791           break;
01792         case MVT::i64:
01793           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
01794           break;
01795         case MVT::f32:
01796           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
01797           break;
01798         case MVT::f64:
01799           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
01800           break;
01801         }
01802         break;
01803       case NVPTXISD::LDGV4:
01804         switch (EltVT.getSimpleVT().SimpleTy) {
01805         default:
01806           return nullptr;
01807         case MVT::i8:
01808           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
01809           break;
01810         case MVT::i16:
01811           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
01812           break;
01813         case MVT::i32:
01814           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
01815           break;
01816         case MVT::f32:
01817           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
01818           break;
01819         }
01820         break;
01821       case NVPTXISD::LDUV4:
01822         switch (EltVT.getSimpleVT().SimpleTy) {
01823         default:
01824           return nullptr;
01825         case MVT::i8:
01826           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
01827           break;
01828         case MVT::i16:
01829           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
01830           break;
01831         case MVT::i32:
01832           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
01833           break;
01834         case MVT::f32:
01835           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
01836           break;
01837         }
01838         break;
01839       }
01840     } else {
01841       switch (N->getOpcode()) {
01842       default:
01843         return nullptr;
01844       case ISD::INTRINSIC_W_CHAIN:
01845         if (IsLDG) {
01846           switch (EltVT.getSimpleVT().SimpleTy) {
01847           default:
01848             return nullptr;
01849           case MVT::i8:
01850             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
01851             break;
01852           case MVT::i16:
01853             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
01854             break;
01855           case MVT::i32:
01856             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
01857             break;
01858           case MVT::i64:
01859             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
01860             break;
01861           case MVT::f32:
01862             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
01863             break;
01864           case MVT::f64:
01865             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
01866             break;
01867           }
01868         } else {
01869           switch (EltVT.getSimpleVT().SimpleTy) {
01870           default:
01871             return nullptr;
01872           case MVT::i8:
01873             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
01874             break;
01875           case MVT::i16:
01876             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
01877             break;
01878           case MVT::i32:
01879             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
01880             break;
01881           case MVT::i64:
01882             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
01883             break;
01884           case MVT::f32:
01885             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
01886             break;
01887           case MVT::f64:
01888             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
01889             break;
01890           }
01891         }
01892         break;
01893       case NVPTXISD::LDGV2:
01894         switch (EltVT.getSimpleVT().SimpleTy) {
01895         default:
01896           return nullptr;
01897         case MVT::i8:
01898           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
01899           break;
01900         case MVT::i16:
01901           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
01902           break;
01903         case MVT::i32:
01904           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
01905           break;
01906         case MVT::i64:
01907           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
01908           break;
01909         case MVT::f32:
01910           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
01911           break;
01912         case MVT::f64:
01913           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
01914           break;
01915         }
01916         break;
01917       case NVPTXISD::LDUV2:
01918         switch (EltVT.getSimpleVT().SimpleTy) {
01919         default:
01920           return nullptr;
01921         case MVT::i8:
01922           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
01923           break;
01924         case MVT::i16:
01925           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
01926           break;
01927         case MVT::i32:
01928           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
01929           break;
01930         case MVT::i64:
01931           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
01932           break;
01933         case MVT::f32:
01934           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
01935           break;
01936         case MVT::f64:
01937           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
01938           break;
01939         }
01940         break;
01941       case NVPTXISD::LDGV4:
01942         switch (EltVT.getSimpleVT().SimpleTy) {
01943         default:
01944           return nullptr;
01945         case MVT::i8:
01946           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
01947           break;
01948         case MVT::i16:
01949           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
01950           break;
01951         case MVT::i32:
01952           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
01953           break;
01954         case MVT::f32:
01955           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
01956           break;
01957         }
01958         break;
01959       case NVPTXISD::LDUV4:
01960         switch (EltVT.getSimpleVT().SimpleTy) {
01961         default:
01962           return nullptr;
01963         case MVT::i8:
01964           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
01965           break;
01966         case MVT::i16:
01967           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
01968           break;
01969         case MVT::i32:
01970           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
01971           break;
01972         case MVT::f32:
01973           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
01974           break;
01975         }
01976         break;
01977       }
01978     }
01979 
01980     SDValue Ops[] = { Op1, Chain };
01981     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01982   }
01983 
01984   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01985   MemRefs0[0] = Mem->getMemOperand();
01986   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
01987 
01988   return LD;
01989 }
01990 
01991 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
01992   SDLoc dl(N);
01993   StoreSDNode *ST = cast<StoreSDNode>(N);
01994   EVT StoreVT = ST->getMemoryVT();
01995   SDNode *NVPTXST = nullptr;
01996 
01997   // do not support pre/post inc/dec
01998   if (ST->isIndexed())
01999     return nullptr;
02000 
02001   if (!StoreVT.isSimple())
02002     return nullptr;
02003 
02004   // Address Space Setting
02005   unsigned int codeAddrSpace = getCodeAddrSpace(ST);
02006 
02007   // Volatile Setting
02008   // - .volatile is only availalble for .global and .shared
02009   bool isVolatile = ST->isVolatile();
02010   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
02011       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
02012       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
02013     isVolatile = false;
02014 
02015   // Vector Setting
02016   MVT SimpleVT = StoreVT.getSimpleVT();
02017   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
02018   if (SimpleVT.isVector()) {
02019     unsigned num = SimpleVT.getVectorNumElements();
02020     if (num == 2)
02021       vecType = NVPTX::PTXLdStInstCode::V2;
02022     else if (num == 4)
02023       vecType = NVPTX::PTXLdStInstCode::V4;
02024     else
02025       return nullptr;
02026   }
02027 
02028   // Type Setting: toType + toTypeWidth
02029   // - for integer type, always use 'u'
02030   //
02031   MVT ScalarVT = SimpleVT.getScalarType();
02032   unsigned toTypeWidth = ScalarVT.getSizeInBits();
02033   unsigned int toType;
02034   if (ScalarVT.isFloatingPoint())
02035     toType = NVPTX::PTXLdStInstCode::Float;
02036   else
02037     toType = NVPTX::PTXLdStInstCode::Unsigned;
02038 
02039   // Create the machine instruction DAG
02040   SDValue Chain = N->getOperand(0);
02041   SDValue N1 = N->getOperand(1);
02042   SDValue N2 = N->getOperand(2);
02043   SDValue Addr;
02044   SDValue Offset, Base;
02045   unsigned Opcode;
02046   MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
02047 
02048   if (SelectDirectAddr(N2, Addr)) {
02049     switch (SourceVT) {
02050     case MVT::i8:
02051       Opcode = NVPTX::ST_i8_avar;
02052       break;
02053     case MVT::i16:
02054       Opcode = NVPTX::ST_i16_avar;
02055       break;
02056     case MVT::i32:
02057       Opcode = NVPTX::ST_i32_avar;
02058       break;
02059     case MVT::i64:
02060       Opcode = NVPTX::ST_i64_avar;
02061       break;
02062     case MVT::f32:
02063       Opcode = NVPTX::ST_f32_avar;
02064       break;
02065     case MVT::f64:
02066       Opcode = NVPTX::ST_f64_avar;
02067       break;
02068     default:
02069       return nullptr;
02070     }
02071     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
02072                       getI32Imm(vecType), getI32Imm(toType),
02073                       getI32Imm(toTypeWidth), Addr, Chain };
02074     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02075   } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
02076                           : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
02077     switch (SourceVT) {
02078     case MVT::i8:
02079       Opcode = NVPTX::ST_i8_asi;
02080       break;
02081     case MVT::i16:
02082       Opcode = NVPTX::ST_i16_asi;
02083       break;
02084     case MVT::i32:
02085       Opcode = NVPTX::ST_i32_asi;
02086       break;
02087     case MVT::i64:
02088       Opcode = NVPTX::ST_i64_asi;
02089       break;
02090     case MVT::f32:
02091       Opcode = NVPTX::ST_f32_asi;
02092       break;
02093     case MVT::f64:
02094       Opcode = NVPTX::ST_f64_asi;
02095       break;
02096     default:
02097       return nullptr;
02098     }
02099     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
02100                       getI32Imm(vecType), getI32Imm(toType),
02101                       getI32Imm(toTypeWidth), Base, Offset, Chain };
02102     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02103   } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
02104                           : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
02105     if (TM.is64Bit()) {
02106       switch (SourceVT) {
02107       case MVT::i8:
02108         Opcode = NVPTX::ST_i8_ari_64;
02109         break;
02110       case MVT::i16:
02111         Opcode = NVPTX::ST_i16_ari_64;
02112         break;
02113       case MVT::i32:
02114         Opcode = NVPTX::ST_i32_ari_64;
02115         break;
02116       case MVT::i64:
02117         Opcode = NVPTX::ST_i64_ari_64;
02118         break;
02119       case MVT::f32:
02120         Opcode = NVPTX::ST_f32_ari_64;
02121         break;
02122       case MVT::f64:
02123         Opcode = NVPTX::ST_f64_ari_64;
02124         break;
02125       default:
02126         return nullptr;
02127       }
02128     } else {
02129       switch (SourceVT) {
02130       case MVT::i8:
02131         Opcode = NVPTX::ST_i8_ari;
02132         break;
02133       case MVT::i16:
02134         Opcode = NVPTX::ST_i16_ari;
02135         break;
02136       case MVT::i32:
02137         Opcode = NVPTX::ST_i32_ari;
02138         break;
02139       case MVT::i64:
02140         Opcode = NVPTX::ST_i64_ari;
02141         break;
02142       case MVT::f32:
02143         Opcode = NVPTX::ST_f32_ari;
02144         break;
02145       case MVT::f64:
02146         Opcode = NVPTX::ST_f64_ari;
02147         break;
02148       default:
02149         return nullptr;
02150       }
02151     }
02152     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
02153                       getI32Imm(vecType), getI32Imm(toType),
02154                       getI32Imm(toTypeWidth), Base, Offset, Chain };
02155     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02156   } else {
02157     if (TM.is64Bit()) {
02158       switch (SourceVT) {
02159       case MVT::i8:
02160         Opcode = NVPTX::ST_i8_areg_64;
02161         break;
02162       case MVT::i16:
02163         Opcode = NVPTX::ST_i16_areg_64;
02164         break;
02165       case MVT::i32:
02166         Opcode = NVPTX::ST_i32_areg_64;
02167         break;
02168       case MVT::i64:
02169         Opcode = NVPTX::ST_i64_areg_64;
02170         break;
02171       case MVT::f32:
02172         Opcode = NVPTX::ST_f32_areg_64;
02173         break;
02174       case MVT::f64:
02175         Opcode = NVPTX::ST_f64_areg_64;
02176         break;
02177       default:
02178         return nullptr;
02179       }
02180     } else {
02181       switch (SourceVT) {
02182       case MVT::i8:
02183         Opcode = NVPTX::ST_i8_areg;
02184         break;
02185       case MVT::i16:
02186         Opcode = NVPTX::ST_i16_areg;
02187         break;
02188       case MVT::i32:
02189         Opcode = NVPTX::ST_i32_areg;
02190         break;
02191       case MVT::i64:
02192         Opcode = NVPTX::ST_i64_areg;
02193         break;
02194       case MVT::f32:
02195         Opcode = NVPTX::ST_f32_areg;
02196         break;
02197       case MVT::f64:
02198         Opcode = NVPTX::ST_f64_areg;
02199         break;
02200       default:
02201         return nullptr;
02202       }
02203     }
02204     SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
02205                       getI32Imm(vecType), getI32Imm(toType),
02206                       getI32Imm(toTypeWidth), N2, Chain };
02207     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02208   }
02209 
02210   if (NVPTXST) {
02211     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02212     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02213     cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
02214   }
02215 
02216   return NVPTXST;
02217 }
02218 
02219 SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
02220   SDValue Chain = N->getOperand(0);
02221   SDValue Op1 = N->getOperand(1);
02222   SDValue Addr, Offset, Base;
02223   unsigned Opcode;
02224   SDLoc DL(N);
02225   SDNode *ST;
02226   EVT EltVT = Op1.getValueType();
02227   MemSDNode *MemSD = cast<MemSDNode>(N);
02228   EVT StoreVT = MemSD->getMemoryVT();
02229 
02230   // Address Space Setting
02231   unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
02232 
02233   if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
02234     report_fatal_error("Cannot store to pointer that points to constant "
02235                        "memory space");
02236   }
02237 
02238   // Volatile Setting
02239   // - .volatile is only availalble for .global and .shared
02240   bool IsVolatile = MemSD->isVolatile();
02241   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
02242       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
02243       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
02244     IsVolatile = false;
02245 
02246   // Type Setting: toType + toTypeWidth
02247   // - for integer type, always use 'u'
02248   assert(StoreVT.isSimple() && "Store value is not simple");
02249   MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
02250   unsigned ToTypeWidth = ScalarVT.getSizeInBits();
02251   unsigned ToType;
02252   if (ScalarVT.isFloatingPoint())
02253     ToType = NVPTX::PTXLdStInstCode::Float;
02254   else
02255     ToType = NVPTX::PTXLdStInstCode::Unsigned;
02256 
02257   SmallVector<SDValue, 12> StOps;
02258   SDValue N2;
02259   unsigned VecType;
02260 
02261   switch (N->getOpcode()) {
02262   case NVPTXISD::StoreV2:
02263     VecType = NVPTX::PTXLdStInstCode::V2;
02264     StOps.push_back(N->getOperand(1));
02265     StOps.push_back(N->getOperand(2));
02266     N2 = N->getOperand(3);
02267     break;
02268   case NVPTXISD::StoreV4:
02269     VecType = NVPTX::PTXLdStInstCode::V4;
02270     StOps.push_back(N->getOperand(1));
02271     StOps.push_back(N->getOperand(2));
02272     StOps.push_back(N->getOperand(3));
02273     StOps.push_back(N->getOperand(4));
02274     N2 = N->getOperand(5);
02275     break;
02276   default:
02277     return nullptr;
02278   }
02279 
02280   StOps.push_back(getI32Imm(IsVolatile));
02281   StOps.push_back(getI32Imm(CodeAddrSpace));
02282   StOps.push_back(getI32Imm(VecType));
02283   StOps.push_back(getI32Imm(ToType));
02284   StOps.push_back(getI32Imm(ToTypeWidth));
02285 
02286   if (SelectDirectAddr(N2, Addr)) {
02287     switch (N->getOpcode()) {
02288     default:
02289       return nullptr;
02290     case NVPTXISD::StoreV2:
02291       switch (EltVT.getSimpleVT().SimpleTy) {
02292       default:
02293         return nullptr;
02294       case MVT::i8:
02295         Opcode = NVPTX::STV_i8_v2_avar;
02296         break;
02297       case MVT::i16:
02298         Opcode = NVPTX::STV_i16_v2_avar;
02299         break;
02300       case MVT::i32:
02301         Opcode = NVPTX::STV_i32_v2_avar;
02302         break;
02303       case MVT::i64:
02304         Opcode = NVPTX::STV_i64_v2_avar;
02305         break;
02306       case MVT::f32:
02307         Opcode = NVPTX::STV_f32_v2_avar;
02308         break;
02309       case MVT::f64:
02310         Opcode = NVPTX::STV_f64_v2_avar;
02311         break;
02312       }
02313       break;
02314     case NVPTXISD::StoreV4:
02315       switch (EltVT.getSimpleVT().SimpleTy) {
02316       default:
02317         return nullptr;
02318       case MVT::i8:
02319         Opcode = NVPTX::STV_i8_v4_avar;
02320         break;
02321       case MVT::i16:
02322         Opcode = NVPTX::STV_i16_v4_avar;
02323         break;
02324       case MVT::i32:
02325         Opcode = NVPTX::STV_i32_v4_avar;
02326         break;
02327       case MVT::f32:
02328         Opcode = NVPTX::STV_f32_v4_avar;
02329         break;
02330       }
02331       break;
02332     }
02333     StOps.push_back(Addr);
02334   } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
02335                           : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
02336     switch (N->getOpcode()) {
02337     default:
02338       return nullptr;
02339     case NVPTXISD::StoreV2:
02340       switch (EltVT.getSimpleVT().SimpleTy) {
02341       default:
02342         return nullptr;
02343       case MVT::i8:
02344         Opcode = NVPTX::STV_i8_v2_asi;
02345         break;
02346       case MVT::i16:
02347         Opcode = NVPTX::STV_i16_v2_asi;
02348         break;
02349       case MVT::i32:
02350         Opcode = NVPTX::STV_i32_v2_asi;
02351         break;
02352       case MVT::i64:
02353         Opcode = NVPTX::STV_i64_v2_asi;
02354         break;
02355       case MVT::f32:
02356         Opcode = NVPTX::STV_f32_v2_asi;
02357         break;
02358       case MVT::f64:
02359         Opcode = NVPTX::STV_f64_v2_asi;
02360         break;
02361       }
02362       break;
02363     case NVPTXISD::StoreV4:
02364       switch (EltVT.getSimpleVT().SimpleTy) {
02365       default:
02366         return nullptr;
02367       case MVT::i8:
02368         Opcode = NVPTX::STV_i8_v4_asi;
02369         break;
02370       case MVT::i16:
02371         Opcode = NVPTX::STV_i16_v4_asi;
02372         break;
02373       case MVT::i32:
02374         Opcode = NVPTX::STV_i32_v4_asi;
02375         break;
02376       case MVT::f32:
02377         Opcode = NVPTX::STV_f32_v4_asi;
02378         break;
02379       }
02380       break;
02381     }
02382     StOps.push_back(Base);
02383     StOps.push_back(Offset);
02384   } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
02385                           : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
02386     if (TM.is64Bit()) {
02387       switch (N->getOpcode()) {
02388       default:
02389         return nullptr;
02390       case NVPTXISD::StoreV2:
02391         switch (EltVT.getSimpleVT().SimpleTy) {
02392         default:
02393           return nullptr;
02394         case MVT::i8:
02395           Opcode = NVPTX::STV_i8_v2_ari_64;
02396           break;
02397         case MVT::i16:
02398           Opcode = NVPTX::STV_i16_v2_ari_64;
02399           break;
02400         case MVT::i32:
02401           Opcode = NVPTX::STV_i32_v2_ari_64;
02402           break;
02403         case MVT::i64:
02404           Opcode = NVPTX::STV_i64_v2_ari_64;
02405           break;
02406         case MVT::f32:
02407           Opcode = NVPTX::STV_f32_v2_ari_64;
02408           break;
02409         case MVT::f64:
02410           Opcode = NVPTX::STV_f64_v2_ari_64;
02411           break;
02412         }
02413         break;
02414       case NVPTXISD::StoreV4:
02415         switch (EltVT.getSimpleVT().SimpleTy) {
02416         default:
02417           return nullptr;
02418         case MVT::i8:
02419           Opcode = NVPTX::STV_i8_v4_ari_64;
02420           break;
02421         case MVT::i16:
02422           Opcode = NVPTX::STV_i16_v4_ari_64;
02423           break;
02424         case MVT::i32:
02425           Opcode = NVPTX::STV_i32_v4_ari_64;
02426           break;
02427         case MVT::f32:
02428           Opcode = NVPTX::STV_f32_v4_ari_64;
02429           break;
02430         }
02431         break;
02432       }
02433     } else {
02434       switch (N->getOpcode()) {
02435       default:
02436         return nullptr;
02437       case NVPTXISD::StoreV2:
02438         switch (EltVT.getSimpleVT().SimpleTy) {
02439         default:
02440           return nullptr;
02441         case MVT::i8:
02442           Opcode = NVPTX::STV_i8_v2_ari;
02443           break;
02444         case MVT::i16:
02445           Opcode = NVPTX::STV_i16_v2_ari;
02446           break;
02447         case MVT::i32:
02448           Opcode = NVPTX::STV_i32_v2_ari;
02449           break;
02450         case MVT::i64:
02451           Opcode = NVPTX::STV_i64_v2_ari;
02452           break;
02453         case MVT::f32:
02454           Opcode = NVPTX::STV_f32_v2_ari;
02455           break;
02456         case MVT::f64:
02457           Opcode = NVPTX::STV_f64_v2_ari;
02458           break;
02459         }
02460         break;
02461       case NVPTXISD::StoreV4:
02462         switch (EltVT.getSimpleVT().SimpleTy) {
02463         default:
02464           return nullptr;
02465         case MVT::i8:
02466           Opcode = NVPTX::STV_i8_v4_ari;
02467           break;
02468         case MVT::i16:
02469           Opcode = NVPTX::STV_i16_v4_ari;
02470           break;
02471         case MVT::i32:
02472           Opcode = NVPTX::STV_i32_v4_ari;
02473           break;
02474         case MVT::f32:
02475           Opcode = NVPTX::STV_f32_v4_ari;
02476           break;
02477         }
02478         break;
02479       }
02480     }
02481     StOps.push_back(Base);
02482     StOps.push_back(Offset);
02483   } else {
02484     if (TM.is64Bit()) {
02485       switch (N->getOpcode()) {
02486       default:
02487         return nullptr;
02488       case NVPTXISD::StoreV2:
02489         switch (EltVT.getSimpleVT().SimpleTy) {
02490         default:
02491           return nullptr;
02492         case MVT::i8:
02493           Opcode = NVPTX::STV_i8_v2_areg_64;
02494           break;
02495         case MVT::i16:
02496           Opcode = NVPTX::STV_i16_v2_areg_64;
02497           break;
02498         case MVT::i32:
02499           Opcode = NVPTX::STV_i32_v2_areg_64;
02500           break;
02501         case MVT::i64:
02502           Opcode = NVPTX::STV_i64_v2_areg_64;
02503           break;
02504         case MVT::f32:
02505           Opcode = NVPTX::STV_f32_v2_areg_64;
02506           break;
02507         case MVT::f64:
02508           Opcode = NVPTX::STV_f64_v2_areg_64;
02509           break;
02510         }
02511         break;
02512       case NVPTXISD::StoreV4:
02513         switch (EltVT.getSimpleVT().SimpleTy) {
02514         default:
02515           return nullptr;
02516         case MVT::i8:
02517           Opcode = NVPTX::STV_i8_v4_areg_64;
02518           break;
02519         case MVT::i16:
02520           Opcode = NVPTX::STV_i16_v4_areg_64;
02521           break;
02522         case MVT::i32:
02523           Opcode = NVPTX::STV_i32_v4_areg_64;
02524           break;
02525         case MVT::f32:
02526           Opcode = NVPTX::STV_f32_v4_areg_64;
02527           break;
02528         }
02529         break;
02530       }
02531     } else {
02532       switch (N->getOpcode()) {
02533       default:
02534         return nullptr;
02535       case NVPTXISD::StoreV2:
02536         switch (EltVT.getSimpleVT().SimpleTy) {
02537         default:
02538           return nullptr;
02539         case MVT::i8:
02540           Opcode = NVPTX::STV_i8_v2_areg;
02541           break;
02542         case MVT::i16:
02543           Opcode = NVPTX::STV_i16_v2_areg;
02544           break;
02545         case MVT::i32:
02546           Opcode = NVPTX::STV_i32_v2_areg;
02547           break;
02548         case MVT::i64:
02549           Opcode = NVPTX::STV_i64_v2_areg;
02550           break;
02551         case MVT::f32:
02552           Opcode = NVPTX::STV_f32_v2_areg;
02553           break;
02554         case MVT::f64:
02555           Opcode = NVPTX::STV_f64_v2_areg;
02556           break;
02557         }
02558         break;
02559       case NVPTXISD::StoreV4:
02560         switch (EltVT.getSimpleVT().SimpleTy) {
02561         default:
02562           return nullptr;
02563         case MVT::i8:
02564           Opcode = NVPTX::STV_i8_v4_areg;
02565           break;
02566         case MVT::i16:
02567           Opcode = NVPTX::STV_i16_v4_areg;
02568           break;
02569         case MVT::i32:
02570           Opcode = NVPTX::STV_i32_v4_areg;
02571           break;
02572         case MVT::f32:
02573           Opcode = NVPTX::STV_f32_v4_areg;
02574           break;
02575         }
02576         break;
02577       }
02578     }
02579     StOps.push_back(N2);
02580   }
02581 
02582   StOps.push_back(Chain);
02583 
02584   ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
02585 
02586   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02587   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02588   cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
02589 
02590   return ST;
02591 }
02592 
02593 SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
02594   SDValue Chain = Node->getOperand(0);
02595   SDValue Offset = Node->getOperand(2);
02596   SDValue Flag = Node->getOperand(3);
02597   SDLoc DL(Node);
02598   MemSDNode *Mem = cast<MemSDNode>(Node);
02599 
02600   unsigned VecSize;
02601   switch (Node->getOpcode()) {
02602   default:
02603     return nullptr;
02604   case NVPTXISD::LoadParam:
02605     VecSize = 1;
02606     break;
02607   case NVPTXISD::LoadParamV2:
02608     VecSize = 2;
02609     break;
02610   case NVPTXISD::LoadParamV4:
02611     VecSize = 4;
02612     break;
02613   }
02614 
02615   EVT EltVT = Node->getValueType(0);
02616   EVT MemVT = Mem->getMemoryVT();
02617 
02618   unsigned Opc = 0;
02619 
02620   switch (VecSize) {
02621   default:
02622     return nullptr;
02623   case 1:
02624     switch (MemVT.getSimpleVT().SimpleTy) {
02625     default:
02626       return nullptr;
02627     case MVT::i1:
02628       Opc = NVPTX::LoadParamMemI8;
02629       break;
02630     case MVT::i8:
02631       Opc = NVPTX::LoadParamMemI8;
02632       break;
02633     case MVT::i16:
02634       Opc = NVPTX::LoadParamMemI16;
02635       break;
02636     case MVT::i32:
02637       Opc = NVPTX::LoadParamMemI32;
02638       break;
02639     case MVT::i64:
02640       Opc = NVPTX::LoadParamMemI64;
02641       break;
02642     case MVT::f32:
02643       Opc = NVPTX::LoadParamMemF32;
02644       break;
02645     case MVT::f64:
02646       Opc = NVPTX::LoadParamMemF64;
02647       break;
02648     }
02649     break;
02650   case 2:
02651     switch (MemVT.getSimpleVT().SimpleTy) {
02652     default:
02653       return nullptr;
02654     case MVT::i1:
02655       Opc = NVPTX::LoadParamMemV2I8;
02656       break;
02657     case MVT::i8:
02658       Opc = NVPTX::LoadParamMemV2I8;
02659       break;
02660     case MVT::i16:
02661       Opc = NVPTX::LoadParamMemV2I16;
02662       break;
02663     case MVT::i32:
02664       Opc = NVPTX::LoadParamMemV2I32;
02665       break;
02666     case MVT::i64:
02667       Opc = NVPTX::LoadParamMemV2I64;
02668       break;
02669     case MVT::f32:
02670       Opc = NVPTX::LoadParamMemV2F32;
02671       break;
02672     case MVT::f64:
02673       Opc = NVPTX::LoadParamMemV2F64;
02674       break;
02675     }
02676     break;
02677   case 4:
02678     switch (MemVT.getSimpleVT().SimpleTy) {
02679     default:
02680       return nullptr;
02681     case MVT::i1:
02682       Opc = NVPTX::LoadParamMemV4I8;
02683       break;
02684     case MVT::i8:
02685       Opc = NVPTX::LoadParamMemV4I8;
02686       break;
02687     case MVT::i16:
02688       Opc = NVPTX::LoadParamMemV4I16;
02689       break;
02690     case MVT::i32:
02691       Opc = NVPTX::LoadParamMemV4I32;
02692       break;
02693     case MVT::f32:
02694       Opc = NVPTX::LoadParamMemV4F32;
02695       break;
02696     }
02697     break;
02698   }
02699 
02700   SDVTList VTs;
02701   if (VecSize == 1) {
02702     VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
02703   } else if (VecSize == 2) {
02704     VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
02705   } else {
02706     EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
02707     VTs = CurDAG->getVTList(EVTs);
02708   }
02709 
02710   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02711 
02712   SmallVector<SDValue, 2> Ops;
02713   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
02714   Ops.push_back(Chain);
02715   Ops.push_back(Flag);
02716 
02717   SDNode *Ret =
02718       CurDAG->getMachineNode(Opc, DL, VTs, Ops);
02719   return Ret;
02720 }
02721 
02722 SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
02723   SDLoc DL(N);
02724   SDValue Chain = N->getOperand(0);
02725   SDValue Offset = N->getOperand(1);
02726   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02727   MemSDNode *Mem = cast<MemSDNode>(N);
02728 
02729   // How many elements do we have?
02730   unsigned NumElts = 1;
02731   switch (N->getOpcode()) {
02732   default:
02733     return nullptr;
02734   case NVPTXISD::StoreRetval:
02735     NumElts = 1;
02736     break;
02737   case NVPTXISD::StoreRetvalV2:
02738     NumElts = 2;
02739     break;
02740   case NVPTXISD::StoreRetvalV4:
02741     NumElts = 4;
02742     break;
02743   }
02744 
02745   // Build vector of operands
02746   SmallVector<SDValue, 6> Ops;
02747   for (unsigned i = 0; i < NumElts; ++i)
02748     Ops.push_back(N->getOperand(i + 2));
02749   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
02750   Ops.push_back(Chain);
02751 
02752   // Determine target opcode
02753   // If we have an i1, use an 8-bit store. The lowering code in
02754   // NVPTXISelLowering will have already emitted an upcast.
02755   unsigned Opcode = 0;
02756   switch (NumElts) {
02757   default:
02758     return nullptr;
02759   case 1:
02760     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02761     default:
02762       return nullptr;
02763     case MVT::i1:
02764       Opcode = NVPTX::StoreRetvalI8;
02765       break;
02766     case MVT::i8:
02767       Opcode = NVPTX::StoreRetvalI8;
02768       break;
02769     case MVT::i16:
02770       Opcode = NVPTX::StoreRetvalI16;
02771       break;
02772     case MVT::i32:
02773       Opcode = NVPTX::StoreRetvalI32;
02774       break;
02775     case MVT::i64:
02776       Opcode = NVPTX::StoreRetvalI64;
02777       break;
02778     case MVT::f32:
02779       Opcode = NVPTX::StoreRetvalF32;
02780       break;
02781     case MVT::f64:
02782       Opcode = NVPTX::StoreRetvalF64;
02783       break;
02784     }
02785     break;
02786   case 2:
02787     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02788     default:
02789       return nullptr;
02790     case MVT::i1:
02791       Opcode = NVPTX::StoreRetvalV2I8;
02792       break;
02793     case MVT::i8:
02794       Opcode = NVPTX::StoreRetvalV2I8;
02795       break;
02796     case MVT::i16:
02797       Opcode = NVPTX::StoreRetvalV2I16;
02798       break;
02799     case MVT::i32:
02800       Opcode = NVPTX::StoreRetvalV2I32;
02801       break;
02802     case MVT::i64:
02803       Opcode = NVPTX::StoreRetvalV2I64;
02804       break;
02805     case MVT::f32:
02806       Opcode = NVPTX::StoreRetvalV2F32;
02807       break;
02808     case MVT::f64:
02809       Opcode = NVPTX::StoreRetvalV2F64;
02810       break;
02811     }
02812     break;
02813   case 4:
02814     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02815     default:
02816       return nullptr;
02817     case MVT::i1:
02818       Opcode = NVPTX::StoreRetvalV4I8;
02819       break;
02820     case MVT::i8:
02821       Opcode = NVPTX::StoreRetvalV4I8;
02822       break;
02823     case MVT::i16:
02824       Opcode = NVPTX::StoreRetvalV4I16;
02825       break;
02826     case MVT::i32:
02827       Opcode = NVPTX::StoreRetvalV4I32;
02828       break;
02829     case MVT::f32:
02830       Opcode = NVPTX::StoreRetvalV4F32;
02831       break;
02832     }
02833     break;
02834   }
02835 
02836   SDNode *Ret =
02837       CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
02838   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02839   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02840   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
02841 
02842   return Ret;
02843 }
02844 
02845 SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
02846   SDLoc DL(N);
02847   SDValue Chain = N->getOperand(0);
02848   SDValue Param = N->getOperand(1);
02849   unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
02850   SDValue Offset = N->getOperand(2);
02851   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02852   MemSDNode *Mem = cast<MemSDNode>(N);
02853   SDValue Flag = N->getOperand(N->getNumOperands() - 1);
02854 
02855   // How many elements do we have?
02856   unsigned NumElts = 1;
02857   switch (N->getOpcode()) {
02858   default:
02859     return nullptr;
02860   case NVPTXISD::StoreParamU32:
02861   case NVPTXISD::StoreParamS32:
02862   case NVPTXISD::StoreParam:
02863     NumElts = 1;
02864     break;
02865   case NVPTXISD::StoreParamV2:
02866     NumElts = 2;
02867     break;
02868   case NVPTXISD::StoreParamV4:
02869     NumElts = 4;
02870     break;
02871   }
02872 
02873   // Build vector of operands
02874   SmallVector<SDValue, 8> Ops;
02875   for (unsigned i = 0; i < NumElts; ++i)
02876     Ops.push_back(N->getOperand(i + 3));
02877   Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
02878   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
02879   Ops.push_back(Chain);
02880   Ops.push_back(Flag);
02881 
02882   // Determine target opcode
02883   // If we have an i1, use an 8-bit store. The lowering code in
02884   // NVPTXISelLowering will have already emitted an upcast.
02885   unsigned Opcode = 0;
02886   switch (N->getOpcode()) {
02887   default:
02888     switch (NumElts) {
02889     default:
02890       return nullptr;
02891     case 1:
02892       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02893       default:
02894         return nullptr;
02895       case MVT::i1:
02896         Opcode = NVPTX::StoreParamI8;
02897         break;
02898       case MVT::i8:
02899         Opcode = NVPTX::StoreParamI8;
02900         break;
02901       case MVT::i16:
02902         Opcode = NVPTX::StoreParamI16;
02903         break;
02904       case MVT::i32:
02905         Opcode = NVPTX::StoreParamI32;
02906         break;
02907       case MVT::i64:
02908         Opcode = NVPTX::StoreParamI64;
02909         break;
02910       case MVT::f32:
02911         Opcode = NVPTX::StoreParamF32;
02912         break;
02913       case MVT::f64:
02914         Opcode = NVPTX::StoreParamF64;
02915         break;
02916       }
02917       break;
02918     case 2:
02919       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02920       default:
02921         return nullptr;
02922       case MVT::i1:
02923         Opcode = NVPTX::StoreParamV2I8;
02924         break;
02925       case MVT::i8:
02926         Opcode = NVPTX::StoreParamV2I8;
02927         break;
02928       case MVT::i16:
02929         Opcode = NVPTX::StoreParamV2I16;
02930         break;
02931       case MVT::i32:
02932         Opcode = NVPTX::StoreParamV2I32;
02933         break;
02934       case MVT::i64:
02935         Opcode = NVPTX::StoreParamV2I64;
02936         break;
02937       case MVT::f32:
02938         Opcode = NVPTX::StoreParamV2F32;
02939         break;
02940       case MVT::f64:
02941         Opcode = NVPTX::StoreParamV2F64;
02942         break;
02943       }
02944       break;
02945     case 4:
02946       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02947       default:
02948         return nullptr;
02949       case MVT::i1:
02950         Opcode = NVPTX::StoreParamV4I8;
02951         break;
02952       case MVT::i8:
02953         Opcode = NVPTX::StoreParamV4I8;
02954         break;
02955       case MVT::i16:
02956         Opcode = NVPTX::StoreParamV4I16;
02957         break;
02958       case MVT::i32:
02959         Opcode = NVPTX::StoreParamV4I32;
02960         break;
02961       case MVT::f32:
02962         Opcode = NVPTX::StoreParamV4F32;
02963         break;
02964       }
02965       break;
02966     }
02967     break;
02968   // Special case: if we have a sign-extend/zero-extend node, insert the
02969   // conversion instruction first, and use that as the value operand to
02970   // the selected StoreParam node.
02971   case NVPTXISD::StoreParamU32: {
02972     Opcode = NVPTX::StoreParamI32;
02973     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
02974                                                 MVT::i32);
02975     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
02976                                          MVT::i32, Ops[0], CvtNone);
02977     Ops[0] = SDValue(Cvt, 0);
02978     break;
02979   }
02980   case NVPTXISD::StoreParamS32: {
02981     Opcode = NVPTX::StoreParamI32;
02982     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
02983                                                 MVT::i32);
02984     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
02985                                          MVT::i32, Ops[0], CvtNone);
02986     Ops[0] = SDValue(Cvt, 0);
02987     break;
02988   }
02989   }
02990 
02991   SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
02992   SDNode *Ret =
02993       CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
02994   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02995   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02996   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
02997 
02998   return Ret;
02999 }
03000 
03001 SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
03002   SDValue Chain = N->getOperand(0);
03003   SDNode *Ret = nullptr;
03004   unsigned Opc = 0;
03005   SmallVector<SDValue, 8> Ops;
03006 
03007   switch (N->getOpcode()) {
03008   default: return nullptr;
03009   case NVPTXISD::Tex1DFloatS32:
03010     Opc = NVPTX::TEX_1D_F32_S32;
03011     break;
03012   case NVPTXISD::Tex1DFloatFloat:
03013     Opc = NVPTX::TEX_1D_F32_F32;
03014     break;
03015   case NVPTXISD::Tex1DFloatFloatLevel:
03016     Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
03017     break;
03018   case NVPTXISD::Tex1DFloatFloatGrad:
03019     Opc = NVPTX::TEX_1D_F32_F32_GRAD;
03020     break;
03021   case NVPTXISD::Tex1DS32S32:
03022     Opc = NVPTX::TEX_1D_S32_S32;
03023     break;
03024   case NVPTXISD::Tex1DS32Float:
03025     Opc = NVPTX::TEX_1D_S32_F32;
03026     break;
03027   case NVPTXISD::Tex1DS32FloatLevel:
03028     Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
03029     break;
03030   case NVPTXISD::Tex1DS32FloatGrad:
03031     Opc = NVPTX::TEX_1D_S32_F32_GRAD;
03032     break;
03033   case NVPTXISD::Tex1DU32S32:
03034     Opc = NVPTX::TEX_1D_U32_S32;
03035     break;
03036   case NVPTXISD::Tex1DU32Float:
03037     Opc = NVPTX::TEX_1D_U32_F32;
03038     break;
03039   case NVPTXISD::Tex1DU32FloatLevel:
03040     Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
03041     break;
03042   case NVPTXISD::Tex1DU32FloatGrad:
03043     Opc = NVPTX::TEX_1D_U32_F32_GRAD;
03044     break;
03045   case NVPTXISD::Tex1DArrayFloatS32:
03046     Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
03047     break;
03048   case NVPTXISD::Tex1DArrayFloatFloat:
03049     Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
03050     break;
03051   case NVPTXISD::Tex1DArrayFloatFloatLevel:
03052     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
03053     break;
03054   case NVPTXISD::Tex1DArrayFloatFloatGrad:
03055     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
03056     break;
03057   case NVPTXISD::Tex1DArrayS32S32:
03058     Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
03059     break;
03060   case NVPTXISD::Tex1DArrayS32Float:
03061     Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
03062     break;
03063   case NVPTXISD::Tex1DArrayS32FloatLevel:
03064     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
03065     break;
03066   case NVPTXISD::Tex1DArrayS32FloatGrad:
03067     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
03068     break;
03069   case NVPTXISD::Tex1DArrayU32S32:
03070     Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
03071     break;
03072   case NVPTXISD::Tex1DArrayU32Float:
03073     Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
03074     break;
03075   case NVPTXISD::Tex1DArrayU32FloatLevel:
03076     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
03077     break;
03078   case NVPTXISD::Tex1DArrayU32FloatGrad:
03079     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
03080     break;
03081   case NVPTXISD::Tex2DFloatS32:
03082     Opc = NVPTX::TEX_2D_F32_S32;
03083     break;
03084   case NVPTXISD::Tex2DFloatFloat:
03085     Opc = NVPTX::TEX_2D_F32_F32;
03086     break;
03087   case NVPTXISD::Tex2DFloatFloatLevel:
03088     Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
03089     break;
03090   case NVPTXISD::Tex2DFloatFloatGrad:
03091     Opc = NVPTX::TEX_2D_F32_F32_GRAD;
03092     break;
03093   case NVPTXISD::Tex2DS32S32:
03094     Opc = NVPTX::TEX_2D_S32_S32;
03095     break;
03096   case NVPTXISD::Tex2DS32Float:
03097     Opc = NVPTX::TEX_2D_S32_F32;
03098     break;
03099   case NVPTXISD::Tex2DS32FloatLevel:
03100     Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
03101     break;
03102   case NVPTXISD::Tex2DS32FloatGrad:
03103     Opc = NVPTX::TEX_2D_S32_F32_GRAD;
03104     break;
03105   case NVPTXISD::Tex2DU32S32:
03106     Opc = NVPTX::TEX_2D_U32_S32;
03107     break;
03108   case NVPTXISD::Tex2DU32Float:
03109     Opc = NVPTX::TEX_2D_U32_F32;
03110     break;
03111   case NVPTXISD::Tex2DU32FloatLevel:
03112     Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
03113     break;
03114   case NVPTXISD::Tex2DU32FloatGrad:
03115     Opc = NVPTX::TEX_2D_U32_F32_GRAD;
03116     break;
03117   case NVPTXISD::Tex2DArrayFloatS32:
03118     Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
03119     break;
03120   case NVPTXISD::Tex2DArrayFloatFloat:
03121     Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
03122     break;
03123   case NVPTXISD::Tex2DArrayFloatFloatLevel:
03124     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
03125     break;
03126   case NVPTXISD::Tex2DArrayFloatFloatGrad:
03127     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
03128     break;
03129   case NVPTXISD::Tex2DArrayS32S32:
03130     Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
03131     break;
03132   case NVPTXISD::Tex2DArrayS32Float:
03133     Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
03134     break;
03135   case NVPTXISD::Tex2DArrayS32FloatLevel:
03136     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
03137     break;
03138   case NVPTXISD::Tex2DArrayS32FloatGrad:
03139     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
03140     break;
03141   case NVPTXISD::Tex2DArrayU32S32:
03142     Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
03143     break;
03144   case NVPTXISD::Tex2DArrayU32Float:
03145     Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
03146     break;
03147   case NVPTXISD::Tex2DArrayU32FloatLevel:
03148     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
03149     break;
03150   case NVPTXISD::Tex2DArrayU32FloatGrad:
03151     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
03152     break;
03153   case NVPTXISD::Tex3DFloatS32:
03154     Opc = NVPTX::TEX_3D_F32_S32;
03155     break;
03156   case NVPTXISD::Tex3DFloatFloat:
03157     Opc = NVPTX::TEX_3D_F32_F32;
03158     break;
03159   case NVPTXISD::Tex3DFloatFloatLevel:
03160     Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
03161     break;
03162   case NVPTXISD::Tex3DFloatFloatGrad:
03163     Opc = NVPTX::TEX_3D_F32_F32_GRAD;
03164     break;
03165   case NVPTXISD::Tex3DS32S32:
03166     Opc = NVPTX::TEX_3D_S32_S32;
03167     break;
03168   case NVPTXISD::Tex3DS32Float:
03169     Opc = NVPTX::TEX_3D_S32_F32;
03170     break;
03171   case NVPTXISD::Tex3DS32FloatLevel:
03172     Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
03173     break;
03174   case NVPTXISD::Tex3DS32FloatGrad:
03175     Opc = NVPTX::TEX_3D_S32_F32_GRAD;
03176     break;
03177   case NVPTXISD::Tex3DU32S32:
03178     Opc = NVPTX::TEX_3D_U32_S32;
03179     break;
03180   case NVPTXISD::Tex3DU32Float:
03181     Opc = NVPTX::TEX_3D_U32_F32;
03182     break;
03183   case NVPTXISD::Tex3DU32FloatLevel:
03184     Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
03185     break;
03186   case NVPTXISD::Tex3DU32FloatGrad:
03187     Opc = NVPTX::TEX_3D_U32_F32_GRAD;
03188     break;
03189   case NVPTXISD::TexCubeFloatFloat:
03190     Opc = NVPTX::TEX_CUBE_F32_F32;
03191     break;
03192   case NVPTXISD::TexCubeFloatFloatLevel:
03193     Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
03194     break;
03195   case NVPTXISD::TexCubeS32Float:
03196     Opc = NVPTX::TEX_CUBE_S32_F32;
03197     break;
03198   case NVPTXISD::TexCubeS32FloatLevel:
03199     Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
03200     break;
03201   case NVPTXISD::TexCubeU32Float:
03202     Opc = NVPTX::TEX_CUBE_U32_F32;
03203     break;
03204   case NVPTXISD::TexCubeU32FloatLevel:
03205     Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
03206     break;
03207   case NVPTXISD::TexCubeArrayFloatFloat:
03208     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
03209     break;
03210   case NVPTXISD::TexCubeArrayFloatFloatLevel:
03211     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
03212     break;
03213   case NVPTXISD::TexCubeArrayS32Float:
03214     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
03215     break;
03216   case NVPTXISD::TexCubeArrayS32FloatLevel:
03217     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
03218     break;
03219   case NVPTXISD::TexCubeArrayU32Float:
03220     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
03221     break;
03222   case NVPTXISD::TexCubeArrayU32FloatLevel:
03223     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
03224     break;
03225   case NVPTXISD::Tld4R2DFloatFloat:
03226     Opc = NVPTX::TLD4_R_2D_F32_F32;
03227     break;
03228   case NVPTXISD::Tld4G2DFloatFloat:
03229     Opc = NVPTX::TLD4_G_2D_F32_F32;
03230     break;
03231   case NVPTXISD::Tld4B2DFloatFloat:
03232     Opc = NVPTX::TLD4_B_2D_F32_F32;
03233     break;
03234   case NVPTXISD::Tld4A2DFloatFloat:
03235     Opc = NVPTX::TLD4_A_2D_F32_F32;
03236     break;
03237   case NVPTXISD::Tld4R2DS64Float:
03238     Opc = NVPTX::TLD4_R_2D_S32_F32;
03239     break;
03240   case NVPTXISD::Tld4G2DS64Float:
03241     Opc = NVPTX::TLD4_G_2D_S32_F32;
03242     break;
03243   case NVPTXISD::Tld4B2DS64Float:
03244     Opc = NVPTX::TLD4_B_2D_S32_F32;
03245     break;
03246   case NVPTXISD::Tld4A2DS64Float:
03247     Opc = NVPTX::TLD4_A_2D_S32_F32;
03248     break;
03249   case NVPTXISD::Tld4R2DU64Float:
03250     Opc = NVPTX::TLD4_R_2D_U32_F32;
03251     break;
03252   case NVPTXISD::Tld4G2DU64Float:
03253     Opc = NVPTX::TLD4_G_2D_U32_F32;
03254     break;
03255   case NVPTXISD::Tld4B2DU64Float:
03256     Opc = NVPTX::TLD4_B_2D_U32_F32;
03257     break;
03258   case NVPTXISD::Tld4A2DU64Float:
03259     Opc = NVPTX::TLD4_A_2D_U32_F32;
03260     break;
03261   case NVPTXISD::TexUnified1DFloatS32:
03262     Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
03263     break;
03264   case NVPTXISD::TexUnified1DFloatFloat:
03265     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
03266     break;
03267   case NVPTXISD::TexUnified1DFloatFloatLevel:
03268     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
03269     break;
03270   case NVPTXISD::TexUnified1DFloatFloatGrad:
03271     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
03272     break;
03273   case NVPTXISD::TexUnified1DS32S32:
03274     Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
03275     break;
03276   case NVPTXISD::TexUnified1DS32Float:
03277     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
03278     break;
03279   case NVPTXISD::TexUnified1DS32FloatLevel:
03280     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
03281     break;
03282   case NVPTXISD::TexUnified1DS32FloatGrad:
03283     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
03284     break;
03285   case NVPTXISD::TexUnified1DU32S32:
03286     Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
03287     break;
03288   case NVPTXISD::TexUnified1DU32Float:
03289     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
03290     break;
03291   case NVPTXISD::TexUnified1DU32FloatLevel:
03292     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
03293     break;
03294   case NVPTXISD::TexUnified1DU32FloatGrad:
03295     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
03296     break;
03297   case NVPTXISD::TexUnified1DArrayFloatS32:
03298     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
03299     break;
03300   case NVPTXISD::TexUnified1DArrayFloatFloat:
03301     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
03302     break;
03303   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
03304     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
03305     break;
03306   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
03307     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
03308     break;
03309   case NVPTXISD::TexUnified1DArrayS32S32:
03310     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
03311     break;
03312   case NVPTXISD::TexUnified1DArrayS32Float:
03313     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
03314     break;
03315   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
03316     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
03317     break;
03318   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
03319     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
03320     break;
03321   case NVPTXISD::TexUnified1DArrayU32S32:
03322     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
03323     break;
03324   case NVPTXISD::TexUnified1DArrayU32Float:
03325     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
03326     break;
03327   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
03328     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
03329     break;
03330   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
03331     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
03332     break;
03333   case NVPTXISD::TexUnified2DFloatS32:
03334     Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
03335     break;
03336   case NVPTXISD::TexUnified2DFloatFloat:
03337     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
03338     break;
03339   case NVPTXISD::TexUnified2DFloatFloatLevel:
03340     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
03341     break;
03342   case NVPTXISD::TexUnified2DFloatFloatGrad:
03343     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
03344     break;
03345   case NVPTXISD::TexUnified2DS32S32:
03346     Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
03347     break;
03348   case NVPTXISD::TexUnified2DS32Float:
03349     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
03350     break;
03351   case NVPTXISD::TexUnified2DS32FloatLevel:
03352     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
03353     break;
03354   case NVPTXISD::TexUnified2DS32FloatGrad:
03355     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
03356     break;
03357   case NVPTXISD::TexUnified2DU32S32:
03358     Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
03359     break;
03360   case NVPTXISD::TexUnified2DU32Float:
03361     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
03362     break;
03363   case NVPTXISD::TexUnified2DU32FloatLevel:
03364     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
03365     break;
03366   case NVPTXISD::TexUnified2DU32FloatGrad:
03367     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
03368     break;
03369   case NVPTXISD::TexUnified2DArrayFloatS32:
03370     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
03371     break;
03372   case NVPTXISD::TexUnified2DArrayFloatFloat:
03373     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
03374     break;
03375   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
03376     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
03377     break;
03378   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
03379     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
03380     break;
03381   case NVPTXISD::TexUnified2DArrayS32S32:
03382     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
03383     break;
03384   case NVPTXISD::TexUnified2DArrayS32Float:
03385     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
03386     break;
03387   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
03388     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
03389     break;
03390   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
03391     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
03392     break;
03393   case NVPTXISD::TexUnified2DArrayU32S32:
03394     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
03395     break;
03396   case NVPTXISD::TexUnified2DArrayU32Float:
03397     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
03398     break;
03399   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
03400     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
03401     break;
03402   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
03403     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
03404     break;
03405   case NVPTXISD::TexUnified3DFloatS32:
03406     Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
03407     break;
03408   case NVPTXISD::TexUnified3DFloatFloat:
03409     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
03410     break;
03411   case NVPTXISD::TexUnified3DFloatFloatLevel:
03412     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
03413     break;
03414   case NVPTXISD::TexUnified3DFloatFloatGrad:
03415     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
03416     break;
03417   case NVPTXISD::TexUnified3DS32S32:
03418     Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
03419     break;
03420   case NVPTXISD::TexUnified3DS32Float:
03421     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
03422     break;
03423   case NVPTXISD::TexUnified3DS32FloatLevel:
03424     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
03425     break;
03426   case NVPTXISD::TexUnified3DS32FloatGrad:
03427     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
03428     break;
03429   case NVPTXISD::TexUnified3DU32S32:
03430     Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
03431     break;
03432   case NVPTXISD::TexUnified3DU32Float:
03433     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
03434     break;
03435   case NVPTXISD::TexUnified3DU32FloatLevel:
03436     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
03437     break;
03438   case NVPTXISD::TexUnified3DU32FloatGrad:
03439     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
03440     break;
03441   case NVPTXISD::TexUnifiedCubeFloatFloat:
03442     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
03443     break;
03444   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
03445     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
03446     break;
03447   case NVPTXISD::TexUnifiedCubeS32Float:
03448     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
03449     break;
03450   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
03451     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
03452     break;
03453   case NVPTXISD::TexUnifiedCubeU32Float:
03454     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
03455     break;
03456   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
03457     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
03458     break;
03459   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
03460     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
03461     break;
03462   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
03463     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
03464     break;
03465   case NVPTXISD::TexUnifiedCubeArrayS32Float:
03466     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
03467     break;
03468   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
03469     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
03470     break;
03471   case NVPTXISD::TexUnifiedCubeArrayU32Float:
03472     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
03473     break;
03474   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
03475     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
03476     break;
03477   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
03478     Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
03479     break;
03480   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
03481     Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
03482     break;
03483   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
03484     Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
03485     break;
03486   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
03487     Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
03488     break;
03489   case NVPTXISD::Tld4UnifiedR2DS64Float:
03490     Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
03491     break;
03492   case NVPTXISD::Tld4UnifiedG2DS64Float:
03493     Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
03494     break;
03495   case NVPTXISD::Tld4UnifiedB2DS64Float:
03496     Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
03497     break;
03498   case NVPTXISD::Tld4UnifiedA2DS64Float:
03499     Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
03500     break;
03501   case NVPTXISD::Tld4UnifiedR2DU64Float:
03502     Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
03503     break;
03504   case NVPTXISD::Tld4UnifiedG2DU64Float:
03505     Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
03506     break;
03507   case NVPTXISD::Tld4UnifiedB2DU64Float:
03508     Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
03509     break;
03510   case NVPTXISD::Tld4UnifiedA2DU64Float:
03511     Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
03512     break;
03513   }
03514 
03515   // Copy over operands
03516   for (unsigned i = 1; i < N->getNumOperands(); ++i) {
03517     Ops.push_back(N->getOperand(i));
03518   }
03519 
03520   Ops.push_back(Chain);
03521   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
03522   return Ret;
03523 }
03524 
03525 SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
03526   SDValue Chain = N->getOperand(0);
03527   SDValue TexHandle = N->getOperand(1);
03528   SDNode *Ret = nullptr;
03529   unsigned Opc = 0;
03530   SmallVector<SDValue, 8> Ops;
03531   switch (N->getOpcode()) {
03532   default: return nullptr;
03533   case NVPTXISD::Suld1DI8Clamp:
03534     Opc = NVPTX::SULD_1D_I8_CLAMP;
03535     Ops.push_back(TexHandle);
03536     Ops.push_back(N->getOperand(2));
03537     Ops.push_back(Chain);
03538     break;
03539   case NVPTXISD::Suld1DI16Clamp:
03540     Opc = NVPTX::SULD_1D_I16_CLAMP;
03541     Ops.push_back(TexHandle);
03542     Ops.push_back(N->getOperand(2));
03543     Ops.push_back(Chain);
03544     break;
03545   case NVPTXISD::Suld1DI32Clamp:
03546     Opc = NVPTX::SULD_1D_I32_CLAMP;
03547     Ops.push_back(TexHandle);
03548     Ops.push_back(N->getOperand(2));
03549     Ops.push_back(Chain);
03550     break;
03551   case NVPTXISD::Suld1DI64Clamp:
03552     Opc = NVPTX::SULD_1D_I64_CLAMP;
03553     Ops.push_back(TexHandle);
03554     Ops.push_back(N->getOperand(2));
03555     Ops.push_back(Chain);
03556     break;
03557   case NVPTXISD::Suld1DV2I8Clamp:
03558     Opc = NVPTX::SULD_1D_V2I8_CLAMP;
03559     Ops.push_back(TexHandle);
03560     Ops.push_back(N->getOperand(2));
03561     Ops.push_back(Chain);
03562     break;
03563   case NVPTXISD::Suld1DV2I16Clamp:
03564     Opc = NVPTX::SULD_1D_V2I16_CLAMP;
03565     Ops.push_back(TexHandle);
03566     Ops.push_back(N->getOperand(2));
03567     Ops.push_back(Chain);
03568     break;
03569   case NVPTXISD::Suld1DV2I32Clamp:
03570     Opc = NVPTX::SULD_1D_V2I32_CLAMP;
03571     Ops.push_back(TexHandle);
03572     Ops.push_back(N->getOperand(2));
03573     Ops.push_back(Chain);
03574     break;
03575   case NVPTXISD::Suld1DV2I64Clamp:
03576     Opc = NVPTX::SULD_1D_V2I64_CLAMP;
03577     Ops.push_back(TexHandle);
03578     Ops.push_back(N->getOperand(2));
03579     Ops.push_back(Chain);
03580     break;
03581   case NVPTXISD::Suld1DV4I8Clamp:
03582     Opc = NVPTX::SULD_1D_V4I8_CLAMP;
03583     Ops.push_back(TexHandle);
03584     Ops.push_back(N->getOperand(2));
03585     Ops.push_back(Chain);
03586     break;
03587   case NVPTXISD::Suld1DV4I16Clamp:
03588     Opc = NVPTX::SULD_1D_V4I16_CLAMP;
03589     Ops.push_back(TexHandle);
03590     Ops.push_back(N->getOperand(2));
03591     Ops.push_back(Chain);
03592     break;
03593   case NVPTXISD::Suld1DV4I32Clamp:
03594     Opc = NVPTX::SULD_1D_V4I32_CLAMP;
03595     Ops.push_back(TexHandle);
03596     Ops.push_back(N->getOperand(2));
03597     Ops.push_back(Chain);
03598     break;
03599   case NVPTXISD::Suld1DArrayI8Clamp:
03600     Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
03601     Ops.push_back(TexHandle);
03602     Ops.push_back(N->getOperand(2));
03603     Ops.push_back(N->getOperand(3));
03604     Ops.push_back(Chain);
03605     break;
03606   case NVPTXISD::Suld1DArrayI16Clamp:
03607     Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
03608     Ops.push_back(TexHandle);
03609     Ops.push_back(N->getOperand(2));
03610     Ops.push_back(N->getOperand(3));
03611     Ops.push_back(Chain);
03612     break;
03613   case NVPTXISD::Suld1DArrayI32Clamp:
03614     Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
03615     Ops.push_back(TexHandle);
03616     Ops.push_back(N->getOperand(2));
03617     Ops.push_back(N->getOperand(3));
03618     Ops.push_back(Chain);
03619     break;
03620   case NVPTXISD::Suld1DArrayI64Clamp:
03621     Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
03622     Ops.push_back(TexHandle);
03623     Ops.push_back(N->getOperand(2));
03624     Ops.push_back(N->getOperand(3));
03625     Ops.push_back(Chain);
03626     break;
03627   case NVPTXISD::Suld1DArrayV2I8Clamp:
03628     Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
03629     Ops.push_back(TexHandle);
03630     Ops.push_back(N->getOperand(2));
03631     Ops.push_back(N->getOperand(3));
03632     Ops.push_back(Chain);
03633     break;
03634   case NVPTXISD::Suld1DArrayV2I16Clamp:
03635     Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
03636     Ops.push_back(TexHandle);
03637     Ops.push_back(N->getOperand(2));
03638     Ops.push_back(N->getOperand(3));
03639     Ops.push_back(Chain);
03640     break;
03641   case NVPTXISD::Suld1DArrayV2I32Clamp:
03642     Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
03643     Ops.push_back(TexHandle);
03644     Ops.push_back(N->getOperand(2));
03645     Ops.push_back(N->getOperand(3));
03646     Ops.push_back(Chain);
03647     break;
03648   case NVPTXISD::Suld1DArrayV2I64Clamp:
03649     Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
03650     Ops.push_back(TexHandle);
03651     Ops.push_back(N->getOperand(2));
03652     Ops.push_back(N->getOperand(3));
03653     Ops.push_back(Chain);
03654     break;
03655   case NVPTXISD::Suld1DArrayV4I8Clamp:
03656     Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
03657     Ops.push_back(TexHandle);
03658     Ops.push_back(N->getOperand(2));
03659     Ops.push_back(N->getOperand(3));
03660     Ops.push_back(Chain);
03661     break;
03662   case NVPTXISD::Suld1DArrayV4I16Clamp:
03663     Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
03664     Ops.push_back(TexHandle);
03665     Ops.push_back(N->getOperand(2));
03666     Ops.push_back(N->getOperand(3));
03667     Ops.push_back(Chain);
03668     break;
03669   case NVPTXISD::Suld1DArrayV4I32Clamp:
03670     Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
03671     Ops.push_back(TexHandle);
03672     Ops.push_back(N->getOperand(2));
03673     Ops.push_back(N->getOperand(3));
03674     Ops.push_back(Chain);
03675     break;
03676   case NVPTXISD::Suld2DI8Clamp:
03677     Opc = NVPTX::SULD_2D_I8_CLAMP;
03678     Ops.push_back(TexHandle);
03679     Ops.push_back(N->getOperand(2));
03680     Ops.push_back(N->getOperand(3));
03681     Ops.push_back(Chain);
03682     break;
03683   case NVPTXISD::Suld2DI16Clamp:
03684     Opc = NVPTX::SULD_2D_I16_CLAMP;
03685     Ops.push_back(TexHandle);
03686     Ops.push_back(N->getOperand(2));
03687     Ops.push_back(N->getOperand(3));
03688     Ops.push_back(Chain);
03689     break;
03690   case NVPTXISD::Suld2DI32Clamp:
03691     Opc = NVPTX::SULD_2D_I32_CLAMP;
03692     Ops.push_back(TexHandle);
03693     Ops.push_back(N->getOperand(2));
03694     Ops.push_back(N->getOperand(3));
03695     Ops.push_back(Chain);
03696     break;
03697   case NVPTXISD::Suld2DI64Clamp:
03698     Opc = NVPTX::SULD_2D_I64_CLAMP;
03699     Ops.push_back(TexHandle);
03700     Ops.push_back(N->getOperand(2));
03701     Ops.push_back(N->getOperand(3));
03702     Ops.push_back(Chain);
03703     break;
03704   case NVPTXISD::Suld2DV2I8Clamp:
03705     Opc = NVPTX::SULD_2D_V2I8_CLAMP;
03706     Ops.push_back(TexHandle);
03707     Ops.push_back(N->getOperand(2));
03708     Ops.push_back(N->getOperand(3));
03709     Ops.push_back(Chain);
03710     break;
03711   case NVPTXISD::Suld2DV2I16Clamp:
03712     Opc = NVPTX::SULD_2D_V2I16_CLAMP;
03713     Ops.push_back(TexHandle);
03714     Ops.push_back(N->getOperand(2));
03715     Ops.push_back(N->getOperand(3));
03716     Ops.push_back(Chain);
03717     break;
03718   case NVPTXISD::Suld2DV2I32Clamp:
03719     Opc = NVPTX::SULD_2D_V2I32_CLAMP;
03720     Ops.push_back(TexHandle);
03721     Ops.push_back(N->getOperand(2));
03722     Ops.push_back(N->getOperand(3));
03723     Ops.push_back(Chain);
03724     break;
03725   case NVPTXISD::Suld2DV2I64Clamp:
03726     Opc = NVPTX::SULD_2D_V2I64_CLAMP;
03727     Ops.push_back(TexHandle);
03728     Ops.push_back(N->getOperand(2));
03729     Ops.push_back(N->getOperand(3));
03730     Ops.push_back(Chain);
03731     break;
03732   case NVPTXISD::Suld2DV4I8Clamp:
03733     Opc = NVPTX::SULD_2D_V4I8_CLAMP;
03734     Ops.push_back(TexHandle);
03735     Ops.push_back(N->getOperand(2));
03736     Ops.push_back(N->getOperand(3));
03737     Ops.push_back(Chain);
03738     break;
03739   case NVPTXISD::Suld2DV4I16Clamp:
03740     Opc = NVPTX::SULD_2D_V4I16_CLAMP;
03741     Ops.push_back(TexHandle);
03742     Ops.push_back(N->getOperand(2));
03743     Ops.push_back(N->getOperand(3));
03744     Ops.push_back(Chain);
03745     break;
03746   case NVPTXISD::Suld2DV4I32Clamp:
03747     Opc = NVPTX::SULD_2D_V4I32_CLAMP;
03748     Ops.push_back(TexHandle);
03749     Ops.push_back(N->getOperand(2));
03750     Ops.push_back(N->getOperand(3));
03751     Ops.push_back(Chain);
03752     break;
03753   case NVPTXISD::Suld2DArrayI8Clamp:
03754     Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
03755     Ops.push_back(TexHandle);
03756     Ops.push_back(N->getOperand(2));
03757     Ops.push_back(N->getOperand(3));
03758     Ops.push_back(N->getOperand(4));
03759     Ops.push_back(Chain);
03760     break;
03761   case NVPTXISD::Suld2DArrayI16Clamp:
03762     Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
03763     Ops.push_back(TexHandle);
03764     Ops.push_back(N->getOperand(2));
03765     Ops.push_back(N->getOperand(3));
03766     Ops.push_back(N->getOperand(4));
03767     Ops.push_back(Chain);
03768     break;
03769   case NVPTXISD::Suld2DArrayI32Clamp:
03770     Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
03771     Ops.push_back(TexHandle);
03772     Ops.push_back(N->getOperand(2));
03773     Ops.push_back(N->getOperand(3));
03774     Ops.push_back(N->getOperand(4));
03775     Ops.push_back(Chain);
03776     break;
03777   case NVPTXISD::Suld2DArrayI64Clamp:
03778     Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
03779     Ops.push_back(TexHandle);
03780     Ops.push_back(N->getOperand(2));
03781     Ops.push_back(N->getOperand(3));
03782     Ops.push_back(N->getOperand(4));
03783     Ops.push_back(Chain);
03784     break;
03785   case NVPTXISD::Suld2DArrayV2I8Clamp:
03786     Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
03787     Ops.push_back(TexHandle);
03788     Ops.push_back(N->getOperand(2));
03789     Ops.push_back(N->getOperand(3));
03790     Ops.push_back(N->getOperand(4));
03791     Ops.push_back(Chain);
03792     break;
03793   case NVPTXISD::Suld2DArrayV2I16Clamp:
03794     Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
03795     Ops.push_back(TexHandle);
03796     Ops.push_back(N->getOperand(2));
03797     Ops.push_back(N->getOperand(3));
03798     Ops.push_back(N->getOperand(4));
03799     Ops.push_back(Chain);
03800     break;
03801   case NVPTXISD::Suld2DArrayV2I32Clamp:
03802     Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
03803     Ops.push_back(TexHandle);
03804     Ops.push_back(N->getOperand(2));
03805     Ops.push_back(N->getOperand(3));
03806     Ops.push_back(N->getOperand(4));
03807     Ops.push_back(Chain);
03808     break;
03809   case NVPTXISD::Suld2DArrayV2I64Clamp:
03810     Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
03811     Ops.push_back(TexHandle);
03812     Ops.push_back(N->getOperand(2));
03813     Ops.push_back(N->getOperand(3));
03814     Ops.push_back(N->getOperand(4));
03815     Ops.push_back(Chain);
03816     break;
03817   case NVPTXISD::Suld2DArrayV4I8Clamp:
03818     Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
03819     Ops.push_back(TexHandle);
03820     Ops.push_back(N->getOperand(2));
03821     Ops.push_back(N->getOperand(3));
03822     Ops.push_back(N->getOperand(4));
03823     Ops.push_back(Chain);
03824     break;
03825   case NVPTXISD::Suld2DArrayV4I16Clamp:
03826     Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
03827     Ops.push_back(TexHandle);
03828     Ops.push_back(N->getOperand(2));
03829     Ops.push_back(N->getOperand(3));
03830     Ops.push_back(N->getOperand(4));
03831     Ops.push_back(Chain);
03832     break;
03833   case NVPTXISD::Suld2DArrayV4I32Clamp:
03834     Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
03835     Ops.push_back(TexHandle);
03836     Ops.push_back(N->getOperand(2));
03837     Ops.push_back(N->getOperand(3));
03838     Ops.push_back(N->getOperand(4));
03839     Ops.push_back(Chain);
03840     break;
03841   case NVPTXISD::Suld3DI8Clamp:
03842     Opc = NVPTX::SULD_3D_I8_CLAMP;
03843     Ops.push_back(TexHandle);
03844     Ops.push_back(N->getOperand(2));
03845     Ops.push_back(N->getOperand(3));
03846     Ops.push_back(N->getOperand(4));
03847     Ops.push_back(Chain);
03848     break;
03849   case NVPTXISD::Suld3DI16Clamp:
03850     Opc = NVPTX::SULD_3D_I16_CLAMP;
03851     Ops.push_back(TexHandle);
03852     Ops.push_back(N->getOperand(2));
03853     Ops.push_back(N->getOperand(3));
03854     Ops.push_back(N->getOperand(4));
03855     Ops.push_back(Chain);
03856     break;
03857   case NVPTXISD::Suld3DI32Clamp:
03858     Opc = NVPTX::SULD_3D_I32_CLAMP;
03859     Ops.push_back(TexHandle);
03860     Ops.push_back(N->getOperand(2));
03861     Ops.push_back(N->getOperand(3));
03862     Ops.push_back(N->getOperand(4));
03863     Ops.push_back(Chain);
03864     break;
03865   case NVPTXISD::Suld3DI64Clamp:
03866     Opc = NVPTX::SULD_3D_I64_CLAMP;
03867     Ops.push_back(TexHandle);
03868     Ops.push_back(N->getOperand(2));
03869     Ops.push_back(N->getOperand(3));
03870     Ops.push_back(N->getOperand(4));
03871     Ops.push_back(Chain);
03872     break;
03873   case NVPTXISD::Suld3DV2I8Clamp:
03874     Opc = NVPTX::SULD_3D_V2I8_CLAMP;
03875     Ops.push_back(TexHandle);
03876     Ops.push_back(N->getOperand(2));
03877     Ops.push_back(N->getOperand(3));
03878     Ops.push_back(N->getOperand(4));
03879     Ops.push_back(Chain);
03880     break;
03881   case NVPTXISD::Suld3DV2I16Clamp:
03882     Opc = NVPTX::SULD_3D_V2I16_CLAMP;
03883     Ops.push_back(TexHandle);
03884     Ops.push_back(N->getOperand(2));
03885     Ops.push_back(N->getOperand(3));
03886     Ops.push_back(N->getOperand(4));
03887     Ops.push_back(Chain);
03888     break;
03889   case NVPTXISD::Suld3DV2I32Clamp:
03890     Opc = NVPTX::SULD_3D_V2I32_CLAMP;
03891     Ops.push_back(TexHandle);
03892     Ops.push_back(N->getOperand(2));
03893     Ops.push_back(N->getOperand(3));
03894     Ops.push_back(N->getOperand(4));
03895     Ops.push_back(Chain);
03896     break;
03897   case NVPTXISD::Suld3DV2I64Clamp:
03898     Opc = NVPTX::SULD_3D_V2I64_CLAMP;
03899     Ops.push_back(TexHandle);
03900     Ops.push_back(N->getOperand(2));
03901     Ops.push_back(N->getOperand(3));
03902     Ops.push_back(N->getOperand(4));
03903     Ops.push_back(Chain);
03904     break;
03905   case NVPTXISD::Suld3DV4I8Clamp:
03906     Opc = NVPTX::SULD_3D_V4I8_CLAMP;
03907     Ops.push_back(TexHandle);
03908     Ops.push_back(N->getOperand(2));
03909     Ops.push_back(N->getOperand(3));
03910     Ops.push_back(N->getOperand(4));
03911     Ops.push_back(Chain);
03912     break;
03913   case NVPTXISD::Suld3DV4I16Clamp:
03914     Opc = NVPTX::SULD_3D_V4I16_CLAMP;
03915     Ops.push_back(TexHandle);
03916     Ops.push_back(N->getOperand(2));
03917     Ops.push_back(N->getOperand(3));
03918     Ops.push_back(N->getOperand(4));
03919     Ops.push_back(Chain);
03920     break;
03921   case NVPTXISD::Suld3DV4I32Clamp:
03922     Opc = NVPTX::SULD_3D_V4I32_CLAMP;
03923     Ops.push_back(TexHandle);
03924     Ops.push_back(N->getOperand(2));
03925     Ops.push_back(N->getOperand(3));
03926     Ops.push_back(N->getOperand(4));
03927     Ops.push_back(Chain);
03928     break;
03929   case NVPTXISD::Suld1DI8Trap:
03930     Opc = NVPTX::SULD_1D_I8_TRAP;
03931     Ops.push_back(TexHandle);
03932     Ops.push_back(N->getOperand(2));
03933     Ops.push_back(Chain);
03934     break;
03935   case NVPTXISD::Suld1DI16Trap:
03936     Opc = NVPTX::SULD_1D_I16_TRAP;
03937     Ops.push_back(TexHandle);
03938     Ops.push_back(N->getOperand(2));
03939     Ops.push_back(Chain);
03940     break;
03941   case NVPTXISD::Suld1DI32Trap:
03942     Opc = NVPTX::SULD_1D_I32_TRAP;
03943     Ops.push_back(TexHandle);
03944     Ops.push_back(N->getOperand(2));
03945     Ops.push_back(Chain);
03946     break;
03947   case NVPTXISD::Suld1DI64Trap:
03948     Opc = NVPTX::SULD_1D_I64_TRAP;
03949     Ops.push_back(TexHandle);
03950     Ops.push_back(N->getOperand(2));
03951     Ops.push_back(Chain);
03952     break;
03953   case NVPTXISD::Suld1DV2I8Trap:
03954     Opc = NVPTX::SULD_1D_V2I8_TRAP;
03955     Ops.push_back(TexHandle);
03956     Ops.push_back(N->getOperand(2));
03957     Ops.push_back(Chain);
03958     break;
03959   case NVPTXISD::Suld1DV2I16Trap:
03960     Opc = NVPTX::SULD_1D_V2I16_TRAP;
03961     Ops.push_back(TexHandle);
03962     Ops.push_back(N->getOperand(2));
03963     Ops.push_back(Chain);
03964     break;
03965   case NVPTXISD::Suld1DV2I32Trap:
03966     Opc = NVPTX::SULD_1D_V2I32_TRAP;
03967     Ops.push_back(TexHandle);
03968     Ops.push_back(N->getOperand(2));
03969     Ops.push_back(Chain);
03970     break;
03971   case NVPTXISD::Suld1DV2I64Trap:
03972     Opc = NVPTX::SULD_1D_V2I64_TRAP;
03973     Ops.push_back(TexHandle);
03974     Ops.push_back(N->getOperand(2));
03975     Ops.push_back(Chain);
03976     break;
03977   case NVPTXISD::Suld1DV4I8Trap:
03978     Opc = NVPTX::SULD_1D_V4I8_TRAP;
03979     Ops.push_back(TexHandle);
03980     Ops.push_back(N->getOperand(2));
03981     Ops.push_back(Chain);
03982     break;
03983   case NVPTXISD::Suld1DV4I16Trap:
03984     Opc = NVPTX::SULD_1D_V4I16_TRAP;
03985     Ops.push_back(TexHandle);
03986     Ops.push_back(N->getOperand(2));
03987     Ops.push_back(Chain);
03988     break;
03989   case NVPTXISD::Suld1DV4I32Trap:
03990     Opc = NVPTX::SULD_1D_V4I32_TRAP;
03991     Ops.push_back(TexHandle);
03992     Ops.push_back(N->getOperand(2));
03993     Ops.push_back(Chain);
03994     break;
03995   case NVPTXISD::Suld1DArrayI8Trap:
03996     Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
03997     Ops.push_back(TexHandle);
03998     Ops.push_back(N->getOperand(2));
03999     Ops.push_back(N->getOperand(3));
04000     Ops.push_back(Chain);
04001     break;
04002   case NVPTXISD::Suld1DArrayI16Trap:
04003     Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
04004     Ops.push_back(TexHandle);
04005     Ops.push_back(N->getOperand(2));
04006     Ops.push_back(N->getOperand(3));
04007     Ops.push_back(Chain);
04008     break;
04009   case NVPTXISD::Suld1DArrayI32Trap:
04010     Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
04011     Ops.push_back(TexHandle);
04012     Ops.push_back(N->getOperand(2));
04013     Ops.push_back(N->getOperand(3));
04014     Ops.push_back(Chain);
04015     break;
04016   case NVPTXISD::Suld1DArrayI64Trap:
04017     Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
04018     Ops.push_back(TexHandle);
04019     Ops.push_back(N->getOperand(2));
04020     Ops.push_back(N->getOperand(3));
04021     Ops.push_back(Chain);
04022     break;
04023   case NVPTXISD::Suld1DArrayV2I8Trap:
04024     Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
04025     Ops.push_back(TexHandle);
04026     Ops.push_back(N->getOperand(2));
04027     Ops.push_back(N->getOperand(3));
04028     Ops.push_back(Chain);
04029     break;
04030   case NVPTXISD::Suld1DArrayV2I16Trap:
04031     Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
04032     Ops.push_back(TexHandle);
04033     Ops.push_back(N->getOperand(2));
04034     Ops.push_back(N->getOperand(3));
04035     Ops.push_back(Chain);
04036     break;
04037   case NVPTXISD::Suld1DArrayV2I32Trap:
04038     Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
04039     Ops.push_back(TexHandle);
04040     Ops.push_back(N->getOperand(2));
04041     Ops.push_back(N->getOperand(3));
04042     Ops.push_back(Chain);
04043     break;
04044   case NVPTXISD::Suld1DArrayV2I64Trap:
04045     Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
04046     Ops.push_back(TexHandle);
04047     Ops.push_back(N->getOperand(2));
04048     Ops.push_back(N->getOperand(3));
04049     Ops.push_back(Chain);
04050     break;
04051   case NVPTXISD::Suld1DArrayV4I8Trap:
04052     Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
04053     Ops.push_back(TexHandle);
04054     Ops.push_back(N->getOperand(2));
04055     Ops.push_back(N->getOperand(3));
04056     Ops.push_back(Chain);
04057     break;
04058   case NVPTXISD::Suld1DArrayV4I16Trap:
04059     Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
04060     Ops.push_back(TexHandle);
04061     Ops.push_back(N->getOperand(2));
04062     Ops.push_back(N->getOperand(3));
04063     Ops.push_back(Chain);
04064     break;
04065   case NVPTXISD::Suld1DArrayV4I32Trap:
04066     Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
04067     Ops.push_back(TexHandle);
04068     Ops.push_back(N->getOperand(2));
04069     Ops.push_back(N->getOperand(3));
04070     Ops.push_back(Chain);
04071     break;
04072   case NVPTXISD::Suld2DI8Trap:
04073     Opc = NVPTX::SULD_2D_I8_TRAP;
04074     Ops.push_back(TexHandle);
04075     Ops.push_back(N->getOperand(2));
04076     Ops.push_back(N->getOperand(3));
04077     Ops.push_back(Chain);
04078     break;
04079   case NVPTXISD::Suld2DI16Trap:
04080     Opc = NVPTX::SULD_2D_I16_TRAP;
04081     Ops.push_back(TexHandle);
04082     Ops.push_back(N->getOperand(2));
04083     Ops.push_back(N->getOperand(3));
04084     Ops.push_back(Chain);
04085     break;
04086   case NVPTXISD::Suld2DI32Trap:
04087     Opc = NVPTX::SULD_2D_I32_TRAP;
04088     Ops.push_back(TexHandle);
04089     Ops.push_back(N->getOperand(2));
04090     Ops.push_back(N->getOperand(3));
04091     Ops.push_back(Chain);
04092     break;
04093   case NVPTXISD::Suld2DI64Trap:
04094     Opc = NVPTX::SULD_2D_I64_TRAP;
04095     Ops.push_back(TexHandle);
04096     Ops.push_back(N->getOperand(2));
04097     Ops.push_back(N->getOperand(3));
04098     Ops.push_back(Chain);
04099     break;
04100   case NVPTXISD::Suld2DV2I8Trap:
04101     Opc = NVPTX::SULD_2D_V2I8_TRAP;
04102     Ops.push_back(TexHandle);
04103     Ops.push_back(N->getOperand(2));
04104     Ops.push_back(N->getOperand(3));
04105     Ops.push_back(Chain);
04106     break;
04107   case NVPTXISD::Suld2DV2I16Trap:
04108     Opc = NVPTX::SULD_2D_V2I16_TRAP;
04109     Ops.push_back(TexHandle);
04110     Ops.push_back(N->getOperand(2));
04111     Ops.push_back(N->getOperand(3));
04112     Ops.push_back(Chain);
04113     break;
04114   case NVPTXISD::Suld2DV2I32Trap:
04115     Opc = NVPTX::SULD_2D_V2I32_TRAP;
04116     Ops.push_back(TexHandle);
04117     Ops.push_back(N->getOperand(2));
04118     Ops.push_back(N->getOperand(3));
04119     Ops.push_back(Chain);
04120     break;
04121   case NVPTXISD::Suld2DV2I64Trap:
04122     Opc = NVPTX::SULD_2D_V2I64_TRAP;
04123     Ops.push_back(TexHandle);
04124     Ops.push_back(N->getOperand(2));
04125     Ops.push_back(N->getOperand(3));
04126     Ops.push_back(Chain);
04127     break;
04128   case NVPTXISD::Suld2DV4I8Trap:
04129     Opc = NVPTX::SULD_2D_V4I8_TRAP;
04130     Ops.push_back(TexHandle);
04131     Ops.push_back(N->getOperand(2));
04132     Ops.push_back(N->getOperand(3));
04133     Ops.push_back(Chain);
04134     break;
04135   case NVPTXISD::Suld2DV4I16Trap:
04136     Opc = NVPTX::SULD_2D_V4I16_TRAP;
04137     Ops.push_back(TexHandle);
04138     Ops.push_back(N->getOperand(2));
04139     Ops.push_back(N->getOperand(3));
04140     Ops.push_back(Chain);
04141     break;
04142   case NVPTXISD::Suld2DV4I32Trap:
04143     Opc = NVPTX::SULD_2D_V4I32_TRAP;
04144     Ops.push_back(TexHandle);
04145     Ops.push_back(N->getOperand(2));
04146     Ops.push_back(N->getOperand(3));
04147     Ops.push_back(Chain);
04148     break;
04149   case NVPTXISD::Suld2DArrayI8Trap:
04150     Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
04151     Ops.push_back(TexHandle);
04152     Ops.push_back(N->getOperand(2));
04153     Ops.push_back(N->getOperand(3));
04154     Ops.push_back(N->getOperand(4));
04155     Ops.push_back(Chain);
04156     break;
04157   case NVPTXISD::Suld2DArrayI16Trap:
04158     Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
04159     Ops.push_back(TexHandle);
04160     Ops.push_back(N->getOperand(2));
04161     Ops.push_back(N->getOperand(3));
04162     Ops.push_back(N->getOperand(4));
04163     Ops.push_back(Chain);
04164     break;
04165   case NVPTXISD::Suld2DArrayI32Trap:
04166     Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
04167     Ops.push_back(TexHandle);
04168     Ops.push_back(N->getOperand(2));
04169     Ops.push_back(N->getOperand(3));
04170     Ops.push_back(N->getOperand(4));
04171     Ops.push_back(Chain);
04172     break;
04173   case NVPTXISD::Suld2DArrayI64Trap:
04174     Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
04175     Ops.push_back(TexHandle);
04176     Ops.push_back(N->getOperand(2));
04177     Ops.push_back(N->getOperand(3));
04178     Ops.push_back(N->getOperand(4));
04179     Ops.push_back(Chain);
04180     break;
04181   case NVPTXISD::Suld2DArrayV2I8Trap:
04182     Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
04183     Ops.push_back(TexHandle);
04184     Ops.push_back(N->getOperand(2));
04185     Ops.push_back(N->getOperand(3));
04186     Ops.push_back(N->getOperand(4));
04187     Ops.push_back(Chain);
04188     break;
04189   case NVPTXISD::Suld2DArrayV2I16Trap:
04190     Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
04191     Ops.push_back(TexHandle);
04192     Ops.push_back(N->getOperand(2));
04193     Ops.push_back(N->getOperand(3));
04194     Ops.push_back(N->getOperand(4));
04195     Ops.push_back(Chain);
04196     break;
04197   case NVPTXISD::Suld2DArrayV2I32Trap:
04198     Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
04199     Ops.push_back(TexHandle);
04200     Ops.push_back(N->getOperand(2));
04201     Ops.push_back(N->getOperand(3));
04202     Ops.push_back(N->getOperand(4));
04203     Ops.push_back(Chain);
04204     break;
04205   case NVPTXISD::Suld2DArrayV2I64Trap:
04206     Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
04207     Ops.push_back(TexHandle);
04208     Ops.push_back(N->getOperand(2));
04209     Ops.push_back(N->getOperand(3));
04210     Ops.push_back(N->getOperand(4));
04211     Ops.push_back(Chain);
04212     break;
04213   case NVPTXISD::Suld2DArrayV4I8Trap:
04214     Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
04215     Ops.push_back(TexHandle);
04216     Ops.push_back(N->getOperand(2));
04217     Ops.push_back(N->getOperand(3));
04218     Ops.push_back(N->getOperand(4));
04219     Ops.push_back(Chain);
04220     break;
04221   case NVPTXISD::Suld2DArrayV4I16Trap:
04222     Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
04223     Ops.push_back(TexHandle);
04224     Ops.push_back(N->getOperand(2));
04225     Ops.push_back(N->getOperand(3));
04226     Ops.push_back(N->getOperand(4));
04227     Ops.push_back(Chain);
04228     break;
04229   case NVPTXISD::Suld2DArrayV4I32Trap:
04230     Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
04231     Ops.push_back(TexHandle);
04232     Ops.push_back(N->getOperand(2));
04233     Ops.push_back(N->getOperand(3));
04234     Ops.push_back(N->getOperand(4));
04235     Ops.push_back(Chain);
04236     break;
04237   case NVPTXISD::Suld3DI8Trap:
04238     Opc = NVPTX::SULD_3D_I8_TRAP;
04239     Ops.push_back(TexHandle);
04240     Ops.push_back(N->getOperand(2));
04241     Ops.push_back(N->getOperand(3));
04242     Ops.push_back(N->getOperand(4));
04243     Ops.push_back(Chain);
04244     break;
04245   case NVPTXISD::Suld3DI16Trap:
04246     Opc = NVPTX::SULD_3D_I16_TRAP;
04247     Ops.push_back(TexHandle);
04248     Ops.push_back(N->getOperand(2));
04249     Ops.push_back(N->getOperand(3));
04250     Ops.push_back(N->getOperand(4));
04251     Ops.push_back(Chain);
04252     break;
04253   case NVPTXISD::Suld3DI32Trap:
04254     Opc = NVPTX::SULD_3D_I32_TRAP;
04255     Ops.push_back(TexHandle);
04256     Ops.push_back(N->getOperand(2));
04257     Ops.push_back(N->getOperand(3));
04258     Ops.push_back(N->getOperand(4));
04259     Ops.push_back(Chain);
04260     break;
04261   case NVPTXISD::Suld3DI64Trap:
04262     Opc = NVPTX::SULD_3D_I64_TRAP;
04263     Ops.push_back(TexHandle);
04264     Ops.push_back(N->getOperand(2));
04265     Ops.push_back(N->getOperand(3));
04266     Ops.push_back(N->getOperand(4));
04267     Ops.push_back(Chain);
04268     break;
04269   case NVPTXISD::Suld3DV2I8Trap:
04270     Opc = NVPTX::SULD_3D_V2I8_TRAP;
04271     Ops.push_back(TexHandle);
04272     Ops.push_back(N->getOperand(2));
04273     Ops.push_back(N->getOperand(3));
04274     Ops.push_back(N->getOperand(4));
04275     Ops.push_back(Chain);
04276     break;
04277   case NVPTXISD::Suld3DV2I16Trap:
04278     Opc = NVPTX::SULD_3D_V2I16_TRAP;
04279     Ops.push_back(TexHandle);
04280     Ops.push_back(N->getOperand(2));
04281     Ops.push_back(N->getOperand(3));
04282     Ops.push_back(N->getOperand(4));
04283     Ops.push_back(Chain);
04284     break;
04285   case NVPTXISD::Suld3DV2I32Trap:
04286     Opc = NVPTX::SULD_3D_V2I32_TRAP;
04287     Ops.push_back(TexHandle);
04288     Ops.push_back(N->getOperand(2));
04289     Ops.push_back(N->getOperand(3));
04290     Ops.push_back(N->getOperand(4));
04291     Ops.push_back(Chain);
04292     break;
04293   case NVPTXISD::Suld3DV2I64Trap:
04294     Opc = NVPTX::SULD_3D_V2I64_TRAP;
04295     Ops.push_back(TexHandle);
04296     Ops.push_back(N->getOperand(2));
04297     Ops.push_back(N->getOperand(3));
04298     Ops.push_back(N->getOperand(4));
04299     Ops.push_back(Chain);
04300     break;
04301   case NVPTXISD::Suld3DV4I8Trap:
04302     Opc = NVPTX::SULD_3D_V4I8_TRAP;
04303     Ops.push_back(TexHandle);
04304     Ops.push_back(N->getOperand(2));
04305     Ops.push_back(N->getOperand(3));
04306     Ops.push_back(N->getOperand(4));
04307     Ops.push_back(Chain);
04308     break;
04309   case NVPTXISD::Suld3DV4I16Trap:
04310     Opc = NVPTX::SULD_3D_V4I16_TRAP;
04311     Ops.push_back(TexHandle);
04312     Ops.push_back(N->getOperand(2));
04313     Ops.push_back(N->getOperand(3));
04314     Ops.push_back(N->getOperand(4));
04315     Ops.push_back(Chain);
04316     break;
04317   case NVPTXISD::Suld3DV4I32Trap:
04318     Opc = NVPTX::SULD_3D_V4I32_TRAP;
04319     Ops.push_back(TexHandle);
04320     Ops.push_back(N->getOperand(2));
04321     Ops.push_back(N->getOperand(3));
04322     Ops.push_back(N->getOperand(4));
04323     Ops.push_back(Chain);
04324     break;
04325   case NVPTXISD::Suld1DI8Zero:
04326     Opc = NVPTX::SULD_1D_I8_ZERO;
04327     Ops.push_back(TexHandle);
04328     Ops.push_back(N->getOperand(2));
04329     Ops.push_back(Chain);
04330     break;
04331   case NVPTXISD::Suld1DI16Zero:
04332     Opc = NVPTX::SULD_1D_I16_ZERO;
04333     Ops.push_back(TexHandle);
04334     Ops.push_back(N->getOperand(2));
04335     Ops.push_back(Chain);
04336     break;
04337   case NVPTXISD::Suld1DI32Zero:
04338     Opc = NVPTX::SULD_1D_I32_ZERO;
04339     Ops.push_back(TexHandle);
04340     Ops.push_back(N->getOperand(2));
04341     Ops.push_back(Chain);
04342     break;
04343   case NVPTXISD::Suld1DI64Zero:
04344     Opc = NVPTX::SULD_1D_I64_ZERO;
04345     Ops.push_back(TexHandle);
04346     Ops.push_back(N->getOperand(2));
04347     Ops.push_back(Chain);
04348     break;
04349   case NVPTXISD::Suld1DV2I8Zero:
04350     Opc = NVPTX::SULD_1D_V2I8_ZERO;
04351     Ops.push_back(TexHandle);
04352     Ops.push_back(N->getOperand(2));
04353     Ops.push_back(Chain);
04354     break;
04355   case NVPTXISD::Suld1DV2I16Zero:
04356     Opc = NVPTX::SULD_1D_V2I16_ZERO;
04357     Ops.push_back(TexHandle);
04358     Ops.push_back(N->getOperand(2));
04359     Ops.push_back(Chain);
04360     break;
04361   case NVPTXISD::Suld1DV2I32Zero:
04362     Opc = NVPTX::SULD_1D_V2I32_ZERO;
04363     Ops.push_back(TexHandle);
04364     Ops.push_back(N->getOperand(2));
04365     Ops.push_back(Chain);
04366     break;
04367   case NVPTXISD::Suld1DV2I64Zero:
04368     Opc = NVPTX::SULD_1D_V2I64_ZERO;
04369     Ops.push_back(TexHandle);
04370     Ops.push_back(N->getOperand(2));
04371     Ops.push_back(Chain);
04372     break;
04373   case NVPTXISD::Suld1DV4I8Zero:
04374     Opc = NVPTX::SULD_1D_V4I8_ZERO;
04375     Ops.push_back(TexHandle);
04376     Ops.push_back(N->getOperand(2));
04377     Ops.push_back(Chain);
04378     break;
04379   case NVPTXISD::Suld1DV4I16Zero:
04380     Opc = NVPTX::SULD_1D_V4I16_ZERO;
04381     Ops.push_back(TexHandle);
04382     Ops.push_back(N->getOperand(2));
04383     Ops.push_back(Chain);
04384     break;
04385   case NVPTXISD::Suld1DV4I32Zero:
04386     Opc = NVPTX::SULD_1D_V4I32_ZERO;
04387     Ops.push_back(TexHandle);
04388     Ops.push_back(N->getOperand(2));
04389     Ops.push_back(Chain);
04390     break;
04391   case NVPTXISD::Suld1DArrayI8Zero:
04392     Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
04393     Ops.push_back(TexHandle);
04394     Ops.push_back(N->getOperand(2));
04395     Ops.push_back(N->getOperand(3));
04396     Ops.push_back(Chain);
04397     break;
04398   case NVPTXISD::Suld1DArrayI16Zero:
04399     Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
04400     Ops.push_back(TexHandle);
04401     Ops.push_back(N->getOperand(2));
04402     Ops.push_back(N->getOperand(3));
04403     Ops.push_back(Chain);
04404     break;
04405   case NVPTXISD::Suld1DArrayI32Zero:
04406     Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
04407     Ops.push_back(TexHandle);
04408     Ops.push_back(N->getOperand(2));
04409     Ops.push_back(N->getOperand(3));
04410     Ops.push_back(Chain);
04411     break;
04412   case NVPTXISD::Suld1DArrayI64Zero:
04413     Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
04414     Ops.push_back(TexHandle);
04415     Ops.push_back(N->getOperand(2));
04416     Ops.push_back(N->getOperand(3));
04417     Ops.push_back(Chain);
04418     break;
04419   case NVPTXISD::Suld1DArrayV2I8Zero:
04420     Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
04421     Ops.push_back(TexHandle);
04422     Ops.push_back(N->getOperand(2));
04423     Ops.push_back(N->getOperand(3));
04424     Ops.push_back(Chain);
04425     break;
04426   case NVPTXISD::Suld1DArrayV2I16Zero:
04427     Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
04428     Ops.push_back(TexHandle);
04429     Ops.push_back(N->getOperand(2));
04430     Ops.push_back(N->getOperand(3));
04431     Ops.push_back(Chain);
04432     break;
04433   case NVPTXISD::Suld1DArrayV2I32Zero:
04434     Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
04435     Ops.push_back(TexHandle);
04436     Ops.push_back(N->getOperand(2));
04437     Ops.push_back(N->getOperand(3));
04438     Ops.push_back(Chain);
04439     break;
04440   case NVPTXISD::Suld1DArrayV2I64Zero:
04441     Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
04442     Ops.push_back(TexHandle);
04443     Ops.push_back(N->getOperand(2));
04444     Ops.push_back(N->getOperand(3));
04445     Ops.push_back(Chain);
04446     break;
04447   case NVPTXISD::Suld1DArrayV4I8Zero:
04448     Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
04449     Ops.push_back(TexHandle);
04450     Ops.push_back(N->getOperand(2));
04451     Ops.push_back(N->getOperand(3));
04452     Ops.push_back(Chain);
04453     break;
04454   case NVPTXISD::Suld1DArrayV4I16Zero:
04455     Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
04456     Ops.push_back(TexHandle);
04457     Ops.push_back(N->getOperand(2));
04458     Ops.push_back(N->getOperand(3));
04459     Ops.push_back(Chain);
04460     break;
04461   case NVPTXISD::Suld1DArrayV4I32Zero:
04462     Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
04463     Ops.push_back(TexHandle);
04464     Ops.push_back(N->getOperand(2));
04465     Ops.push_back(N->getOperand(3));
04466     Ops.push_back(Chain);
04467     break;
04468   case NVPTXISD::Suld2DI8Zero:
04469     Opc = NVPTX::SULD_2D_I8_ZERO;
04470     Ops.push_back(TexHandle);
04471     Ops.push_back(N->getOperand(2));
04472     Ops.push_back(N->getOperand(3));
04473     Ops.push_back(Chain);
04474     break;
04475   case NVPTXISD::Suld2DI16Zero:
04476     Opc = NVPTX::SULD_2D_I16_ZERO;
04477     Ops.push_back(TexHandle);
04478     Ops.push_back(N->getOperand(2));
04479     Ops.push_back(N->getOperand(3));
04480     Ops.push_back(Chain);
04481     break;
04482   case NVPTXISD::Suld2DI32Zero:
04483     Opc = NVPTX::SULD_2D_I32_ZERO;
04484     Ops.push_back(TexHandle);
04485     Ops.push_back(N->getOperand(2));
04486     Ops.push_back(N->getOperand(3));
04487     Ops.push_back(Chain);
04488     break;
04489   case NVPTXISD::Suld2DI64Zero:
04490     Opc = NVPTX::SULD_2D_I64_ZERO;
04491     Ops.push_back(TexHandle);
04492     Ops.push_back(N->getOperand(2));
04493     Ops.push_back(N->getOperand(3));
04494     Ops.push_back(Chain);
04495     break;
04496   case NVPTXISD::Suld2DV2I8Zero:
04497     Opc = NVPTX::SULD_2D_V2I8_ZERO;
04498     Ops.push_back(TexHandle);
04499     Ops.push_back(N->getOperand(2));
04500     Ops.push_back(N->getOperand(3));
04501     Ops.push_back(Chain);
04502     break;
04503   case NVPTXISD::Suld2DV2I16Zero:
04504     Opc = NVPTX::SULD_2D_V2I16_ZERO;
04505     Ops.push_back(TexHandle);
04506     Ops.push_back(N->getOperand(2));
04507     Ops.push_back(N->getOperand(3));
04508     Ops.push_back(Chain);
04509     break;
04510   case NVPTXISD::Suld2DV2I32Zero:
04511     Opc = NVPTX::SULD_2D_V2I32_ZERO;
04512     Ops.push_back(TexHandle);
04513     Ops.push_back(N->getOperand(2));
04514     Ops.push_back(N->getOperand(3));
04515     Ops.push_back(Chain);
04516     break;
04517   case NVPTXISD::Suld2DV2I64Zero:
04518     Opc = NVPTX::SULD_2D_V2I64_ZERO;
04519     Ops.push_back(TexHandle);
04520     Ops.push_back(N->getOperand(2));
04521     Ops.push_back(N->getOperand(3));
04522     Ops.push_back(Chain);
04523     break;
04524   case NVPTXISD::Suld2DV4I8Zero:
04525     Opc = NVPTX::SULD_2D_V4I8_ZERO;
04526     Ops.push_back(TexHandle);
04527     Ops.push_back(N->getOperand(2));
04528     Ops.push_back(N->getOperand(3));
04529     Ops.push_back(Chain);
04530     break;
04531   case NVPTXISD::Suld2DV4I16Zero:
04532     Opc = NVPTX::SULD_2D_V4I16_ZERO;
04533     Ops.push_back(TexHandle);
04534     Ops.push_back(N->getOperand(2));
04535     Ops.push_back(N->getOperand(3));
04536     Ops.push_back(Chain);
04537     break;
04538   case NVPTXISD::Suld2DV4I32Zero:
04539     Opc = NVPTX::SULD_2D_V4I32_ZERO;
04540     Ops.push_back(TexHandle);
04541     Ops.push_back(N->getOperand(2));
04542     Ops.push_back(N->getOperand(3));
04543     Ops.push_back(Chain);
04544     break;
04545   case NVPTXISD::Suld2DArrayI8Zero:
04546     Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
04547     Ops.push_back(TexHandle);
04548     Ops.push_back(N->getOperand(2));
04549     Ops.push_back(N->getOperand(3));
04550     Ops.push_back(N->getOperand(4));
04551     Ops.push_back(Chain);
04552     break;
04553   case NVPTXISD::Suld2DArrayI16Zero:
04554     Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
04555     Ops.push_back(TexHandle);
04556     Ops.push_back(N->getOperand(2));
04557     Ops.push_back(N->getOperand(3));
04558     Ops.push_back(N->getOperand(4));
04559     Ops.push_back(Chain);
04560     break;
04561   case NVPTXISD::Suld2DArrayI32Zero:
04562     Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
04563     Ops.push_back(TexHandle);
04564     Ops.push_back(N->getOperand(2));
04565     Ops.push_back(N->getOperand(3));
04566     Ops.push_back(N->getOperand(4));
04567     Ops.push_back(Chain);
04568     break;
04569   case NVPTXISD::Suld2DArrayI64Zero:
04570     Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
04571     Ops.push_back(TexHandle);
04572     Ops.push_back(N->getOperand(2));
04573     Ops.push_back(N->getOperand(3));
04574     Ops.push_back(N->getOperand(4));
04575     Ops.push_back(Chain);
04576     break;
04577   case NVPTXISD::Suld2DArrayV2I8Zero:
04578     Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
04579     Ops.push_back(TexHandle);
04580     Ops.push_back(N->getOperand(2));
04581     Ops.push_back(N->getOperand(3));
04582     Ops.push_back(N->getOperand(4));
04583     Ops.push_back(Chain);
04584     break;
04585   case NVPTXISD::Suld2DArrayV2I16Zero:
04586     Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
04587     Ops.push_back(TexHandle);
04588     Ops.push_back(N->getOperand(2));
04589     Ops.push_back(N->getOperand(3));
04590     Ops.push_back(N->getOperand(4));
04591     Ops.push_back(Chain);
04592     break;
04593   case NVPTXISD::Suld2DArrayV2I32Zero:
04594     Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
04595     Ops.push_back(TexHandle);
04596     Ops.push_back(N->getOperand(2));
04597     Ops.push_back(N->getOperand(3));
04598     Ops.push_back(N->getOperand(4));
04599     Ops.push_back(Chain);
04600     break;
04601   case NVPTXISD::Suld2DArrayV2I64Zero:
04602     Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
04603     Ops.push_back(TexHandle);
04604     Ops.push_back(N->getOperand(2));
04605     Ops.push_back(N->getOperand(3));
04606     Ops.push_back(N->getOperand(4));
04607     Ops.push_back(Chain);
04608     break;
04609   case NVPTXISD::Suld2DArrayV4I8Zero:
04610     Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
04611     Ops.push_back(TexHandle);
04612     Ops.push_back(N->getOperand(2));
04613     Ops.push_back(N->getOperand(3));
04614     Ops.push_back(N->getOperand(4));
04615     Ops.push_back(Chain);
04616     break;
04617   case NVPTXISD::Suld2DArrayV4I16Zero:
04618     Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
04619     Ops.push_back(TexHandle);
04620     Ops.push_back(N->getOperand(2));
04621     Ops.push_back(N->getOperand(3));
04622     Ops.push_back(N->getOperand(4));
04623     Ops.push_back(Chain);
04624     break;
04625   case NVPTXISD::Suld2DArrayV4I32Zero:
04626     Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
04627     Ops.push_back(TexHandle);
04628     Ops.push_back(N->getOperand(2));
04629     Ops.push_back(N->getOperand(3));
04630     Ops.push_back(N->getOperand(4));
04631     Ops.push_back(Chain);
04632     break;
04633   case NVPTXISD::Suld3DI8Zero:
04634     Opc = NVPTX::SULD_3D_I8_ZERO;
04635     Ops.push_back(TexHandle);
04636     Ops.push_back(N->getOperand(2));
04637     Ops.push_back(N->getOperand(3));
04638     Ops.push_back(N->getOperand(4));
04639     Ops.push_back(Chain);
04640     break;
04641   case NVPTXISD::Suld3DI16Zero:
04642     Opc = NVPTX::SULD_3D_I16_ZERO;
04643     Ops.push_back(TexHandle);
04644     Ops.push_back(N->getOperand(2));
04645     Ops.push_back(N->getOperand(3));
04646     Ops.push_back(N->getOperand(4));
04647     Ops.push_back(Chain);
04648     break;
04649   case NVPTXISD::Suld3DI32Zero:
04650     Opc = NVPTX::SULD_3D_I32_ZERO;
04651     Ops.push_back(TexHandle);
04652     Ops.push_back(N->getOperand(2));
04653     Ops.push_back(N->getOperand(3));
04654     Ops.push_back(N->getOperand(4));
04655     Ops.push_back(Chain);
04656     break;
04657   case NVPTXISD::Suld3DI64Zero:
04658     Opc = NVPTX::SULD_3D_I64_ZERO;
04659     Ops.push_back(TexHandle);
04660     Ops.push_back(N->getOperand(2));
04661     Ops.push_back(N->getOperand(3));
04662     Ops.push_back(N->getOperand(4));
04663     Ops.push_back(Chain);
04664     break;
04665   case NVPTXISD::Suld3DV2I8Zero:
04666     Opc = NVPTX::SULD_3D_V2I8_ZERO;
04667     Ops.push_back(TexHandle);
04668     Ops.push_back(N->getOperand(2));
04669     Ops.push_back(N->getOperand(3));
04670     Ops.push_back(N->getOperand(4));
04671     Ops.push_back(Chain);
04672     break;
04673   case NVPTXISD::Suld3DV2I16Zero:
04674     Opc = NVPTX::SULD_3D_V2I16_ZERO;
04675     Ops.push_back(TexHandle);
04676     Ops.push_back(N->getOperand(2));
04677     Ops.push_back(N->getOperand(3));
04678     Ops.push_back(N->getOperand(4));
04679     Ops.push_back(Chain);
04680     break;
04681   case NVPTXISD::Suld3DV2I32Zero:
04682     Opc = NVPTX::SULD_3D_V2I32_ZERO;
04683     Ops.push_back(TexHandle);
04684     Ops.push_back(N->getOperand(2));
04685     Ops.push_back(N->getOperand(3));
04686     Ops.push_back(N->getOperand(4));
04687     Ops.push_back(Chain);
04688     break;
04689   case NVPTXISD::Suld3DV2I64Zero:
04690     Opc = NVPTX::SULD_3D_V2I64_ZERO;
04691     Ops.push_back(TexHandle);
04692     Ops.push_back(N->getOperand(2));
04693     Ops.push_back(N->getOperand(3));
04694     Ops.push_back(N->getOperand(4));
04695     Ops.push_back(Chain);
04696     break;
04697   case NVPTXISD::Suld3DV4I8Zero:
04698     Opc = NVPTX::SULD_3D_V4I8_ZERO;
04699     Ops.push_back(TexHandle);
04700     Ops.push_back(N->getOperand(2));
04701     Ops.push_back(N->getOperand(3));
04702     Ops.push_back(N->getOperand(4));
04703     Ops.push_back(Chain);
04704     break;
04705   case NVPTXISD::Suld3DV4I16Zero:
04706     Opc = NVPTX::SULD_3D_V4I16_ZERO;
04707     Ops.push_back(TexHandle);
04708     Ops.push_back(N->getOperand(2));
04709     Ops.push_back(N->getOperand(3));
04710     Ops.push_back(N->getOperand(4));
04711     Ops.push_back(Chain);
04712     break;
04713   case NVPTXISD::Suld3DV4I32Zero:
04714     Opc = NVPTX::SULD_3D_V4I32_ZERO;
04715     Ops.push_back(TexHandle);
04716     Ops.push_back(N->getOperand(2));
04717     Ops.push_back(N->getOperand(3));
04718     Ops.push_back(N->getOperand(4));
04719     Ops.push_back(Chain);
04720     break;
04721   }
04722   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
04723   return Ret;
04724 }
04725 
04726 
04727 /// SelectBFE - Look for instruction sequences that can be made more efficient
04728 /// by using the 'bfe' (bit-field extract) PTX instruction
04729 SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
04730   SDValue LHS = N->getOperand(0);
04731   SDValue RHS = N->getOperand(1);
04732   SDValue Len;
04733   SDValue Start;
04734   SDValue Val;
04735   bool IsSigned = false;
04736 
04737   if (N->getOpcode() == ISD::AND) {
04738     // Canonicalize the operands
04739     // We want 'and %val, %mask'
04740     if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
04741       std::swap(LHS, RHS);
04742     }
04743 
04744     ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
04745     if (!Mask) {
04746       // We need a constant mask on the RHS of the AND
04747       return NULL;
04748     }
04749 
04750     // Extract the mask bits
04751     uint64_t MaskVal = Mask->getZExtValue();
04752     if (!isMask_64(MaskVal)) {
04753       // We *could* handle shifted masks here, but doing so would require an
04754       // 'and' operation to fix up the low-order bits so we would trade
04755       // shr+and for bfe+and, which has the same throughput
04756       return NULL;
04757     }
04758 
04759     // How many bits are in our mask?
04760     uint64_t NumBits = countTrailingOnes(MaskVal);
04761     Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
04762 
04763     if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
04764       // We have a 'srl/and' pair, extract the effective start bit and length
04765       Val = LHS.getNode()->getOperand(0);
04766       Start = LHS.getNode()->getOperand(1);
04767       ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
04768       if (StartConst) {
04769         uint64_t StartVal = StartConst->getZExtValue();
04770         // How many "good" bits do we have left?  "good" is defined here as bits
04771         // that exist in the original value, not shifted in.
04772         uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
04773         if (NumBits > GoodBits) {
04774           // Do not handle the case where bits have been shifted in. In theory
04775           // we could handle this, but the cost is likely higher than just
04776           // emitting the srl/and pair.
04777           return NULL;
04778         }
04779         Start = CurDAG->getTargetConstant(StartVal, MVT::i32);
04780       } else {
04781         // Do not handle the case where the shift amount (can be zero if no srl
04782         // was found) is not constant. We could handle this case, but it would
04783         // require run-time logic that would be more expensive than just
04784         // emitting the srl/and pair.
04785         return NULL;
04786       }
04787     } else {
04788       // Do not handle the case where the LHS of the and is not a shift. While
04789       // it would be trivial to handle this case, it would just transform
04790       // 'and' -> 'bfe', but 'and' has higher-throughput.
04791       return NULL;
04792     }
04793   } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
04794     if (LHS->getOpcode() == ISD::AND) {
04795       ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
04796       if (!ShiftCnst) {
04797         // Shift amount must be constant
04798         return NULL;
04799       }
04800 
04801       uint64_t ShiftAmt = ShiftCnst->getZExtValue();
04802 
04803       SDValue AndLHS = LHS->getOperand(0);
04804       SDValue AndRHS = LHS->getOperand(1);
04805 
04806       // Canonicalize the AND to have the mask on the RHS
04807       if (isa<ConstantSDNode>(AndLHS)) {
04808         std::swap(AndLHS, AndRHS);
04809       }
04810 
04811       ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
04812       if (!MaskCnst) {
04813         // Mask must be constant
04814         return NULL;
04815       }
04816 
04817       uint64_t MaskVal = MaskCnst->getZExtValue();
04818       uint64_t NumZeros;
04819       uint64_t NumBits;
04820       if (isMask_64(MaskVal)) {
04821         NumZeros = 0;
04822         // The number of bits in the result bitfield will be the number of
04823         // trailing ones (the AND) minus the number of bits we shift off
04824         NumBits = countTrailingOnes(MaskVal) - ShiftAmt;
04825       } else if (isShiftedMask_64(MaskVal)) {
04826         NumZeros = countTrailingZeros(MaskVal);
04827         unsigned NumOnes = countTrailingOnes(MaskVal >> NumZeros);
04828         // The number of bits in the result bitfield will be the number of
04829         // trailing zeros plus the number of set bits in the mask minus the
04830         // number of bits we shift off
04831         NumBits = NumZeros + NumOnes - ShiftAmt;
04832       } else {
04833         // This is not a mask we can handle
04834         return NULL;
04835       }
04836 
04837       if (ShiftAmt < NumZeros) {
04838         // Handling this case would require extra logic that would make this
04839         // transformation non-profitable
04840         return NULL;
04841       }
04842 
04843       Val = AndLHS;
04844       Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32);
04845       Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
04846     } else if (LHS->getOpcode() == ISD::SHL) {
04847       // Here, we have a pattern like:
04848       //
04849       // (sra (shl val, NN), MM)
04850       // or
04851       // (srl (shl val, NN), MM)
04852       //
04853       // If MM >= NN, we can efficiently optimize this with bfe
04854       Val = LHS->getOperand(0);
04855 
04856       SDValue ShlRHS = LHS->getOperand(1);
04857       ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
04858       if (!ShlCnst) {
04859         // Shift amount must be constant
04860         return NULL;
04861       }
04862       uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
04863 
04864       SDValue ShrRHS = RHS;
04865       ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
04866       if (!ShrCnst) {
04867         // Shift amount must be constant
04868         return NULL;
04869       }
04870       uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
04871 
04872       // To avoid extra codegen and be profitable, we need Outer >= Inner
04873       if (OuterShiftAmt < InnerShiftAmt) {
04874         return NULL;
04875       }
04876 
04877       // If the outer shift is more than the type size, we have no bitfield to
04878       // extract (since we also check that the inner shift is <= the outer shift
04879       // then this also implies that the inner shift is < the type size)
04880       if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
04881         return NULL;
04882       }
04883 
04884       Start =
04885         CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32);
04886       Len =
04887         CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
04888                                   OuterShiftAmt, MVT::i32);
04889 
04890       if (N->getOpcode() == ISD::SRA) {
04891         // If we have a arithmetic right shift, we need to use the signed bfe
04892         // variant
04893         IsSigned = true;
04894       }
04895     } else {
04896       // No can do...
04897       return NULL;
04898     }
04899   } else {
04900     // No can do...
04901     return NULL;
04902   }
04903 
04904 
04905   unsigned Opc;
04906   // For the BFE operations we form here from "and" and "srl", always use the
04907   // unsigned variants.
04908   if (Val.getValueType() == MVT::i32) {
04909     if (IsSigned) {
04910       Opc = NVPTX::BFE_S32rii;
04911     } else {
04912       Opc = NVPTX::BFE_U32rii;
04913     }
04914   } else if (Val.getValueType() == MVT::i64) {
04915     if (IsSigned) {
04916       Opc = NVPTX::BFE_S64rii;
04917     } else {
04918       Opc = NVPTX::BFE_U64rii;
04919     }
04920   } else {
04921     // We cannot handle this type
04922     return NULL;
04923   }
04924 
04925   SDValue Ops[] = {
04926     Val, Start, Len
04927   };
04928 
04929   SDNode *Ret =
04930     CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
04931 
04932   return Ret;
04933 }
04934 
04935 // SelectDirectAddr - Match a direct address for DAG.
04936 // A direct address could be a globaladdress or externalsymbol.
04937 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
04938   // Return true if TGA or ES.
04939   if (N.getOpcode() == ISD::TargetGlobalAddress ||
04940       N.getOpcode() == ISD::TargetExternalSymbol) {
04941     Address = N;
04942     return true;
04943   }
04944   if (N.getOpcode() == NVPTXISD::Wrapper) {
04945     Address = N.getOperand(0);
04946     return true;
04947   }
04948   if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
04949     unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
04950     if (IID == Intrinsic::nvvm_ptr_gen_to_param)
04951       if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
04952         return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
04953   }
04954   return false;
04955 }
04956 
04957 // symbol+offset
04958 bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
04959     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
04960   if (Addr.getOpcode() == ISD::ADD) {
04961     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
04962       SDValue base = Addr.getOperand(0);
04963       if (SelectDirectAddr(base, Base)) {
04964         Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
04965         return true;
04966       }
04967     }
04968   }
04969   return false;
04970 }
04971 
04972 // symbol+offset
04973 bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
04974                                      SDValue &Base, SDValue &Offset) {
04975   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
04976 }
04977 
04978 // symbol+offset
04979 bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
04980                                        SDValue &Base, SDValue &Offset) {
04981   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
04982 }
04983 
04984 // register+offset
04985 bool NVPTXDAGToDAGISel::SelectADDRri_imp(
04986     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
04987   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
04988     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
04989     Offset = CurDAG->getTargetConstant(0, mvt);
04990     return true;
04991   }
04992   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
04993       Addr.getOpcode() == ISD::TargetGlobalAddress)
04994     return false; // direct calls.
04995 
04996   if (Addr.getOpcode() == ISD::ADD) {
04997     if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
04998       return false;
04999     }
05000     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
05001       if (FrameIndexSDNode *FIN =
05002               dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
05003         // Constant offset from frame ref.
05004         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
05005       else
05006         Base = Addr.getOperand(0);
05007       Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
05008       return true;
05009     }
05010   }
05011   return false;
05012 }
05013 
05014 // register+offset
05015 bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
05016                                      SDValue &Base, SDValue &Offset) {
05017   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
05018 }
05019 
05020 // register+offset
05021 bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
05022                                        SDValue &Base, SDValue &Offset) {
05023   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
05024 }
05025 
05026 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
05027                                                  unsigned int spN) const {
05028   const Value *Src = nullptr;
05029   if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
05030     if (spN == 0 && mN->getMemOperand()->getPseudoValue())
05031       return true;
05032     Src = mN->getMemOperand()->getValue();
05033   }
05034   if (!Src)
05035     return false;
05036   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
05037     return (PT->getAddressSpace() == spN);
05038   return false;
05039 }
05040 
05041 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
05042 /// inline asm expressions.
05043 bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
05044     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
05045   SDValue Op0, Op1;
05046   switch (ConstraintID) {
05047   default:
05048     return true;
05049   case InlineAsm::Constraint_m: // memory
05050     if (SelectDirectAddr(Op, Op0)) {
05051       OutOps.push_back(Op0);
05052       OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
05053       return false;
05054     }
05055     if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
05056       OutOps.push_back(Op0);
05057       OutOps.push_back(Op1);
05058       return false;
05059     }
05060     break;
05061   }
05062   return true;
05063 }