LLVM  mainline
NVPTXISelDAGToDAG.cpp
Go to the documentation of this file.
00001 //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file defines an instruction selector for the NVPTX target.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "NVPTXISelDAGToDAG.h"
00015 #include "llvm/IR/GlobalValue.h"
00016 #include "llvm/IR/Instructions.h"
00017 #include "llvm/Support/CommandLine.h"
00018 #include "llvm/Support/Debug.h"
00019 #include "llvm/Support/ErrorHandling.h"
00020 #include "llvm/Support/raw_ostream.h"
00021 #include "llvm/Target/TargetIntrinsicInfo.h"
00022 
00023 using namespace llvm;
00024 
00025 #define DEBUG_TYPE "nvptx-isel"
00026 
00027 static cl::opt<int> UsePrecDivF32(
00028     "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
00029     cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
00030              " IEEE Compliant F32 div.rnd if available."),
00031     cl::init(2));
00032 
00033 static cl::opt<bool>
00034 UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
00035           cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
00036           cl::init(true));
00037 
00038 static cl::opt<bool>
00039 FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
00040            cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
00041            cl::init(false));
00042 
00043 
00044 /// createNVPTXISelDag - This pass converts a legalized DAG into a
00045 /// NVPTX-specific DAG, ready for instruction scheduling.
00046 FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
00047                                        llvm::CodeGenOpt::Level OptLevel) {
00048   return new NVPTXDAGToDAGISel(TM, OptLevel);
00049 }
00050 
00051 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
00052                                      CodeGenOpt::Level OptLevel)
00053     : SelectionDAGISel(tm, OptLevel), TM(tm) {
00054   doMulWide = (OptLevel > 0);
00055 }
00056 
00057 bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
00058     Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget());
00059     return SelectionDAGISel::runOnMachineFunction(MF);
00060 }
00061 
00062 int NVPTXDAGToDAGISel::getDivF32Level() const {
00063   if (UsePrecDivF32.getNumOccurrences() > 0) {
00064     // If nvptx-prec-div32=N is used on the command-line, always honor it
00065     return UsePrecDivF32;
00066   } else {
00067     // Otherwise, use div.approx if fast math is enabled
00068     if (TM.Options.UnsafeFPMath)
00069       return 0;
00070     else
00071       return 2;
00072   }
00073 }
00074 
00075 bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
00076   if (UsePrecSqrtF32.getNumOccurrences() > 0) {
00077     // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
00078     return UsePrecSqrtF32;
00079   } else {
00080     // Otherwise, use sqrt.approx if fast math is enabled
00081     return !TM.Options.UnsafeFPMath;
00082   }
00083 }
00084 
00085 bool NVPTXDAGToDAGISel::useF32FTZ() const {
00086   if (FtzEnabled.getNumOccurrences() > 0) {
00087     // If nvptx-f32ftz is used on the command-line, always honor it
00088     return FtzEnabled;
00089   } else {
00090     const Function *F = MF->getFunction();
00091     // Otherwise, check for an nvptx-f32ftz attribute on the function
00092     if (F->hasFnAttribute("nvptx-f32ftz"))
00093       return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true";
00094     else
00095       return false;
00096   }
00097 }
00098 
00099 bool NVPTXDAGToDAGISel::allowFMA() const {
00100   const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
00101   return TL->allowFMA(*MF, OptLevel);
00102 }
00103 
00104 /// Select - Select instructions not customized! Used for
00105 /// expanded, promoted and normal instructions.
00106 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
00107 
00108   if (N->isMachineOpcode()) {
00109     N->setNodeId(-1);
00110     return nullptr; // Already selected.
00111   }
00112 
00113   SDNode *ResNode = nullptr;
00114   switch (N->getOpcode()) {
00115   case ISD::LOAD:
00116     ResNode = SelectLoad(N);
00117     break;
00118   case ISD::STORE:
00119     ResNode = SelectStore(N);
00120     break;
00121   case NVPTXISD::LoadV2:
00122   case NVPTXISD::LoadV4:
00123     ResNode = SelectLoadVector(N);
00124     break;
00125   case NVPTXISD::LDGV2:
00126   case NVPTXISD::LDGV4:
00127   case NVPTXISD::LDUV2:
00128   case NVPTXISD::LDUV4:
00129     ResNode = SelectLDGLDU(N);
00130     break;
00131   case NVPTXISD::StoreV2:
00132   case NVPTXISD::StoreV4:
00133     ResNode = SelectStoreVector(N);
00134     break;
00135   case NVPTXISD::LoadParam:
00136   case NVPTXISD::LoadParamV2:
00137   case NVPTXISD::LoadParamV4:
00138     ResNode = SelectLoadParam(N);
00139     break;
00140   case NVPTXISD::StoreRetval:
00141   case NVPTXISD::StoreRetvalV2:
00142   case NVPTXISD::StoreRetvalV4:
00143     ResNode = SelectStoreRetval(N);
00144     break;
00145   case NVPTXISD::StoreParam:
00146   case NVPTXISD::StoreParamV2:
00147   case NVPTXISD::StoreParamV4:
00148   case NVPTXISD::StoreParamS32:
00149   case NVPTXISD::StoreParamU32:
00150     ResNode = SelectStoreParam(N);
00151     break;
00152   case ISD::INTRINSIC_WO_CHAIN:
00153     ResNode = SelectIntrinsicNoChain(N);
00154     break;
00155   case ISD::INTRINSIC_W_CHAIN:
00156     ResNode = SelectIntrinsicChain(N);
00157     break;
00158   case NVPTXISD::Tex1DFloatS32:
00159   case NVPTXISD::Tex1DFloatFloat:
00160   case NVPTXISD::Tex1DFloatFloatLevel:
00161   case NVPTXISD::Tex1DFloatFloatGrad:
00162   case NVPTXISD::Tex1DS32S32:
00163   case NVPTXISD::Tex1DS32Float:
00164   case NVPTXISD::Tex1DS32FloatLevel:
00165   case NVPTXISD::Tex1DS32FloatGrad:
00166   case NVPTXISD::Tex1DU32S32:
00167   case NVPTXISD::Tex1DU32Float:
00168   case NVPTXISD::Tex1DU32FloatLevel:
00169   case NVPTXISD::Tex1DU32FloatGrad:
00170   case NVPTXISD::Tex1DArrayFloatS32:
00171   case NVPTXISD::Tex1DArrayFloatFloat:
00172   case NVPTXISD::Tex1DArrayFloatFloatLevel:
00173   case NVPTXISD::Tex1DArrayFloatFloatGrad:
00174   case NVPTXISD::Tex1DArrayS32S32:
00175   case NVPTXISD::Tex1DArrayS32Float:
00176   case NVPTXISD::Tex1DArrayS32FloatLevel:
00177   case NVPTXISD::Tex1DArrayS32FloatGrad:
00178   case NVPTXISD::Tex1DArrayU32S32:
00179   case NVPTXISD::Tex1DArrayU32Float:
00180   case NVPTXISD::Tex1DArrayU32FloatLevel:
00181   case NVPTXISD::Tex1DArrayU32FloatGrad:
00182   case NVPTXISD::Tex2DFloatS32:
00183   case NVPTXISD::Tex2DFloatFloat:
00184   case NVPTXISD::Tex2DFloatFloatLevel:
00185   case NVPTXISD::Tex2DFloatFloatGrad:
00186   case NVPTXISD::Tex2DS32S32:
00187   case NVPTXISD::Tex2DS32Float:
00188   case NVPTXISD::Tex2DS32FloatLevel:
00189   case NVPTXISD::Tex2DS32FloatGrad:
00190   case NVPTXISD::Tex2DU32S32:
00191   case NVPTXISD::Tex2DU32Float:
00192   case NVPTXISD::Tex2DU32FloatLevel:
00193   case NVPTXISD::Tex2DU32FloatGrad:
00194   case NVPTXISD::Tex2DArrayFloatS32:
00195   case NVPTXISD::Tex2DArrayFloatFloat:
00196   case NVPTXISD::Tex2DArrayFloatFloatLevel:
00197   case NVPTXISD::Tex2DArrayFloatFloatGrad:
00198   case NVPTXISD::Tex2DArrayS32S32:
00199   case NVPTXISD::Tex2DArrayS32Float:
00200   case NVPTXISD::Tex2DArrayS32FloatLevel:
00201   case NVPTXISD::Tex2DArrayS32FloatGrad:
00202   case NVPTXISD::Tex2DArrayU32S32:
00203   case NVPTXISD::Tex2DArrayU32Float:
00204   case NVPTXISD::Tex2DArrayU32FloatLevel:
00205   case NVPTXISD::Tex2DArrayU32FloatGrad:
00206   case NVPTXISD::Tex3DFloatS32:
00207   case NVPTXISD::Tex3DFloatFloat:
00208   case NVPTXISD::Tex3DFloatFloatLevel:
00209   case NVPTXISD::Tex3DFloatFloatGrad:
00210   case NVPTXISD::Tex3DS32S32:
00211   case NVPTXISD::Tex3DS32Float:
00212   case NVPTXISD::Tex3DS32FloatLevel:
00213   case NVPTXISD::Tex3DS32FloatGrad:
00214   case NVPTXISD::Tex3DU32S32:
00215   case NVPTXISD::Tex3DU32Float:
00216   case NVPTXISD::Tex3DU32FloatLevel:
00217   case NVPTXISD::Tex3DU32FloatGrad:
00218   case NVPTXISD::TexCubeFloatFloat:
00219   case NVPTXISD::TexCubeFloatFloatLevel:
00220   case NVPTXISD::TexCubeS32Float:
00221   case NVPTXISD::TexCubeS32FloatLevel:
00222   case NVPTXISD::TexCubeU32Float:
00223   case NVPTXISD::TexCubeU32FloatLevel:
00224   case NVPTXISD::TexCubeArrayFloatFloat:
00225   case NVPTXISD::TexCubeArrayFloatFloatLevel:
00226   case NVPTXISD::TexCubeArrayS32Float:
00227   case NVPTXISD::TexCubeArrayS32FloatLevel:
00228   case NVPTXISD::TexCubeArrayU32Float:
00229   case NVPTXISD::TexCubeArrayU32FloatLevel:
00230   case NVPTXISD::Tld4R2DFloatFloat:
00231   case NVPTXISD::Tld4G2DFloatFloat:
00232   case NVPTXISD::Tld4B2DFloatFloat:
00233   case NVPTXISD::Tld4A2DFloatFloat:
00234   case NVPTXISD::Tld4R2DS64Float:
00235   case NVPTXISD::Tld4G2DS64Float:
00236   case NVPTXISD::Tld4B2DS64Float:
00237   case NVPTXISD::Tld4A2DS64Float:
00238   case NVPTXISD::Tld4R2DU64Float:
00239   case NVPTXISD::Tld4G2DU64Float:
00240   case NVPTXISD::Tld4B2DU64Float:
00241   case NVPTXISD::Tld4A2DU64Float:
00242   case NVPTXISD::TexUnified1DFloatS32:
00243   case NVPTXISD::TexUnified1DFloatFloat:
00244   case NVPTXISD::TexUnified1DFloatFloatLevel:
00245   case NVPTXISD::TexUnified1DFloatFloatGrad:
00246   case NVPTXISD::TexUnified1DS32S32:
00247   case NVPTXISD::TexUnified1DS32Float:
00248   case NVPTXISD::TexUnified1DS32FloatLevel:
00249   case NVPTXISD::TexUnified1DS32FloatGrad:
00250   case NVPTXISD::TexUnified1DU32S32:
00251   case NVPTXISD::TexUnified1DU32Float:
00252   case NVPTXISD::TexUnified1DU32FloatLevel:
00253   case NVPTXISD::TexUnified1DU32FloatGrad:
00254   case NVPTXISD::TexUnified1DArrayFloatS32:
00255   case NVPTXISD::TexUnified1DArrayFloatFloat:
00256   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
00257   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
00258   case NVPTXISD::TexUnified1DArrayS32S32:
00259   case NVPTXISD::TexUnified1DArrayS32Float:
00260   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
00261   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
00262   case NVPTXISD::TexUnified1DArrayU32S32:
00263   case NVPTXISD::TexUnified1DArrayU32Float:
00264   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
00265   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
00266   case NVPTXISD::TexUnified2DFloatS32:
00267   case NVPTXISD::TexUnified2DFloatFloat:
00268   case NVPTXISD::TexUnified2DFloatFloatLevel:
00269   case NVPTXISD::TexUnified2DFloatFloatGrad:
00270   case NVPTXISD::TexUnified2DS32S32:
00271   case NVPTXISD::TexUnified2DS32Float:
00272   case NVPTXISD::TexUnified2DS32FloatLevel:
00273   case NVPTXISD::TexUnified2DS32FloatGrad:
00274   case NVPTXISD::TexUnified2DU32S32:
00275   case NVPTXISD::TexUnified2DU32Float:
00276   case NVPTXISD::TexUnified2DU32FloatLevel:
00277   case NVPTXISD::TexUnified2DU32FloatGrad:
00278   case NVPTXISD::TexUnified2DArrayFloatS32:
00279   case NVPTXISD::TexUnified2DArrayFloatFloat:
00280   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
00281   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
00282   case NVPTXISD::TexUnified2DArrayS32S32:
00283   case NVPTXISD::TexUnified2DArrayS32Float:
00284   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
00285   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
00286   case NVPTXISD::TexUnified2DArrayU32S32:
00287   case NVPTXISD::TexUnified2DArrayU32Float:
00288   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
00289   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
00290   case NVPTXISD::TexUnified3DFloatS32:
00291   case NVPTXISD::TexUnified3DFloatFloat:
00292   case NVPTXISD::TexUnified3DFloatFloatLevel:
00293   case NVPTXISD::TexUnified3DFloatFloatGrad:
00294   case NVPTXISD::TexUnified3DS32S32:
00295   case NVPTXISD::TexUnified3DS32Float:
00296   case NVPTXISD::TexUnified3DS32FloatLevel:
00297   case NVPTXISD::TexUnified3DS32FloatGrad:
00298   case NVPTXISD::TexUnified3DU32S32:
00299   case NVPTXISD::TexUnified3DU32Float:
00300   case NVPTXISD::TexUnified3DU32FloatLevel:
00301   case NVPTXISD::TexUnified3DU32FloatGrad:
00302   case NVPTXISD::TexUnifiedCubeFloatFloat:
00303   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
00304   case NVPTXISD::TexUnifiedCubeS32Float:
00305   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
00306   case NVPTXISD::TexUnifiedCubeU32Float:
00307   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
00308   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
00309   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
00310   case NVPTXISD::TexUnifiedCubeArrayS32Float:
00311   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
00312   case NVPTXISD::TexUnifiedCubeArrayU32Float:
00313   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
00314   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
00315   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
00316   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
00317   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
00318   case NVPTXISD::Tld4UnifiedR2DS64Float:
00319   case NVPTXISD::Tld4UnifiedG2DS64Float:
00320   case NVPTXISD::Tld4UnifiedB2DS64Float:
00321   case NVPTXISD::Tld4UnifiedA2DS64Float:
00322   case NVPTXISD::Tld4UnifiedR2DU64Float:
00323   case NVPTXISD::Tld4UnifiedG2DU64Float:
00324   case NVPTXISD::Tld4UnifiedB2DU64Float:
00325   case NVPTXISD::Tld4UnifiedA2DU64Float:
00326     ResNode = SelectTextureIntrinsic(N);
00327     break;
00328   case NVPTXISD::Suld1DI8Clamp:
00329   case NVPTXISD::Suld1DI16Clamp:
00330   case NVPTXISD::Suld1DI32Clamp:
00331   case NVPTXISD::Suld1DI64Clamp:
00332   case NVPTXISD::Suld1DV2I8Clamp:
00333   case NVPTXISD::Suld1DV2I16Clamp:
00334   case NVPTXISD::Suld1DV2I32Clamp:
00335   case NVPTXISD::Suld1DV2I64Clamp:
00336   case NVPTXISD::Suld1DV4I8Clamp:
00337   case NVPTXISD::Suld1DV4I16Clamp:
00338   case NVPTXISD::Suld1DV4I32Clamp:
00339   case NVPTXISD::Suld1DArrayI8Clamp:
00340   case NVPTXISD::Suld1DArrayI16Clamp:
00341   case NVPTXISD::Suld1DArrayI32Clamp:
00342   case NVPTXISD::Suld1DArrayI64Clamp:
00343   case NVPTXISD::Suld1DArrayV2I8Clamp:
00344   case NVPTXISD::Suld1DArrayV2I16Clamp:
00345   case NVPTXISD::Suld1DArrayV2I32Clamp:
00346   case NVPTXISD::Suld1DArrayV2I64Clamp:
00347   case NVPTXISD::Suld1DArrayV4I8Clamp:
00348   case NVPTXISD::Suld1DArrayV4I16Clamp:
00349   case NVPTXISD::Suld1DArrayV4I32Clamp:
00350   case NVPTXISD::Suld2DI8Clamp:
00351   case NVPTXISD::Suld2DI16Clamp:
00352   case NVPTXISD::Suld2DI32Clamp:
00353   case NVPTXISD::Suld2DI64Clamp:
00354   case NVPTXISD::Suld2DV2I8Clamp:
00355   case NVPTXISD::Suld2DV2I16Clamp:
00356   case NVPTXISD::Suld2DV2I32Clamp:
00357   case NVPTXISD::Suld2DV2I64Clamp:
00358   case NVPTXISD::Suld2DV4I8Clamp:
00359   case NVPTXISD::Suld2DV4I16Clamp:
00360   case NVPTXISD::Suld2DV4I32Clamp:
00361   case NVPTXISD::Suld2DArrayI8Clamp:
00362   case NVPTXISD::Suld2DArrayI16Clamp:
00363   case NVPTXISD::Suld2DArrayI32Clamp:
00364   case NVPTXISD::Suld2DArrayI64Clamp:
00365   case NVPTXISD::Suld2DArrayV2I8Clamp:
00366   case NVPTXISD::Suld2DArrayV2I16Clamp:
00367   case NVPTXISD::Suld2DArrayV2I32Clamp:
00368   case NVPTXISD::Suld2DArrayV2I64Clamp:
00369   case NVPTXISD::Suld2DArrayV4I8Clamp:
00370   case NVPTXISD::Suld2DArrayV4I16Clamp:
00371   case NVPTXISD::Suld2DArrayV4I32Clamp:
00372   case NVPTXISD::Suld3DI8Clamp:
00373   case NVPTXISD::Suld3DI16Clamp:
00374   case NVPTXISD::Suld3DI32Clamp:
00375   case NVPTXISD::Suld3DI64Clamp:
00376   case NVPTXISD::Suld3DV2I8Clamp:
00377   case NVPTXISD::Suld3DV2I16Clamp:
00378   case NVPTXISD::Suld3DV2I32Clamp:
00379   case NVPTXISD::Suld3DV2I64Clamp:
00380   case NVPTXISD::Suld3DV4I8Clamp:
00381   case NVPTXISD::Suld3DV4I16Clamp:
00382   case NVPTXISD::Suld3DV4I32Clamp:
00383   case NVPTXISD::Suld1DI8Trap:
00384   case NVPTXISD::Suld1DI16Trap:
00385   case NVPTXISD::Suld1DI32Trap:
00386   case NVPTXISD::Suld1DI64Trap:
00387   case NVPTXISD::Suld1DV2I8Trap:
00388   case NVPTXISD::Suld1DV2I16Trap:
00389   case NVPTXISD::Suld1DV2I32Trap:
00390   case NVPTXISD::Suld1DV2I64Trap:
00391   case NVPTXISD::Suld1DV4I8Trap:
00392   case NVPTXISD::Suld1DV4I16Trap:
00393   case NVPTXISD::Suld1DV4I32Trap:
00394   case NVPTXISD::Suld1DArrayI8Trap:
00395   case NVPTXISD::Suld1DArrayI16Trap:
00396   case NVPTXISD::Suld1DArrayI32Trap:
00397   case NVPTXISD::Suld1DArrayI64Trap:
00398   case NVPTXISD::Suld1DArrayV2I8Trap:
00399   case NVPTXISD::Suld1DArrayV2I16Trap:
00400   case NVPTXISD::Suld1DArrayV2I32Trap:
00401   case NVPTXISD::Suld1DArrayV2I64Trap:
00402   case NVPTXISD::Suld1DArrayV4I8Trap:
00403   case NVPTXISD::Suld1DArrayV4I16Trap:
00404   case NVPTXISD::Suld1DArrayV4I32Trap:
00405   case NVPTXISD::Suld2DI8Trap:
00406   case NVPTXISD::Suld2DI16Trap:
00407   case NVPTXISD::Suld2DI32Trap:
00408   case NVPTXISD::Suld2DI64Trap:
00409   case NVPTXISD::Suld2DV2I8Trap:
00410   case NVPTXISD::Suld2DV2I16Trap:
00411   case NVPTXISD::Suld2DV2I32Trap:
00412   case NVPTXISD::Suld2DV2I64Trap:
00413   case NVPTXISD::Suld2DV4I8Trap:
00414   case NVPTXISD::Suld2DV4I16Trap:
00415   case NVPTXISD::Suld2DV4I32Trap:
00416   case NVPTXISD::Suld2DArrayI8Trap:
00417   case NVPTXISD::Suld2DArrayI16Trap:
00418   case NVPTXISD::Suld2DArrayI32Trap:
00419   case NVPTXISD::Suld2DArrayI64Trap:
00420   case NVPTXISD::Suld2DArrayV2I8Trap:
00421   case NVPTXISD::Suld2DArrayV2I16Trap:
00422   case NVPTXISD::Suld2DArrayV2I32Trap:
00423   case NVPTXISD::Suld2DArrayV2I64Trap:
00424   case NVPTXISD::Suld2DArrayV4I8Trap:
00425   case NVPTXISD::Suld2DArrayV4I16Trap:
00426   case NVPTXISD::Suld2DArrayV4I32Trap:
00427   case NVPTXISD::Suld3DI8Trap:
00428   case NVPTXISD::Suld3DI16Trap:
00429   case NVPTXISD::Suld3DI32Trap:
00430   case NVPTXISD::Suld3DI64Trap:
00431   case NVPTXISD::Suld3DV2I8Trap:
00432   case NVPTXISD::Suld3DV2I16Trap:
00433   case NVPTXISD::Suld3DV2I32Trap:
00434   case NVPTXISD::Suld3DV2I64Trap:
00435   case NVPTXISD::Suld3DV4I8Trap:
00436   case NVPTXISD::Suld3DV4I16Trap:
00437   case NVPTXISD::Suld3DV4I32Trap:
00438   case NVPTXISD::Suld1DI8Zero:
00439   case NVPTXISD::Suld1DI16Zero:
00440   case NVPTXISD::Suld1DI32Zero:
00441   case NVPTXISD::Suld1DI64Zero:
00442   case NVPTXISD::Suld1DV2I8Zero:
00443   case NVPTXISD::Suld1DV2I16Zero:
00444   case NVPTXISD::Suld1DV2I32Zero:
00445   case NVPTXISD::Suld1DV2I64Zero:
00446   case NVPTXISD::Suld1DV4I8Zero:
00447   case NVPTXISD::Suld1DV4I16Zero:
00448   case NVPTXISD::Suld1DV4I32Zero:
00449   case NVPTXISD::Suld1DArrayI8Zero:
00450   case NVPTXISD::Suld1DArrayI16Zero:
00451   case NVPTXISD::Suld1DArrayI32Zero:
00452   case NVPTXISD::Suld1DArrayI64Zero:
00453   case NVPTXISD::Suld1DArrayV2I8Zero:
00454   case NVPTXISD::Suld1DArrayV2I16Zero:
00455   case NVPTXISD::Suld1DArrayV2I32Zero:
00456   case NVPTXISD::Suld1DArrayV2I64Zero:
00457   case NVPTXISD::Suld1DArrayV4I8Zero:
00458   case NVPTXISD::Suld1DArrayV4I16Zero:
00459   case NVPTXISD::Suld1DArrayV4I32Zero:
00460   case NVPTXISD::Suld2DI8Zero:
00461   case NVPTXISD::Suld2DI16Zero:
00462   case NVPTXISD::Suld2DI32Zero:
00463   case NVPTXISD::Suld2DI64Zero:
00464   case NVPTXISD::Suld2DV2I8Zero:
00465   case NVPTXISD::Suld2DV2I16Zero:
00466   case NVPTXISD::Suld2DV2I32Zero:
00467   case NVPTXISD::Suld2DV2I64Zero:
00468   case NVPTXISD::Suld2DV4I8Zero:
00469   case NVPTXISD::Suld2DV4I16Zero:
00470   case NVPTXISD::Suld2DV4I32Zero:
00471   case NVPTXISD::Suld2DArrayI8Zero:
00472   case NVPTXISD::Suld2DArrayI16Zero:
00473   case NVPTXISD::Suld2DArrayI32Zero:
00474   case NVPTXISD::Suld2DArrayI64Zero:
00475   case NVPTXISD::Suld2DArrayV2I8Zero:
00476   case NVPTXISD::Suld2DArrayV2I16Zero:
00477   case NVPTXISD::Suld2DArrayV2I32Zero:
00478   case NVPTXISD::Suld2DArrayV2I64Zero:
00479   case NVPTXISD::Suld2DArrayV4I8Zero:
00480   case NVPTXISD::Suld2DArrayV4I16Zero:
00481   case NVPTXISD::Suld2DArrayV4I32Zero:
00482   case NVPTXISD::Suld3DI8Zero:
00483   case NVPTXISD::Suld3DI16Zero:
00484   case NVPTXISD::Suld3DI32Zero:
00485   case NVPTXISD::Suld3DI64Zero:
00486   case NVPTXISD::Suld3DV2I8Zero:
00487   case NVPTXISD::Suld3DV2I16Zero:
00488   case NVPTXISD::Suld3DV2I32Zero:
00489   case NVPTXISD::Suld3DV2I64Zero:
00490   case NVPTXISD::Suld3DV4I8Zero:
00491   case NVPTXISD::Suld3DV4I16Zero:
00492   case NVPTXISD::Suld3DV4I32Zero:
00493     ResNode = SelectSurfaceIntrinsic(N);
00494     break;
00495   case ISD::AND:
00496   case ISD::SRA:
00497   case ISD::SRL:
00498     // Try to select BFE
00499     ResNode = SelectBFE(N);
00500     break;
00501   case ISD::ADDRSPACECAST:
00502     ResNode = SelectAddrSpaceCast(N);
00503     break;
00504   default:
00505     break;
00506   }
00507   if (ResNode)
00508     return ResNode;
00509   return SelectCode(N);
00510 }
00511 
00512 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
00513   unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
00514   switch (IID) {
00515   default:
00516     return NULL;
00517   case Intrinsic::nvvm_ldg_global_f:
00518   case Intrinsic::nvvm_ldg_global_i:
00519   case Intrinsic::nvvm_ldg_global_p:
00520   case Intrinsic::nvvm_ldu_global_f:
00521   case Intrinsic::nvvm_ldu_global_i:
00522   case Intrinsic::nvvm_ldu_global_p:
00523     return SelectLDGLDU(N);
00524   }
00525 }
00526 
00527 static unsigned int getCodeAddrSpace(MemSDNode *N) {
00528   const Value *Src = N->getMemOperand()->getValue();
00529 
00530   if (!Src)
00531     return NVPTX::PTXLdStInstCode::GENERIC;
00532 
00533   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
00534     switch (PT->getAddressSpace()) {
00535     case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
00536     case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
00537     case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
00538     case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
00539     case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
00540     case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
00541     default: break;
00542     }
00543   }
00544   return NVPTX::PTXLdStInstCode::GENERIC;
00545 }
00546 
00547 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
00548   unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
00549   switch (IID) {
00550   default:
00551     return nullptr;
00552   case Intrinsic::nvvm_texsurf_handle_internal:
00553     return SelectTexSurfHandle(N);
00554   }
00555 }
00556 
00557 SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
00558   // Op 0 is the intrinsic ID
00559   SDValue Wrapper = N->getOperand(1);
00560   SDValue GlobalVal = Wrapper.getOperand(0);
00561   return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
00562                                 GlobalVal);
00563 }
00564 
00565 SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
00566   SDValue Src = N->getOperand(0);
00567   AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
00568   unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
00569   unsigned DstAddrSpace = CastN->getDestAddressSpace();
00570 
00571   assert(SrcAddrSpace != DstAddrSpace &&
00572          "addrspacecast must be between different address spaces");
00573 
00574   if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
00575     // Specific to generic
00576     unsigned Opc;
00577     switch (SrcAddrSpace) {
00578     default: report_fatal_error("Bad address space in addrspacecast");
00579     case ADDRESS_SPACE_GLOBAL:
00580       Opc = TM.is64Bit() ? NVPTX::cvta_global_yes_64 : NVPTX::cvta_global_yes;
00581       break;
00582     case ADDRESS_SPACE_SHARED:
00583       Opc = TM.is64Bit() ? NVPTX::cvta_shared_yes_64 : NVPTX::cvta_shared_yes;
00584       break;
00585     case ADDRESS_SPACE_CONST:
00586       Opc = TM.is64Bit() ? NVPTX::cvta_const_yes_64 : NVPTX::cvta_const_yes;
00587       break;
00588     case ADDRESS_SPACE_LOCAL:
00589       Opc = TM.is64Bit() ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes;
00590       break;
00591     }
00592     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00593   } else {
00594     // Generic to specific
00595     if (SrcAddrSpace != 0)
00596       report_fatal_error("Cannot cast between two non-generic address spaces");
00597     unsigned Opc;
00598     switch (DstAddrSpace) {
00599     default: report_fatal_error("Bad address space in addrspacecast");
00600     case ADDRESS_SPACE_GLOBAL:
00601       Opc = TM.is64Bit() ? NVPTX::cvta_to_global_yes_64
00602                          : NVPTX::cvta_to_global_yes;
00603       break;
00604     case ADDRESS_SPACE_SHARED:
00605       Opc = TM.is64Bit() ? NVPTX::cvta_to_shared_yes_64
00606                          : NVPTX::cvta_to_shared_yes;
00607       break;
00608     case ADDRESS_SPACE_CONST:
00609       Opc =
00610           TM.is64Bit() ? NVPTX::cvta_to_const_yes_64 : NVPTX::cvta_to_const_yes;
00611       break;
00612     case ADDRESS_SPACE_LOCAL:
00613       Opc =
00614           TM.is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes;
00615       break;
00616     }
00617     return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
00618   }
00619 }
00620 
00621 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
00622   SDLoc dl(N);
00623   LoadSDNode *LD = cast<LoadSDNode>(N);
00624   EVT LoadedVT = LD->getMemoryVT();
00625   SDNode *NVPTXLD = nullptr;
00626 
00627   // do not support pre/post inc/dec
00628   if (LD->isIndexed())
00629     return nullptr;
00630 
00631   if (!LoadedVT.isSimple())
00632     return nullptr;
00633 
00634   // Address Space Setting
00635   unsigned int codeAddrSpace = getCodeAddrSpace(LD);
00636 
00637   // Volatile Setting
00638   // - .volatile is only availalble for .global and .shared
00639   bool isVolatile = LD->isVolatile();
00640   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00641       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00642       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00643     isVolatile = false;
00644 
00645   // Vector Setting
00646   MVT SimpleVT = LoadedVT.getSimpleVT();
00647   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
00648   if (SimpleVT.isVector()) {
00649     unsigned num = SimpleVT.getVectorNumElements();
00650     if (num == 2)
00651       vecType = NVPTX::PTXLdStInstCode::V2;
00652     else if (num == 4)
00653       vecType = NVPTX::PTXLdStInstCode::V4;
00654     else
00655       return nullptr;
00656   }
00657 
00658   // Type Setting: fromType + fromTypeWidth
00659   //
00660   // Sign   : ISD::SEXTLOAD
00661   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00662   //          type is integer
00663   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00664   MVT ScalarVT = SimpleVT.getScalarType();
00665   // Read at least 8 bits (predicates are stored as 8-bit values)
00666   unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00667   unsigned int fromType;
00668   if ((LD->getExtensionType() == ISD::SEXTLOAD))
00669     fromType = NVPTX::PTXLdStInstCode::Signed;
00670   else if (ScalarVT.isFloatingPoint())
00671     fromType = NVPTX::PTXLdStInstCode::Float;
00672   else
00673     fromType = NVPTX::PTXLdStInstCode::Unsigned;
00674 
00675   // Create the machine instruction DAG
00676   SDValue Chain = N->getOperand(0);
00677   SDValue N1 = N->getOperand(1);
00678   SDValue Addr;
00679   SDValue Offset, Base;
00680   unsigned Opcode;
00681   MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
00682 
00683   if (SelectDirectAddr(N1, Addr)) {
00684     switch (TargetVT) {
00685     case MVT::i8:
00686       Opcode = NVPTX::LD_i8_avar;
00687       break;
00688     case MVT::i16:
00689       Opcode = NVPTX::LD_i16_avar;
00690       break;
00691     case MVT::i32:
00692       Opcode = NVPTX::LD_i32_avar;
00693       break;
00694     case MVT::i64:
00695       Opcode = NVPTX::LD_i64_avar;
00696       break;
00697     case MVT::f32:
00698       Opcode = NVPTX::LD_f32_avar;
00699       break;
00700     case MVT::f64:
00701       Opcode = NVPTX::LD_f64_avar;
00702       break;
00703     default:
00704       return nullptr;
00705     }
00706     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
00707                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
00708                       getI32Imm(fromTypeWidth, dl), Addr, Chain };
00709     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00710   } else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
00711                           : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
00712     switch (TargetVT) {
00713     case MVT::i8:
00714       Opcode = NVPTX::LD_i8_asi;
00715       break;
00716     case MVT::i16:
00717       Opcode = NVPTX::LD_i16_asi;
00718       break;
00719     case MVT::i32:
00720       Opcode = NVPTX::LD_i32_asi;
00721       break;
00722     case MVT::i64:
00723       Opcode = NVPTX::LD_i64_asi;
00724       break;
00725     case MVT::f32:
00726       Opcode = NVPTX::LD_f32_asi;
00727       break;
00728     case MVT::f64:
00729       Opcode = NVPTX::LD_f64_asi;
00730       break;
00731     default:
00732       return nullptr;
00733     }
00734     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
00735                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
00736                       getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
00737     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00738   } else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
00739                           : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
00740     if (TM.is64Bit()) {
00741       switch (TargetVT) {
00742       case MVT::i8:
00743         Opcode = NVPTX::LD_i8_ari_64;
00744         break;
00745       case MVT::i16:
00746         Opcode = NVPTX::LD_i16_ari_64;
00747         break;
00748       case MVT::i32:
00749         Opcode = NVPTX::LD_i32_ari_64;
00750         break;
00751       case MVT::i64:
00752         Opcode = NVPTX::LD_i64_ari_64;
00753         break;
00754       case MVT::f32:
00755         Opcode = NVPTX::LD_f32_ari_64;
00756         break;
00757       case MVT::f64:
00758         Opcode = NVPTX::LD_f64_ari_64;
00759         break;
00760       default:
00761         return nullptr;
00762       }
00763     } else {
00764       switch (TargetVT) {
00765       case MVT::i8:
00766         Opcode = NVPTX::LD_i8_ari;
00767         break;
00768       case MVT::i16:
00769         Opcode = NVPTX::LD_i16_ari;
00770         break;
00771       case MVT::i32:
00772         Opcode = NVPTX::LD_i32_ari;
00773         break;
00774       case MVT::i64:
00775         Opcode = NVPTX::LD_i64_ari;
00776         break;
00777       case MVT::f32:
00778         Opcode = NVPTX::LD_f32_ari;
00779         break;
00780       case MVT::f64:
00781         Opcode = NVPTX::LD_f64_ari;
00782         break;
00783       default:
00784         return nullptr;
00785       }
00786     }
00787     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
00788                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
00789                       getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
00790     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00791   } else {
00792     if (TM.is64Bit()) {
00793       switch (TargetVT) {
00794       case MVT::i8:
00795         Opcode = NVPTX::LD_i8_areg_64;
00796         break;
00797       case MVT::i16:
00798         Opcode = NVPTX::LD_i16_areg_64;
00799         break;
00800       case MVT::i32:
00801         Opcode = NVPTX::LD_i32_areg_64;
00802         break;
00803       case MVT::i64:
00804         Opcode = NVPTX::LD_i64_areg_64;
00805         break;
00806       case MVT::f32:
00807         Opcode = NVPTX::LD_f32_areg_64;
00808         break;
00809       case MVT::f64:
00810         Opcode = NVPTX::LD_f64_areg_64;
00811         break;
00812       default:
00813         return nullptr;
00814       }
00815     } else {
00816       switch (TargetVT) {
00817       case MVT::i8:
00818         Opcode = NVPTX::LD_i8_areg;
00819         break;
00820       case MVT::i16:
00821         Opcode = NVPTX::LD_i16_areg;
00822         break;
00823       case MVT::i32:
00824         Opcode = NVPTX::LD_i32_areg;
00825         break;
00826       case MVT::i64:
00827         Opcode = NVPTX::LD_i64_areg;
00828         break;
00829       case MVT::f32:
00830         Opcode = NVPTX::LD_f32_areg;
00831         break;
00832       case MVT::f64:
00833         Opcode = NVPTX::LD_f64_areg;
00834         break;
00835       default:
00836         return nullptr;
00837       }
00838     }
00839     SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
00840                       getI32Imm(vecType, dl), getI32Imm(fromType, dl),
00841                       getI32Imm(fromTypeWidth, dl), N1, Chain };
00842     NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
00843   }
00844 
00845   if (NVPTXLD) {
00846     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
00847     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
00848     cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
00849   }
00850 
00851   return NVPTXLD;
00852 }
00853 
00854 SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
00855 
00856   SDValue Chain = N->getOperand(0);
00857   SDValue Op1 = N->getOperand(1);
00858   SDValue Addr, Offset, Base;
00859   unsigned Opcode;
00860   SDLoc DL(N);
00861   SDNode *LD;
00862   MemSDNode *MemSD = cast<MemSDNode>(N);
00863   EVT LoadedVT = MemSD->getMemoryVT();
00864 
00865   if (!LoadedVT.isSimple())
00866     return nullptr;
00867 
00868   // Address Space Setting
00869   unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
00870 
00871   // Volatile Setting
00872   // - .volatile is only availalble for .global and .shared
00873   bool IsVolatile = MemSD->isVolatile();
00874   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
00875       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
00876       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
00877     IsVolatile = false;
00878 
00879   // Vector Setting
00880   MVT SimpleVT = LoadedVT.getSimpleVT();
00881 
00882   // Type Setting: fromType + fromTypeWidth
00883   //
00884   // Sign   : ISD::SEXTLOAD
00885   // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
00886   //          type is integer
00887   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
00888   MVT ScalarVT = SimpleVT.getScalarType();
00889   // Read at least 8 bits (predicates are stored as 8-bit values)
00890   unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
00891   unsigned int FromType;
00892   // The last operand holds the original LoadSDNode::getExtensionType() value
00893   unsigned ExtensionType = cast<ConstantSDNode>(
00894       N->getOperand(N->getNumOperands() - 1))->getZExtValue();
00895   if (ExtensionType == ISD::SEXTLOAD)
00896     FromType = NVPTX::PTXLdStInstCode::Signed;
00897   else if (ScalarVT.isFloatingPoint())
00898     FromType = NVPTX::PTXLdStInstCode::Float;
00899   else
00900     FromType = NVPTX::PTXLdStInstCode::Unsigned;
00901 
00902   unsigned VecType;
00903 
00904   switch (N->getOpcode()) {
00905   case NVPTXISD::LoadV2:
00906     VecType = NVPTX::PTXLdStInstCode::V2;
00907     break;
00908   case NVPTXISD::LoadV4:
00909     VecType = NVPTX::PTXLdStInstCode::V4;
00910     break;
00911   default:
00912     return nullptr;
00913   }
00914 
00915   EVT EltVT = N->getValueType(0);
00916 
00917   if (SelectDirectAddr(Op1, Addr)) {
00918     switch (N->getOpcode()) {
00919     default:
00920       return nullptr;
00921     case NVPTXISD::LoadV2:
00922       switch (EltVT.getSimpleVT().SimpleTy) {
00923       default:
00924         return nullptr;
00925       case MVT::i8:
00926         Opcode = NVPTX::LDV_i8_v2_avar;
00927         break;
00928       case MVT::i16:
00929         Opcode = NVPTX::LDV_i16_v2_avar;
00930         break;
00931       case MVT::i32:
00932         Opcode = NVPTX::LDV_i32_v2_avar;
00933         break;
00934       case MVT::i64:
00935         Opcode = NVPTX::LDV_i64_v2_avar;
00936         break;
00937       case MVT::f32:
00938         Opcode = NVPTX::LDV_f32_v2_avar;
00939         break;
00940       case MVT::f64:
00941         Opcode = NVPTX::LDV_f64_v2_avar;
00942         break;
00943       }
00944       break;
00945     case NVPTXISD::LoadV4:
00946       switch (EltVT.getSimpleVT().SimpleTy) {
00947       default:
00948         return nullptr;
00949       case MVT::i8:
00950         Opcode = NVPTX::LDV_i8_v4_avar;
00951         break;
00952       case MVT::i16:
00953         Opcode = NVPTX::LDV_i16_v4_avar;
00954         break;
00955       case MVT::i32:
00956         Opcode = NVPTX::LDV_i32_v4_avar;
00957         break;
00958       case MVT::f32:
00959         Opcode = NVPTX::LDV_f32_v4_avar;
00960         break;
00961       }
00962       break;
00963     }
00964 
00965     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
00966                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
00967                       getI32Imm(FromTypeWidth, DL), Addr, Chain };
00968     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
00969   } else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
00970                           : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
00971     switch (N->getOpcode()) {
00972     default:
00973       return nullptr;
00974     case NVPTXISD::LoadV2:
00975       switch (EltVT.getSimpleVT().SimpleTy) {
00976       default:
00977         return nullptr;
00978       case MVT::i8:
00979         Opcode = NVPTX::LDV_i8_v2_asi;
00980         break;
00981       case MVT::i16:
00982         Opcode = NVPTX::LDV_i16_v2_asi;
00983         break;
00984       case MVT::i32:
00985         Opcode = NVPTX::LDV_i32_v2_asi;
00986         break;
00987       case MVT::i64:
00988         Opcode = NVPTX::LDV_i64_v2_asi;
00989         break;
00990       case MVT::f32:
00991         Opcode = NVPTX::LDV_f32_v2_asi;
00992         break;
00993       case MVT::f64:
00994         Opcode = NVPTX::LDV_f64_v2_asi;
00995         break;
00996       }
00997       break;
00998     case NVPTXISD::LoadV4:
00999       switch (EltVT.getSimpleVT().SimpleTy) {
01000       default:
01001         return nullptr;
01002       case MVT::i8:
01003         Opcode = NVPTX::LDV_i8_v4_asi;
01004         break;
01005       case MVT::i16:
01006         Opcode = NVPTX::LDV_i16_v4_asi;
01007         break;
01008       case MVT::i32:
01009         Opcode = NVPTX::LDV_i32_v4_asi;
01010         break;
01011       case MVT::f32:
01012         Opcode = NVPTX::LDV_f32_v4_asi;
01013         break;
01014       }
01015       break;
01016     }
01017 
01018     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
01019                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
01020                       getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
01021     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01022   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
01023                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
01024     if (TM.is64Bit()) {
01025       switch (N->getOpcode()) {
01026       default:
01027         return nullptr;
01028       case NVPTXISD::LoadV2:
01029         switch (EltVT.getSimpleVT().SimpleTy) {
01030         default:
01031           return nullptr;
01032         case MVT::i8:
01033           Opcode = NVPTX::LDV_i8_v2_ari_64;
01034           break;
01035         case MVT::i16:
01036           Opcode = NVPTX::LDV_i16_v2_ari_64;
01037           break;
01038         case MVT::i32:
01039           Opcode = NVPTX::LDV_i32_v2_ari_64;
01040           break;
01041         case MVT::i64:
01042           Opcode = NVPTX::LDV_i64_v2_ari_64;
01043           break;
01044         case MVT::f32:
01045           Opcode = NVPTX::LDV_f32_v2_ari_64;
01046           break;
01047         case MVT::f64:
01048           Opcode = NVPTX::LDV_f64_v2_ari_64;
01049           break;
01050         }
01051         break;
01052       case NVPTXISD::LoadV4:
01053         switch (EltVT.getSimpleVT().SimpleTy) {
01054         default:
01055           return nullptr;
01056         case MVT::i8:
01057           Opcode = NVPTX::LDV_i8_v4_ari_64;
01058           break;
01059         case MVT::i16:
01060           Opcode = NVPTX::LDV_i16_v4_ari_64;
01061           break;
01062         case MVT::i32:
01063           Opcode = NVPTX::LDV_i32_v4_ari_64;
01064           break;
01065         case MVT::f32:
01066           Opcode = NVPTX::LDV_f32_v4_ari_64;
01067           break;
01068         }
01069         break;
01070       }
01071     } else {
01072       switch (N->getOpcode()) {
01073       default:
01074         return nullptr;
01075       case NVPTXISD::LoadV2:
01076         switch (EltVT.getSimpleVT().SimpleTy) {
01077         default:
01078           return nullptr;
01079         case MVT::i8:
01080           Opcode = NVPTX::LDV_i8_v2_ari;
01081           break;
01082         case MVT::i16:
01083           Opcode = NVPTX::LDV_i16_v2_ari;
01084           break;
01085         case MVT::i32:
01086           Opcode = NVPTX::LDV_i32_v2_ari;
01087           break;
01088         case MVT::i64:
01089           Opcode = NVPTX::LDV_i64_v2_ari;
01090           break;
01091         case MVT::f32:
01092           Opcode = NVPTX::LDV_f32_v2_ari;
01093           break;
01094         case MVT::f64:
01095           Opcode = NVPTX::LDV_f64_v2_ari;
01096           break;
01097         }
01098         break;
01099       case NVPTXISD::LoadV4:
01100         switch (EltVT.getSimpleVT().SimpleTy) {
01101         default:
01102           return nullptr;
01103         case MVT::i8:
01104           Opcode = NVPTX::LDV_i8_v4_ari;
01105           break;
01106         case MVT::i16:
01107           Opcode = NVPTX::LDV_i16_v4_ari;
01108           break;
01109         case MVT::i32:
01110           Opcode = NVPTX::LDV_i32_v4_ari;
01111           break;
01112         case MVT::f32:
01113           Opcode = NVPTX::LDV_f32_v4_ari;
01114           break;
01115         }
01116         break;
01117       }
01118     }
01119 
01120     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
01121                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
01122                       getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
01123 
01124     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01125   } else {
01126     if (TM.is64Bit()) {
01127       switch (N->getOpcode()) {
01128       default:
01129         return nullptr;
01130       case NVPTXISD::LoadV2:
01131         switch (EltVT.getSimpleVT().SimpleTy) {
01132         default:
01133           return nullptr;
01134         case MVT::i8:
01135           Opcode = NVPTX::LDV_i8_v2_areg_64;
01136           break;
01137         case MVT::i16:
01138           Opcode = NVPTX::LDV_i16_v2_areg_64;
01139           break;
01140         case MVT::i32:
01141           Opcode = NVPTX::LDV_i32_v2_areg_64;
01142           break;
01143         case MVT::i64:
01144           Opcode = NVPTX::LDV_i64_v2_areg_64;
01145           break;
01146         case MVT::f32:
01147           Opcode = NVPTX::LDV_f32_v2_areg_64;
01148           break;
01149         case MVT::f64:
01150           Opcode = NVPTX::LDV_f64_v2_areg_64;
01151           break;
01152         }
01153         break;
01154       case NVPTXISD::LoadV4:
01155         switch (EltVT.getSimpleVT().SimpleTy) {
01156         default:
01157           return nullptr;
01158         case MVT::i8:
01159           Opcode = NVPTX::LDV_i8_v4_areg_64;
01160           break;
01161         case MVT::i16:
01162           Opcode = NVPTX::LDV_i16_v4_areg_64;
01163           break;
01164         case MVT::i32:
01165           Opcode = NVPTX::LDV_i32_v4_areg_64;
01166           break;
01167         case MVT::f32:
01168           Opcode = NVPTX::LDV_f32_v4_areg_64;
01169           break;
01170         }
01171         break;
01172       }
01173     } else {
01174       switch (N->getOpcode()) {
01175       default:
01176         return nullptr;
01177       case NVPTXISD::LoadV2:
01178         switch (EltVT.getSimpleVT().SimpleTy) {
01179         default:
01180           return nullptr;
01181         case MVT::i8:
01182           Opcode = NVPTX::LDV_i8_v2_areg;
01183           break;
01184         case MVT::i16:
01185           Opcode = NVPTX::LDV_i16_v2_areg;
01186           break;
01187         case MVT::i32:
01188           Opcode = NVPTX::LDV_i32_v2_areg;
01189           break;
01190         case MVT::i64:
01191           Opcode = NVPTX::LDV_i64_v2_areg;
01192           break;
01193         case MVT::f32:
01194           Opcode = NVPTX::LDV_f32_v2_areg;
01195           break;
01196         case MVT::f64:
01197           Opcode = NVPTX::LDV_f64_v2_areg;
01198           break;
01199         }
01200         break;
01201       case NVPTXISD::LoadV4:
01202         switch (EltVT.getSimpleVT().SimpleTy) {
01203         default:
01204           return nullptr;
01205         case MVT::i8:
01206           Opcode = NVPTX::LDV_i8_v4_areg;
01207           break;
01208         case MVT::i16:
01209           Opcode = NVPTX::LDV_i16_v4_areg;
01210           break;
01211         case MVT::i32:
01212           Opcode = NVPTX::LDV_i32_v4_areg;
01213           break;
01214         case MVT::f32:
01215           Opcode = NVPTX::LDV_f32_v4_areg;
01216           break;
01217         }
01218         break;
01219       }
01220     }
01221 
01222     SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
01223                       getI32Imm(VecType, DL), getI32Imm(FromType, DL),
01224                       getI32Imm(FromTypeWidth, DL), Op1, Chain };
01225     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01226   }
01227 
01228   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01229   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
01230   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
01231 
01232   return LD;
01233 }
01234 
01235 SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
01236 
01237   SDValue Chain = N->getOperand(0);
01238   SDValue Op1;
01239   MemSDNode *Mem;
01240   bool IsLDG = true;
01241 
01242   // If this is an LDG intrinsic, the address is the third operand. Its its an
01243   // LDG/LDU SD node (from custom vector handling), then its the second operand
01244   if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
01245     Op1 = N->getOperand(2);
01246     Mem = cast<MemIntrinsicSDNode>(N);
01247     unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
01248     switch (IID) {
01249     default:
01250       return NULL;
01251     case Intrinsic::nvvm_ldg_global_f:
01252     case Intrinsic::nvvm_ldg_global_i:
01253     case Intrinsic::nvvm_ldg_global_p:
01254       IsLDG = true;
01255       break;
01256     case Intrinsic::nvvm_ldu_global_f:
01257     case Intrinsic::nvvm_ldu_global_i:
01258     case Intrinsic::nvvm_ldu_global_p:
01259       IsLDG = false;
01260       break;
01261     }
01262   } else {
01263     Op1 = N->getOperand(1);
01264     Mem = cast<MemSDNode>(N);
01265   }
01266 
01267   unsigned Opcode;
01268   SDLoc DL(N);
01269   SDNode *LD;
01270   SDValue Base, Offset, Addr;
01271 
01272   EVT EltVT = Mem->getMemoryVT();
01273   if (EltVT.isVector()) {
01274     EltVT = EltVT.getVectorElementType();
01275   }
01276 
01277   if (SelectDirectAddr(Op1, Addr)) {
01278     switch (N->getOpcode()) {
01279     default:
01280       return nullptr;
01281     case ISD::INTRINSIC_W_CHAIN:
01282       if (IsLDG) {
01283         switch (EltVT.getSimpleVT().SimpleTy) {
01284         default:
01285           return nullptr;
01286         case MVT::i8:
01287           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
01288           break;
01289         case MVT::i16:
01290           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
01291           break;
01292         case MVT::i32:
01293           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
01294           break;
01295         case MVT::i64:
01296           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
01297           break;
01298         case MVT::f32:
01299           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
01300           break;
01301         case MVT::f64:
01302           Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
01303           break;
01304         }
01305       } else {
01306         switch (EltVT.getSimpleVT().SimpleTy) {
01307         default:
01308           return nullptr;
01309         case MVT::i8:
01310           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
01311           break;
01312         case MVT::i16:
01313           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
01314           break;
01315         case MVT::i32:
01316           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
01317           break;
01318         case MVT::i64:
01319           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
01320           break;
01321         case MVT::f32:
01322           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
01323           break;
01324         case MVT::f64:
01325           Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
01326           break;
01327         }
01328       }
01329       break;
01330     case NVPTXISD::LDGV2:
01331       switch (EltVT.getSimpleVT().SimpleTy) {
01332       default:
01333         return nullptr;
01334       case MVT::i8:
01335         Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
01336         break;
01337       case MVT::i16:
01338         Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
01339         break;
01340       case MVT::i32:
01341         Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
01342         break;
01343       case MVT::i64:
01344         Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
01345         break;
01346       case MVT::f32:
01347         Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
01348         break;
01349       case MVT::f64:
01350         Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
01351         break;
01352       }
01353       break;
01354     case NVPTXISD::LDUV2:
01355       switch (EltVT.getSimpleVT().SimpleTy) {
01356       default:
01357         return nullptr;
01358       case MVT::i8:
01359         Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
01360         break;
01361       case MVT::i16:
01362         Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
01363         break;
01364       case MVT::i32:
01365         Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
01366         break;
01367       case MVT::i64:
01368         Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
01369         break;
01370       case MVT::f32:
01371         Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
01372         break;
01373       case MVT::f64:
01374         Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
01375         break;
01376       }
01377       break;
01378     case NVPTXISD::LDGV4:
01379       switch (EltVT.getSimpleVT().SimpleTy) {
01380       default:
01381         return nullptr;
01382       case MVT::i8:
01383         Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
01384         break;
01385       case MVT::i16:
01386         Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
01387         break;
01388       case MVT::i32:
01389         Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
01390         break;
01391       case MVT::f32:
01392         Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
01393         break;
01394       }
01395       break;
01396     case NVPTXISD::LDUV4:
01397       switch (EltVT.getSimpleVT().SimpleTy) {
01398       default:
01399         return nullptr;
01400       case MVT::i8:
01401         Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
01402         break;
01403       case MVT::i16:
01404         Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
01405         break;
01406       case MVT::i32:
01407         Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
01408         break;
01409       case MVT::f32:
01410         Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
01411         break;
01412       }
01413       break;
01414     }
01415 
01416     SDValue Ops[] = { Addr, Chain };
01417     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01418   } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
01419                           : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
01420     if (TM.is64Bit()) {
01421       switch (N->getOpcode()) {
01422       default:
01423         return nullptr;
01424       case ISD::INTRINSIC_W_CHAIN:
01425         if (IsLDG) {
01426           switch (EltVT.getSimpleVT().SimpleTy) {
01427           default:
01428             return nullptr;
01429           case MVT::i8:
01430             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
01431             break;
01432           case MVT::i16:
01433             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
01434             break;
01435           case MVT::i32:
01436             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
01437             break;
01438           case MVT::i64:
01439             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
01440             break;
01441           case MVT::f32:
01442             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
01443             break;
01444           case MVT::f64:
01445             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
01446             break;
01447           }
01448         } else {
01449           switch (EltVT.getSimpleVT().SimpleTy) {
01450           default:
01451             return nullptr;
01452           case MVT::i8:
01453             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
01454             break;
01455           case MVT::i16:
01456             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
01457             break;
01458           case MVT::i32:
01459             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
01460             break;
01461           case MVT::i64:
01462             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
01463             break;
01464           case MVT::f32:
01465             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
01466             break;
01467           case MVT::f64:
01468             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
01469             break;
01470           }
01471         }
01472         break;
01473       case NVPTXISD::LDGV2:
01474         switch (EltVT.getSimpleVT().SimpleTy) {
01475         default:
01476           return nullptr;
01477         case MVT::i8:
01478           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
01479           break;
01480         case MVT::i16:
01481           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
01482           break;
01483         case MVT::i32:
01484           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
01485           break;
01486         case MVT::i64:
01487           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
01488           break;
01489         case MVT::f32:
01490           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
01491           break;
01492         case MVT::f64:
01493           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
01494           break;
01495         }
01496         break;
01497       case NVPTXISD::LDUV2:
01498         switch (EltVT.getSimpleVT().SimpleTy) {
01499         default:
01500           return nullptr;
01501         case MVT::i8:
01502           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
01503           break;
01504         case MVT::i16:
01505           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
01506           break;
01507         case MVT::i32:
01508           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
01509           break;
01510         case MVT::i64:
01511           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
01512           break;
01513         case MVT::f32:
01514           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
01515           break;
01516         case MVT::f64:
01517           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
01518           break;
01519         }
01520         break;
01521       case NVPTXISD::LDGV4:
01522         switch (EltVT.getSimpleVT().SimpleTy) {
01523         default:
01524           return nullptr;
01525         case MVT::i8:
01526           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
01527           break;
01528         case MVT::i16:
01529           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
01530           break;
01531         case MVT::i32:
01532           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
01533           break;
01534         case MVT::f32:
01535           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
01536           break;
01537         }
01538         break;
01539       case NVPTXISD::LDUV4:
01540         switch (EltVT.getSimpleVT().SimpleTy) {
01541         default:
01542           return nullptr;
01543         case MVT::i8:
01544           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
01545           break;
01546         case MVT::i16:
01547           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
01548           break;
01549         case MVT::i32:
01550           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
01551           break;
01552         case MVT::f32:
01553           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
01554           break;
01555         }
01556         break;
01557       }
01558     } else {
01559       switch (N->getOpcode()) {
01560       default:
01561         return nullptr;
01562       case ISD::INTRINSIC_W_CHAIN:
01563         if (IsLDG) {
01564           switch (EltVT.getSimpleVT().SimpleTy) {
01565           default:
01566             return nullptr;
01567           case MVT::i8:
01568             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
01569             break;
01570           case MVT::i16:
01571             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
01572             break;
01573           case MVT::i32:
01574             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
01575             break;
01576           case MVT::i64:
01577             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
01578             break;
01579           case MVT::f32:
01580             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
01581             break;
01582           case MVT::f64:
01583             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
01584             break;
01585           }
01586         } else {
01587           switch (EltVT.getSimpleVT().SimpleTy) {
01588           default:
01589             return nullptr;
01590           case MVT::i8:
01591             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
01592             break;
01593           case MVT::i16:
01594             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
01595             break;
01596           case MVT::i32:
01597             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
01598             break;
01599           case MVT::i64:
01600             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
01601             break;
01602           case MVT::f32:
01603             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
01604             break;
01605           case MVT::f64:
01606             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
01607             break;
01608           }
01609         }
01610         break;
01611       case NVPTXISD::LDGV2:
01612         switch (EltVT.getSimpleVT().SimpleTy) {
01613         default:
01614           return nullptr;
01615         case MVT::i8:
01616           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
01617           break;
01618         case MVT::i16:
01619           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
01620           break;
01621         case MVT::i32:
01622           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
01623           break;
01624         case MVT::i64:
01625           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
01626           break;
01627         case MVT::f32:
01628           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
01629           break;
01630         case MVT::f64:
01631           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
01632           break;
01633         }
01634         break;
01635       case NVPTXISD::LDUV2:
01636         switch (EltVT.getSimpleVT().SimpleTy) {
01637         default:
01638           return nullptr;
01639         case MVT::i8:
01640           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
01641           break;
01642         case MVT::i16:
01643           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
01644           break;
01645         case MVT::i32:
01646           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
01647           break;
01648         case MVT::i64:
01649           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
01650           break;
01651         case MVT::f32:
01652           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
01653           break;
01654         case MVT::f64:
01655           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
01656           break;
01657         }
01658         break;
01659       case NVPTXISD::LDGV4:
01660         switch (EltVT.getSimpleVT().SimpleTy) {
01661         default:
01662           return nullptr;
01663         case MVT::i8:
01664           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
01665           break;
01666         case MVT::i16:
01667           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
01668           break;
01669         case MVT::i32:
01670           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
01671           break;
01672         case MVT::f32:
01673           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
01674           break;
01675         }
01676         break;
01677       case NVPTXISD::LDUV4:
01678         switch (EltVT.getSimpleVT().SimpleTy) {
01679         default:
01680           return nullptr;
01681         case MVT::i8:
01682           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
01683           break;
01684         case MVT::i16:
01685           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
01686           break;
01687         case MVT::i32:
01688           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
01689           break;
01690         case MVT::f32:
01691           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
01692           break;
01693         }
01694         break;
01695       }
01696     }
01697 
01698     SDValue Ops[] = { Base, Offset, Chain };
01699 
01700     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01701   } else {
01702     if (TM.is64Bit()) {
01703       switch (N->getOpcode()) {
01704       default:
01705         return nullptr;
01706       case ISD::INTRINSIC_W_CHAIN:
01707         if (IsLDG) {
01708           switch (EltVT.getSimpleVT().SimpleTy) {
01709           default:
01710             return nullptr;
01711           case MVT::i8:
01712             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
01713             break;
01714           case MVT::i16:
01715             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
01716             break;
01717           case MVT::i32:
01718             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
01719             break;
01720           case MVT::i64:
01721             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
01722             break;
01723           case MVT::f32:
01724             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
01725             break;
01726           case MVT::f64:
01727             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
01728             break;
01729           }
01730         } else {
01731           switch (EltVT.getSimpleVT().SimpleTy) {
01732           default:
01733             return nullptr;
01734           case MVT::i8:
01735             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
01736             break;
01737           case MVT::i16:
01738             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
01739             break;
01740           case MVT::i32:
01741             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
01742             break;
01743           case MVT::i64:
01744             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
01745             break;
01746           case MVT::f32:
01747             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
01748             break;
01749           case MVT::f64:
01750             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
01751             break;
01752           }
01753         }
01754         break;
01755       case NVPTXISD::LDGV2:
01756         switch (EltVT.getSimpleVT().SimpleTy) {
01757         default:
01758           return nullptr;
01759         case MVT::i8:
01760           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
01761           break;
01762         case MVT::i16:
01763           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
01764           break;
01765         case MVT::i32:
01766           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
01767           break;
01768         case MVT::i64:
01769           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
01770           break;
01771         case MVT::f32:
01772           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
01773           break;
01774         case MVT::f64:
01775           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
01776           break;
01777         }
01778         break;
01779       case NVPTXISD::LDUV2:
01780         switch (EltVT.getSimpleVT().SimpleTy) {
01781         default:
01782           return nullptr;
01783         case MVT::i8:
01784           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
01785           break;
01786         case MVT::i16:
01787           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
01788           break;
01789         case MVT::i32:
01790           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
01791           break;
01792         case MVT::i64:
01793           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
01794           break;
01795         case MVT::f32:
01796           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
01797           break;
01798         case MVT::f64:
01799           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
01800           break;
01801         }
01802         break;
01803       case NVPTXISD::LDGV4:
01804         switch (EltVT.getSimpleVT().SimpleTy) {
01805         default:
01806           return nullptr;
01807         case MVT::i8:
01808           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
01809           break;
01810         case MVT::i16:
01811           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
01812           break;
01813         case MVT::i32:
01814           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
01815           break;
01816         case MVT::f32:
01817           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
01818           break;
01819         }
01820         break;
01821       case NVPTXISD::LDUV4:
01822         switch (EltVT.getSimpleVT().SimpleTy) {
01823         default:
01824           return nullptr;
01825         case MVT::i8:
01826           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
01827           break;
01828         case MVT::i16:
01829           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
01830           break;
01831         case MVT::i32:
01832           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
01833           break;
01834         case MVT::f32:
01835           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
01836           break;
01837         }
01838         break;
01839       }
01840     } else {
01841       switch (N->getOpcode()) {
01842       default:
01843         return nullptr;
01844       case ISD::INTRINSIC_W_CHAIN:
01845         if (IsLDG) {
01846           switch (EltVT.getSimpleVT().SimpleTy) {
01847           default:
01848             return nullptr;
01849           case MVT::i8:
01850             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
01851             break;
01852           case MVT::i16:
01853             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
01854             break;
01855           case MVT::i32:
01856             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
01857             break;
01858           case MVT::i64:
01859             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
01860             break;
01861           case MVT::f32:
01862             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
01863             break;
01864           case MVT::f64:
01865             Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
01866             break;
01867           }
01868         } else {
01869           switch (EltVT.getSimpleVT().SimpleTy) {
01870           default:
01871             return nullptr;
01872           case MVT::i8:
01873             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
01874             break;
01875           case MVT::i16:
01876             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
01877             break;
01878           case MVT::i32:
01879             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
01880             break;
01881           case MVT::i64:
01882             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
01883             break;
01884           case MVT::f32:
01885             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
01886             break;
01887           case MVT::f64:
01888             Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
01889             break;
01890           }
01891         }
01892         break;
01893       case NVPTXISD::LDGV2:
01894         switch (EltVT.getSimpleVT().SimpleTy) {
01895         default:
01896           return nullptr;
01897         case MVT::i8:
01898           Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
01899           break;
01900         case MVT::i16:
01901           Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
01902           break;
01903         case MVT::i32:
01904           Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
01905           break;
01906         case MVT::i64:
01907           Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
01908           break;
01909         case MVT::f32:
01910           Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
01911           break;
01912         case MVT::f64:
01913           Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
01914           break;
01915         }
01916         break;
01917       case NVPTXISD::LDUV2:
01918         switch (EltVT.getSimpleVT().SimpleTy) {
01919         default:
01920           return nullptr;
01921         case MVT::i8:
01922           Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
01923           break;
01924         case MVT::i16:
01925           Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
01926           break;
01927         case MVT::i32:
01928           Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
01929           break;
01930         case MVT::i64:
01931           Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
01932           break;
01933         case MVT::f32:
01934           Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
01935           break;
01936         case MVT::f64:
01937           Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
01938           break;
01939         }
01940         break;
01941       case NVPTXISD::LDGV4:
01942         switch (EltVT.getSimpleVT().SimpleTy) {
01943         default:
01944           return nullptr;
01945         case MVT::i8:
01946           Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
01947           break;
01948         case MVT::i16:
01949           Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
01950           break;
01951         case MVT::i32:
01952           Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
01953           break;
01954         case MVT::f32:
01955           Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
01956           break;
01957         }
01958         break;
01959       case NVPTXISD::LDUV4:
01960         switch (EltVT.getSimpleVT().SimpleTy) {
01961         default:
01962           return nullptr;
01963         case MVT::i8:
01964           Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
01965           break;
01966         case MVT::i16:
01967           Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
01968           break;
01969         case MVT::i32:
01970           Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
01971           break;
01972         case MVT::f32:
01973           Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
01974           break;
01975         }
01976         break;
01977       }
01978     }
01979 
01980     SDValue Ops[] = { Op1, Chain };
01981     LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
01982   }
01983 
01984   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
01985   MemRefs0[0] = Mem->getMemOperand();
01986   cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
01987 
01988   return LD;
01989 }
01990 
01991 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
01992   SDLoc dl(N);
01993   StoreSDNode *ST = cast<StoreSDNode>(N);
01994   EVT StoreVT = ST->getMemoryVT();
01995   SDNode *NVPTXST = nullptr;
01996 
01997   // do not support pre/post inc/dec
01998   if (ST->isIndexed())
01999     return nullptr;
02000 
02001   if (!StoreVT.isSimple())
02002     return nullptr;
02003 
02004   // Address Space Setting
02005   unsigned int codeAddrSpace = getCodeAddrSpace(ST);
02006 
02007   // Volatile Setting
02008   // - .volatile is only availalble for .global and .shared
02009   bool isVolatile = ST->isVolatile();
02010   if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
02011       codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
02012       codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
02013     isVolatile = false;
02014 
02015   // Vector Setting
02016   MVT SimpleVT = StoreVT.getSimpleVT();
02017   unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
02018   if (SimpleVT.isVector()) {
02019     unsigned num = SimpleVT.getVectorNumElements();
02020     if (num == 2)
02021       vecType = NVPTX::PTXLdStInstCode::V2;
02022     else if (num == 4)
02023       vecType = NVPTX::PTXLdStInstCode::V4;
02024     else
02025       return nullptr;
02026   }
02027 
02028   // Type Setting: toType + toTypeWidth
02029   // - for integer type, always use 'u'
02030   //
02031   MVT ScalarVT = SimpleVT.getScalarType();
02032   unsigned toTypeWidth = ScalarVT.getSizeInBits();
02033   unsigned int toType;
02034   if (ScalarVT.isFloatingPoint())
02035     toType = NVPTX::PTXLdStInstCode::Float;
02036   else
02037     toType = NVPTX::PTXLdStInstCode::Unsigned;
02038 
02039   // Create the machine instruction DAG
02040   SDValue Chain = N->getOperand(0);
02041   SDValue N1 = N->getOperand(1);
02042   SDValue N2 = N->getOperand(2);
02043   SDValue Addr;
02044   SDValue Offset, Base;
02045   unsigned Opcode;
02046   MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
02047 
02048   if (SelectDirectAddr(N2, Addr)) {
02049     switch (SourceVT) {
02050     case MVT::i8:
02051       Opcode = NVPTX::ST_i8_avar;
02052       break;
02053     case MVT::i16:
02054       Opcode = NVPTX::ST_i16_avar;
02055       break;
02056     case MVT::i32:
02057       Opcode = NVPTX::ST_i32_avar;
02058       break;
02059     case MVT::i64:
02060       Opcode = NVPTX::ST_i64_avar;
02061       break;
02062     case MVT::f32:
02063       Opcode = NVPTX::ST_f32_avar;
02064       break;
02065     case MVT::f64:
02066       Opcode = NVPTX::ST_f64_avar;
02067       break;
02068     default:
02069       return nullptr;
02070     }
02071     SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
02072                       getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
02073                       getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Addr,
02074                       Chain };
02075     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02076   } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
02077                           : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
02078     switch (SourceVT) {
02079     case MVT::i8:
02080       Opcode = NVPTX::ST_i8_asi;
02081       break;
02082     case MVT::i16:
02083       Opcode = NVPTX::ST_i16_asi;
02084       break;
02085     case MVT::i32:
02086       Opcode = NVPTX::ST_i32_asi;
02087       break;
02088     case MVT::i64:
02089       Opcode = NVPTX::ST_i64_asi;
02090       break;
02091     case MVT::f32:
02092       Opcode = NVPTX::ST_f32_asi;
02093       break;
02094     case MVT::f64:
02095       Opcode = NVPTX::ST_f64_asi;
02096       break;
02097     default:
02098       return nullptr;
02099     }
02100     SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
02101                       getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
02102                       getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
02103                       Offset, Chain };
02104     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02105   } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
02106                           : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
02107     if (TM.is64Bit()) {
02108       switch (SourceVT) {
02109       case MVT::i8:
02110         Opcode = NVPTX::ST_i8_ari_64;
02111         break;
02112       case MVT::i16:
02113         Opcode = NVPTX::ST_i16_ari_64;
02114         break;
02115       case MVT::i32:
02116         Opcode = NVPTX::ST_i32_ari_64;
02117         break;
02118       case MVT::i64:
02119         Opcode = NVPTX::ST_i64_ari_64;
02120         break;
02121       case MVT::f32:
02122         Opcode = NVPTX::ST_f32_ari_64;
02123         break;
02124       case MVT::f64:
02125         Opcode = NVPTX::ST_f64_ari_64;
02126         break;
02127       default:
02128         return nullptr;
02129       }
02130     } else {
02131       switch (SourceVT) {
02132       case MVT::i8:
02133         Opcode = NVPTX::ST_i8_ari;
02134         break;
02135       case MVT::i16:
02136         Opcode = NVPTX::ST_i16_ari;
02137         break;
02138       case MVT::i32:
02139         Opcode = NVPTX::ST_i32_ari;
02140         break;
02141       case MVT::i64:
02142         Opcode = NVPTX::ST_i64_ari;
02143         break;
02144       case MVT::f32:
02145         Opcode = NVPTX::ST_f32_ari;
02146         break;
02147       case MVT::f64:
02148         Opcode = NVPTX::ST_f64_ari;
02149         break;
02150       default:
02151         return nullptr;
02152       }
02153     }
02154     SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
02155                       getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
02156                       getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
02157                       Offset, Chain };
02158     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02159   } else {
02160     if (TM.is64Bit()) {
02161       switch (SourceVT) {
02162       case MVT::i8:
02163         Opcode = NVPTX::ST_i8_areg_64;
02164         break;
02165       case MVT::i16:
02166         Opcode = NVPTX::ST_i16_areg_64;
02167         break;
02168       case MVT::i32:
02169         Opcode = NVPTX::ST_i32_areg_64;
02170         break;
02171       case MVT::i64:
02172         Opcode = NVPTX::ST_i64_areg_64;
02173         break;
02174       case MVT::f32:
02175         Opcode = NVPTX::ST_f32_areg_64;
02176         break;
02177       case MVT::f64:
02178         Opcode = NVPTX::ST_f64_areg_64;
02179         break;
02180       default:
02181         return nullptr;
02182       }
02183     } else {
02184       switch (SourceVT) {
02185       case MVT::i8:
02186         Opcode = NVPTX::ST_i8_areg;
02187         break;
02188       case MVT::i16:
02189         Opcode = NVPTX::ST_i16_areg;
02190         break;
02191       case MVT::i32:
02192         Opcode = NVPTX::ST_i32_areg;
02193         break;
02194       case MVT::i64:
02195         Opcode = NVPTX::ST_i64_areg;
02196         break;
02197       case MVT::f32:
02198         Opcode = NVPTX::ST_f32_areg;
02199         break;
02200       case MVT::f64:
02201         Opcode = NVPTX::ST_f64_areg;
02202         break;
02203       default:
02204         return nullptr;
02205       }
02206     }
02207     SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
02208                       getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
02209                       getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), N2,
02210                       Chain };
02211     NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
02212   }
02213 
02214   if (NVPTXST) {
02215     MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02216     MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02217     cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
02218   }
02219 
02220   return NVPTXST;
02221 }
02222 
02223 SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
02224   SDValue Chain = N->getOperand(0);
02225   SDValue Op1 = N->getOperand(1);
02226   SDValue Addr, Offset, Base;
02227   unsigned Opcode;
02228   SDLoc DL(N);
02229   SDNode *ST;
02230   EVT EltVT = Op1.getValueType();
02231   MemSDNode *MemSD = cast<MemSDNode>(N);
02232   EVT StoreVT = MemSD->getMemoryVT();
02233 
02234   // Address Space Setting
02235   unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
02236 
02237   if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
02238     report_fatal_error("Cannot store to pointer that points to constant "
02239                        "memory space");
02240   }
02241 
02242   // Volatile Setting
02243   // - .volatile is only availalble for .global and .shared
02244   bool IsVolatile = MemSD->isVolatile();
02245   if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
02246       CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
02247       CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
02248     IsVolatile = false;
02249 
02250   // Type Setting: toType + toTypeWidth
02251   // - for integer type, always use 'u'
02252   assert(StoreVT.isSimple() && "Store value is not simple");
02253   MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
02254   unsigned ToTypeWidth = ScalarVT.getSizeInBits();
02255   unsigned ToType;
02256   if (ScalarVT.isFloatingPoint())
02257     ToType = NVPTX::PTXLdStInstCode::Float;
02258   else
02259     ToType = NVPTX::PTXLdStInstCode::Unsigned;
02260 
02261   SmallVector<SDValue, 12> StOps;
02262   SDValue N2;
02263   unsigned VecType;
02264 
02265   switch (N->getOpcode()) {
02266   case NVPTXISD::StoreV2:
02267     VecType = NVPTX::PTXLdStInstCode::V2;
02268     StOps.push_back(N->getOperand(1));
02269     StOps.push_back(N->getOperand(2));
02270     N2 = N->getOperand(3);
02271     break;
02272   case NVPTXISD::StoreV4:
02273     VecType = NVPTX::PTXLdStInstCode::V4;
02274     StOps.push_back(N->getOperand(1));
02275     StOps.push_back(N->getOperand(2));
02276     StOps.push_back(N->getOperand(3));
02277     StOps.push_back(N->getOperand(4));
02278     N2 = N->getOperand(5);
02279     break;
02280   default:
02281     return nullptr;
02282   }
02283 
02284   StOps.push_back(getI32Imm(IsVolatile, DL));
02285   StOps.push_back(getI32Imm(CodeAddrSpace, DL));
02286   StOps.push_back(getI32Imm(VecType, DL));
02287   StOps.push_back(getI32Imm(ToType, DL));
02288   StOps.push_back(getI32Imm(ToTypeWidth, DL));
02289 
02290   if (SelectDirectAddr(N2, Addr)) {
02291     switch (N->getOpcode()) {
02292     default:
02293       return nullptr;
02294     case NVPTXISD::StoreV2:
02295       switch (EltVT.getSimpleVT().SimpleTy) {
02296       default:
02297         return nullptr;
02298       case MVT::i8:
02299         Opcode = NVPTX::STV_i8_v2_avar;
02300         break;
02301       case MVT::i16:
02302         Opcode = NVPTX::STV_i16_v2_avar;
02303         break;
02304       case MVT::i32:
02305         Opcode = NVPTX::STV_i32_v2_avar;
02306         break;
02307       case MVT::i64:
02308         Opcode = NVPTX::STV_i64_v2_avar;
02309         break;
02310       case MVT::f32:
02311         Opcode = NVPTX::STV_f32_v2_avar;
02312         break;
02313       case MVT::f64:
02314         Opcode = NVPTX::STV_f64_v2_avar;
02315         break;
02316       }
02317       break;
02318     case NVPTXISD::StoreV4:
02319       switch (EltVT.getSimpleVT().SimpleTy) {
02320       default:
02321         return nullptr;
02322       case MVT::i8:
02323         Opcode = NVPTX::STV_i8_v4_avar;
02324         break;
02325       case MVT::i16:
02326         Opcode = NVPTX::STV_i16_v4_avar;
02327         break;
02328       case MVT::i32:
02329         Opcode = NVPTX::STV_i32_v4_avar;
02330         break;
02331       case MVT::f32:
02332         Opcode = NVPTX::STV_f32_v4_avar;
02333         break;
02334       }
02335       break;
02336     }
02337     StOps.push_back(Addr);
02338   } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
02339                           : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
02340     switch (N->getOpcode()) {
02341     default:
02342       return nullptr;
02343     case NVPTXISD::StoreV2:
02344       switch (EltVT.getSimpleVT().SimpleTy) {
02345       default:
02346         return nullptr;
02347       case MVT::i8:
02348         Opcode = NVPTX::STV_i8_v2_asi;
02349         break;
02350       case MVT::i16:
02351         Opcode = NVPTX::STV_i16_v2_asi;
02352         break;
02353       case MVT::i32:
02354         Opcode = NVPTX::STV_i32_v2_asi;
02355         break;
02356       case MVT::i64:
02357         Opcode = NVPTX::STV_i64_v2_asi;
02358         break;
02359       case MVT::f32:
02360         Opcode = NVPTX::STV_f32_v2_asi;
02361         break;
02362       case MVT::f64:
02363         Opcode = NVPTX::STV_f64_v2_asi;
02364         break;
02365       }
02366       break;
02367     case NVPTXISD::StoreV4:
02368       switch (EltVT.getSimpleVT().SimpleTy) {
02369       default:
02370         return nullptr;
02371       case MVT::i8:
02372         Opcode = NVPTX::STV_i8_v4_asi;
02373         break;
02374       case MVT::i16:
02375         Opcode = NVPTX::STV_i16_v4_asi;
02376         break;
02377       case MVT::i32:
02378         Opcode = NVPTX::STV_i32_v4_asi;
02379         break;
02380       case MVT::f32:
02381         Opcode = NVPTX::STV_f32_v4_asi;
02382         break;
02383       }
02384       break;
02385     }
02386     StOps.push_back(Base);
02387     StOps.push_back(Offset);
02388   } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
02389                           : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
02390     if (TM.is64Bit()) {
02391       switch (N->getOpcode()) {
02392       default:
02393         return nullptr;
02394       case NVPTXISD::StoreV2:
02395         switch (EltVT.getSimpleVT().SimpleTy) {
02396         default:
02397           return nullptr;
02398         case MVT::i8:
02399           Opcode = NVPTX::STV_i8_v2_ari_64;
02400           break;
02401         case MVT::i16:
02402           Opcode = NVPTX::STV_i16_v2_ari_64;
02403           break;
02404         case MVT::i32:
02405           Opcode = NVPTX::STV_i32_v2_ari_64;
02406           break;
02407         case MVT::i64:
02408           Opcode = NVPTX::STV_i64_v2_ari_64;
02409           break;
02410         case MVT::f32:
02411           Opcode = NVPTX::STV_f32_v2_ari_64;
02412           break;
02413         case MVT::f64:
02414           Opcode = NVPTX::STV_f64_v2_ari_64;
02415           break;
02416         }
02417         break;
02418       case NVPTXISD::StoreV4:
02419         switch (EltVT.getSimpleVT().SimpleTy) {
02420         default:
02421           return nullptr;
02422         case MVT::i8:
02423           Opcode = NVPTX::STV_i8_v4_ari_64;
02424           break;
02425         case MVT::i16:
02426           Opcode = NVPTX::STV_i16_v4_ari_64;
02427           break;
02428         case MVT::i32:
02429           Opcode = NVPTX::STV_i32_v4_ari_64;
02430           break;
02431         case MVT::f32:
02432           Opcode = NVPTX::STV_f32_v4_ari_64;
02433           break;
02434         }
02435         break;
02436       }
02437     } else {
02438       switch (N->getOpcode()) {
02439       default:
02440         return nullptr;
02441       case NVPTXISD::StoreV2:
02442         switch (EltVT.getSimpleVT().SimpleTy) {
02443         default:
02444           return nullptr;
02445         case MVT::i8:
02446           Opcode = NVPTX::STV_i8_v2_ari;
02447           break;
02448         case MVT::i16:
02449           Opcode = NVPTX::STV_i16_v2_ari;
02450           break;
02451         case MVT::i32:
02452           Opcode = NVPTX::STV_i32_v2_ari;
02453           break;
02454         case MVT::i64:
02455           Opcode = NVPTX::STV_i64_v2_ari;
02456           break;
02457         case MVT::f32:
02458           Opcode = NVPTX::STV_f32_v2_ari;
02459           break;
02460         case MVT::f64:
02461           Opcode = NVPTX::STV_f64_v2_ari;
02462           break;
02463         }
02464         break;
02465       case NVPTXISD::StoreV4:
02466         switch (EltVT.getSimpleVT().SimpleTy) {
02467         default:
02468           return nullptr;
02469         case MVT::i8:
02470           Opcode = NVPTX::STV_i8_v4_ari;
02471           break;
02472         case MVT::i16:
02473           Opcode = NVPTX::STV_i16_v4_ari;
02474           break;
02475         case MVT::i32:
02476           Opcode = NVPTX::STV_i32_v4_ari;
02477           break;
02478         case MVT::f32:
02479           Opcode = NVPTX::STV_f32_v4_ari;
02480           break;
02481         }
02482         break;
02483       }
02484     }
02485     StOps.push_back(Base);
02486     StOps.push_back(Offset);
02487   } else {
02488     if (TM.is64Bit()) {
02489       switch (N->getOpcode()) {
02490       default:
02491         return nullptr;
02492       case NVPTXISD::StoreV2:
02493         switch (EltVT.getSimpleVT().SimpleTy) {
02494         default:
02495           return nullptr;
02496         case MVT::i8:
02497           Opcode = NVPTX::STV_i8_v2_areg_64;
02498           break;
02499         case MVT::i16:
02500           Opcode = NVPTX::STV_i16_v2_areg_64;
02501           break;
02502         case MVT::i32:
02503           Opcode = NVPTX::STV_i32_v2_areg_64;
02504           break;
02505         case MVT::i64:
02506           Opcode = NVPTX::STV_i64_v2_areg_64;
02507           break;
02508         case MVT::f32:
02509           Opcode = NVPTX::STV_f32_v2_areg_64;
02510           break;
02511         case MVT::f64:
02512           Opcode = NVPTX::STV_f64_v2_areg_64;
02513           break;
02514         }
02515         break;
02516       case NVPTXISD::StoreV4:
02517         switch (EltVT.getSimpleVT().SimpleTy) {
02518         default:
02519           return nullptr;
02520         case MVT::i8:
02521           Opcode = NVPTX::STV_i8_v4_areg_64;
02522           break;
02523         case MVT::i16:
02524           Opcode = NVPTX::STV_i16_v4_areg_64;
02525           break;
02526         case MVT::i32:
02527           Opcode = NVPTX::STV_i32_v4_areg_64;
02528           break;
02529         case MVT::f32:
02530           Opcode = NVPTX::STV_f32_v4_areg_64;
02531           break;
02532         }
02533         break;
02534       }
02535     } else {
02536       switch (N->getOpcode()) {
02537       default:
02538         return nullptr;
02539       case NVPTXISD::StoreV2:
02540         switch (EltVT.getSimpleVT().SimpleTy) {
02541         default:
02542           return nullptr;
02543         case MVT::i8:
02544           Opcode = NVPTX::STV_i8_v2_areg;
02545           break;
02546         case MVT::i16:
02547           Opcode = NVPTX::STV_i16_v2_areg;
02548           break;
02549         case MVT::i32:
02550           Opcode = NVPTX::STV_i32_v2_areg;
02551           break;
02552         case MVT::i64:
02553           Opcode = NVPTX::STV_i64_v2_areg;
02554           break;
02555         case MVT::f32:
02556           Opcode = NVPTX::STV_f32_v2_areg;
02557           break;
02558         case MVT::f64:
02559           Opcode = NVPTX::STV_f64_v2_areg;
02560           break;
02561         }
02562         break;
02563       case NVPTXISD::StoreV4:
02564         switch (EltVT.getSimpleVT().SimpleTy) {
02565         default:
02566           return nullptr;
02567         case MVT::i8:
02568           Opcode = NVPTX::STV_i8_v4_areg;
02569           break;
02570         case MVT::i16:
02571           Opcode = NVPTX::STV_i16_v4_areg;
02572           break;
02573         case MVT::i32:
02574           Opcode = NVPTX::STV_i32_v4_areg;
02575           break;
02576         case MVT::f32:
02577           Opcode = NVPTX::STV_f32_v4_areg;
02578           break;
02579         }
02580         break;
02581       }
02582     }
02583     StOps.push_back(N2);
02584   }
02585 
02586   StOps.push_back(Chain);
02587 
02588   ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
02589 
02590   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02591   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02592   cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
02593 
02594   return ST;
02595 }
02596 
02597 SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
02598   SDValue Chain = Node->getOperand(0);
02599   SDValue Offset = Node->getOperand(2);
02600   SDValue Flag = Node->getOperand(3);
02601   SDLoc DL(Node);
02602   MemSDNode *Mem = cast<MemSDNode>(Node);
02603 
02604   unsigned VecSize;
02605   switch (Node->getOpcode()) {
02606   default:
02607     return nullptr;
02608   case NVPTXISD::LoadParam:
02609     VecSize = 1;
02610     break;
02611   case NVPTXISD::LoadParamV2:
02612     VecSize = 2;
02613     break;
02614   case NVPTXISD::LoadParamV4:
02615     VecSize = 4;
02616     break;
02617   }
02618 
02619   EVT EltVT = Node->getValueType(0);
02620   EVT MemVT = Mem->getMemoryVT();
02621 
02622   unsigned Opc = 0;
02623 
02624   switch (VecSize) {
02625   default:
02626     return nullptr;
02627   case 1:
02628     switch (MemVT.getSimpleVT().SimpleTy) {
02629     default:
02630       return nullptr;
02631     case MVT::i1:
02632       Opc = NVPTX::LoadParamMemI8;
02633       break;
02634     case MVT::i8:
02635       Opc = NVPTX::LoadParamMemI8;
02636       break;
02637     case MVT::i16:
02638       Opc = NVPTX::LoadParamMemI16;
02639       break;
02640     case MVT::i32:
02641       Opc = NVPTX::LoadParamMemI32;
02642       break;
02643     case MVT::i64:
02644       Opc = NVPTX::LoadParamMemI64;
02645       break;
02646     case MVT::f32:
02647       Opc = NVPTX::LoadParamMemF32;
02648       break;
02649     case MVT::f64:
02650       Opc = NVPTX::LoadParamMemF64;
02651       break;
02652     }
02653     break;
02654   case 2:
02655     switch (MemVT.getSimpleVT().SimpleTy) {
02656     default:
02657       return nullptr;
02658     case MVT::i1:
02659       Opc = NVPTX::LoadParamMemV2I8;
02660       break;
02661     case MVT::i8:
02662       Opc = NVPTX::LoadParamMemV2I8;
02663       break;
02664     case MVT::i16:
02665       Opc = NVPTX::LoadParamMemV2I16;
02666       break;
02667     case MVT::i32:
02668       Opc = NVPTX::LoadParamMemV2I32;
02669       break;
02670     case MVT::i64:
02671       Opc = NVPTX::LoadParamMemV2I64;
02672       break;
02673     case MVT::f32:
02674       Opc = NVPTX::LoadParamMemV2F32;
02675       break;
02676     case MVT::f64:
02677       Opc = NVPTX::LoadParamMemV2F64;
02678       break;
02679     }
02680     break;
02681   case 4:
02682     switch (MemVT.getSimpleVT().SimpleTy) {
02683     default:
02684       return nullptr;
02685     case MVT::i1:
02686       Opc = NVPTX::LoadParamMemV4I8;
02687       break;
02688     case MVT::i8:
02689       Opc = NVPTX::LoadParamMemV4I8;
02690       break;
02691     case MVT::i16:
02692       Opc = NVPTX::LoadParamMemV4I16;
02693       break;
02694     case MVT::i32:
02695       Opc = NVPTX::LoadParamMemV4I32;
02696       break;
02697     case MVT::f32:
02698       Opc = NVPTX::LoadParamMemV4F32;
02699       break;
02700     }
02701     break;
02702   }
02703 
02704   SDVTList VTs;
02705   if (VecSize == 1) {
02706     VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
02707   } else if (VecSize == 2) {
02708     VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
02709   } else {
02710     EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
02711     VTs = CurDAG->getVTList(EVTs);
02712   }
02713 
02714   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02715 
02716   SmallVector<SDValue, 2> Ops;
02717   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
02718   Ops.push_back(Chain);
02719   Ops.push_back(Flag);
02720 
02721   return CurDAG->getMachineNode(Opc, DL, VTs, Ops);
02722 }
02723 
02724 SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
02725   SDLoc DL(N);
02726   SDValue Chain = N->getOperand(0);
02727   SDValue Offset = N->getOperand(1);
02728   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02729   MemSDNode *Mem = cast<MemSDNode>(N);
02730 
02731   // How many elements do we have?
02732   unsigned NumElts = 1;
02733   switch (N->getOpcode()) {
02734   default:
02735     return nullptr;
02736   case NVPTXISD::StoreRetval:
02737     NumElts = 1;
02738     break;
02739   case NVPTXISD::StoreRetvalV2:
02740     NumElts = 2;
02741     break;
02742   case NVPTXISD::StoreRetvalV4:
02743     NumElts = 4;
02744     break;
02745   }
02746 
02747   // Build vector of operands
02748   SmallVector<SDValue, 6> Ops;
02749   for (unsigned i = 0; i < NumElts; ++i)
02750     Ops.push_back(N->getOperand(i + 2));
02751   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
02752   Ops.push_back(Chain);
02753 
02754   // Determine target opcode
02755   // If we have an i1, use an 8-bit store. The lowering code in
02756   // NVPTXISelLowering will have already emitted an upcast.
02757   unsigned Opcode = 0;
02758   switch (NumElts) {
02759   default:
02760     return nullptr;
02761   case 1:
02762     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02763     default:
02764       return nullptr;
02765     case MVT::i1:
02766       Opcode = NVPTX::StoreRetvalI8;
02767       break;
02768     case MVT::i8:
02769       Opcode = NVPTX::StoreRetvalI8;
02770       break;
02771     case MVT::i16:
02772       Opcode = NVPTX::StoreRetvalI16;
02773       break;
02774     case MVT::i32:
02775       Opcode = NVPTX::StoreRetvalI32;
02776       break;
02777     case MVT::i64:
02778       Opcode = NVPTX::StoreRetvalI64;
02779       break;
02780     case MVT::f32:
02781       Opcode = NVPTX::StoreRetvalF32;
02782       break;
02783     case MVT::f64:
02784       Opcode = NVPTX::StoreRetvalF64;
02785       break;
02786     }
02787     break;
02788   case 2:
02789     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02790     default:
02791       return nullptr;
02792     case MVT::i1:
02793       Opcode = NVPTX::StoreRetvalV2I8;
02794       break;
02795     case MVT::i8:
02796       Opcode = NVPTX::StoreRetvalV2I8;
02797       break;
02798     case MVT::i16:
02799       Opcode = NVPTX::StoreRetvalV2I16;
02800       break;
02801     case MVT::i32:
02802       Opcode = NVPTX::StoreRetvalV2I32;
02803       break;
02804     case MVT::i64:
02805       Opcode = NVPTX::StoreRetvalV2I64;
02806       break;
02807     case MVT::f32:
02808       Opcode = NVPTX::StoreRetvalV2F32;
02809       break;
02810     case MVT::f64:
02811       Opcode = NVPTX::StoreRetvalV2F64;
02812       break;
02813     }
02814     break;
02815   case 4:
02816     switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02817     default:
02818       return nullptr;
02819     case MVT::i1:
02820       Opcode = NVPTX::StoreRetvalV4I8;
02821       break;
02822     case MVT::i8:
02823       Opcode = NVPTX::StoreRetvalV4I8;
02824       break;
02825     case MVT::i16:
02826       Opcode = NVPTX::StoreRetvalV4I16;
02827       break;
02828     case MVT::i32:
02829       Opcode = NVPTX::StoreRetvalV4I32;
02830       break;
02831     case MVT::f32:
02832       Opcode = NVPTX::StoreRetvalV4F32;
02833       break;
02834     }
02835     break;
02836   }
02837 
02838   SDNode *Ret =
02839       CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
02840   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02841   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02842   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
02843 
02844   return Ret;
02845 }
02846 
02847 SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
02848   SDLoc DL(N);
02849   SDValue Chain = N->getOperand(0);
02850   SDValue Param = N->getOperand(1);
02851   unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
02852   SDValue Offset = N->getOperand(2);
02853   unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
02854   MemSDNode *Mem = cast<MemSDNode>(N);
02855   SDValue Flag = N->getOperand(N->getNumOperands() - 1);
02856 
02857   // How many elements do we have?
02858   unsigned NumElts = 1;
02859   switch (N->getOpcode()) {
02860   default:
02861     return nullptr;
02862   case NVPTXISD::StoreParamU32:
02863   case NVPTXISD::StoreParamS32:
02864   case NVPTXISD::StoreParam:
02865     NumElts = 1;
02866     break;
02867   case NVPTXISD::StoreParamV2:
02868     NumElts = 2;
02869     break;
02870   case NVPTXISD::StoreParamV4:
02871     NumElts = 4;
02872     break;
02873   }
02874 
02875   // Build vector of operands
02876   SmallVector<SDValue, 8> Ops;
02877   for (unsigned i = 0; i < NumElts; ++i)
02878     Ops.push_back(N->getOperand(i + 3));
02879   Ops.push_back(CurDAG->getTargetConstant(ParamVal, DL, MVT::i32));
02880   Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
02881   Ops.push_back(Chain);
02882   Ops.push_back(Flag);
02883 
02884   // Determine target opcode
02885   // If we have an i1, use an 8-bit store. The lowering code in
02886   // NVPTXISelLowering will have already emitted an upcast.
02887   unsigned Opcode = 0;
02888   switch (N->getOpcode()) {
02889   default:
02890     switch (NumElts) {
02891     default:
02892       return nullptr;
02893     case 1:
02894       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02895       default:
02896         return nullptr;
02897       case MVT::i1:
02898         Opcode = NVPTX::StoreParamI8;
02899         break;
02900       case MVT::i8:
02901         Opcode = NVPTX::StoreParamI8;
02902         break;
02903       case MVT::i16:
02904         Opcode = NVPTX::StoreParamI16;
02905         break;
02906       case MVT::i32:
02907         Opcode = NVPTX::StoreParamI32;
02908         break;
02909       case MVT::i64:
02910         Opcode = NVPTX::StoreParamI64;
02911         break;
02912       case MVT::f32:
02913         Opcode = NVPTX::StoreParamF32;
02914         break;
02915       case MVT::f64:
02916         Opcode = NVPTX::StoreParamF64;
02917         break;
02918       }
02919       break;
02920     case 2:
02921       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02922       default:
02923         return nullptr;
02924       case MVT::i1:
02925         Opcode = NVPTX::StoreParamV2I8;
02926         break;
02927       case MVT::i8:
02928         Opcode = NVPTX::StoreParamV2I8;
02929         break;
02930       case MVT::i16:
02931         Opcode = NVPTX::StoreParamV2I16;
02932         break;
02933       case MVT::i32:
02934         Opcode = NVPTX::StoreParamV2I32;
02935         break;
02936       case MVT::i64:
02937         Opcode = NVPTX::StoreParamV2I64;
02938         break;
02939       case MVT::f32:
02940         Opcode = NVPTX::StoreParamV2F32;
02941         break;
02942       case MVT::f64:
02943         Opcode = NVPTX::StoreParamV2F64;
02944         break;
02945       }
02946       break;
02947     case 4:
02948       switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
02949       default:
02950         return nullptr;
02951       case MVT::i1:
02952         Opcode = NVPTX::StoreParamV4I8;
02953         break;
02954       case MVT::i8:
02955         Opcode = NVPTX::StoreParamV4I8;
02956         break;
02957       case MVT::i16:
02958         Opcode = NVPTX::StoreParamV4I16;
02959         break;
02960       case MVT::i32:
02961         Opcode = NVPTX::StoreParamV4I32;
02962         break;
02963       case MVT::f32:
02964         Opcode = NVPTX::StoreParamV4F32;
02965         break;
02966       }
02967       break;
02968     }
02969     break;
02970   // Special case: if we have a sign-extend/zero-extend node, insert the
02971   // conversion instruction first, and use that as the value operand to
02972   // the selected StoreParam node.
02973   case NVPTXISD::StoreParamU32: {
02974     Opcode = NVPTX::StoreParamI32;
02975     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
02976                                                 MVT::i32);
02977     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
02978                                          MVT::i32, Ops[0], CvtNone);
02979     Ops[0] = SDValue(Cvt, 0);
02980     break;
02981   }
02982   case NVPTXISD::StoreParamS32: {
02983     Opcode = NVPTX::StoreParamI32;
02984     SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
02985                                                 MVT::i32);
02986     SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
02987                                          MVT::i32, Ops[0], CvtNone);
02988     Ops[0] = SDValue(Cvt, 0);
02989     break;
02990   }
02991   }
02992 
02993   SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
02994   SDNode *Ret =
02995       CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
02996   MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
02997   MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
02998   cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
02999 
03000   return Ret;
03001 }
03002 
03003 SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
03004   SDValue Chain = N->getOperand(0);
03005   SDNode *Ret = nullptr;
03006   unsigned Opc = 0;
03007   SmallVector<SDValue, 8> Ops;
03008 
03009   switch (N->getOpcode()) {
03010   default: return nullptr;
03011   case NVPTXISD::Tex1DFloatS32:
03012     Opc = NVPTX::TEX_1D_F32_S32;
03013     break;
03014   case NVPTXISD::Tex1DFloatFloat:
03015     Opc = NVPTX::TEX_1D_F32_F32;
03016     break;
03017   case NVPTXISD::Tex1DFloatFloatLevel:
03018     Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
03019     break;
03020   case NVPTXISD::Tex1DFloatFloatGrad:
03021     Opc = NVPTX::TEX_1D_F32_F32_GRAD;
03022     break;
03023   case NVPTXISD::Tex1DS32S32:
03024     Opc = NVPTX::TEX_1D_S32_S32;
03025     break;
03026   case NVPTXISD::Tex1DS32Float:
03027     Opc = NVPTX::TEX_1D_S32_F32;
03028     break;
03029   case NVPTXISD::Tex1DS32FloatLevel:
03030     Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
03031     break;
03032   case NVPTXISD::Tex1DS32FloatGrad:
03033     Opc = NVPTX::TEX_1D_S32_F32_GRAD;
03034     break;
03035   case NVPTXISD::Tex1DU32S32:
03036     Opc = NVPTX::TEX_1D_U32_S32;
03037     break;
03038   case NVPTXISD::Tex1DU32Float:
03039     Opc = NVPTX::TEX_1D_U32_F32;
03040     break;
03041   case NVPTXISD::Tex1DU32FloatLevel:
03042     Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
03043     break;
03044   case NVPTXISD::Tex1DU32FloatGrad:
03045     Opc = NVPTX::TEX_1D_U32_F32_GRAD;
03046     break;
03047   case NVPTXISD::Tex1DArrayFloatS32:
03048     Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
03049     break;
03050   case NVPTXISD::Tex1DArrayFloatFloat:
03051     Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
03052     break;
03053   case NVPTXISD::Tex1DArrayFloatFloatLevel:
03054     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
03055     break;
03056   case NVPTXISD::Tex1DArrayFloatFloatGrad:
03057     Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
03058     break;
03059   case NVPTXISD::Tex1DArrayS32S32:
03060     Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
03061     break;
03062   case NVPTXISD::Tex1DArrayS32Float:
03063     Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
03064     break;
03065   case NVPTXISD::Tex1DArrayS32FloatLevel:
03066     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
03067     break;
03068   case NVPTXISD::Tex1DArrayS32FloatGrad:
03069     Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
03070     break;
03071   case NVPTXISD::Tex1DArrayU32S32:
03072     Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
03073     break;
03074   case NVPTXISD::Tex1DArrayU32Float:
03075     Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
03076     break;
03077   case NVPTXISD::Tex1DArrayU32FloatLevel:
03078     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
03079     break;
03080   case NVPTXISD::Tex1DArrayU32FloatGrad:
03081     Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
03082     break;
03083   case NVPTXISD::Tex2DFloatS32:
03084     Opc = NVPTX::TEX_2D_F32_S32;
03085     break;
03086   case NVPTXISD::Tex2DFloatFloat:
03087     Opc = NVPTX::TEX_2D_F32_F32;
03088     break;
03089   case NVPTXISD::Tex2DFloatFloatLevel:
03090     Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
03091     break;
03092   case NVPTXISD::Tex2DFloatFloatGrad:
03093     Opc = NVPTX::TEX_2D_F32_F32_GRAD;
03094     break;
03095   case NVPTXISD::Tex2DS32S32:
03096     Opc = NVPTX::TEX_2D_S32_S32;
03097     break;
03098   case NVPTXISD::Tex2DS32Float:
03099     Opc = NVPTX::TEX_2D_S32_F32;
03100     break;
03101   case NVPTXISD::Tex2DS32FloatLevel:
03102     Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
03103     break;
03104   case NVPTXISD::Tex2DS32FloatGrad:
03105     Opc = NVPTX::TEX_2D_S32_F32_GRAD;
03106     break;
03107   case NVPTXISD::Tex2DU32S32:
03108     Opc = NVPTX::TEX_2D_U32_S32;
03109     break;
03110   case NVPTXISD::Tex2DU32Float:
03111     Opc = NVPTX::TEX_2D_U32_F32;
03112     break;
03113   case NVPTXISD::Tex2DU32FloatLevel:
03114     Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
03115     break;
03116   case NVPTXISD::Tex2DU32FloatGrad:
03117     Opc = NVPTX::TEX_2D_U32_F32_GRAD;
03118     break;
03119   case NVPTXISD::Tex2DArrayFloatS32:
03120     Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
03121     break;
03122   case NVPTXISD::Tex2DArrayFloatFloat:
03123     Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
03124     break;
03125   case NVPTXISD::Tex2DArrayFloatFloatLevel:
03126     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
03127     break;
03128   case NVPTXISD::Tex2DArrayFloatFloatGrad:
03129     Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
03130     break;
03131   case NVPTXISD::Tex2DArrayS32S32:
03132     Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
03133     break;
03134   case NVPTXISD::Tex2DArrayS32Float:
03135     Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
03136     break;
03137   case NVPTXISD::Tex2DArrayS32FloatLevel:
03138     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
03139     break;
03140   case NVPTXISD::Tex2DArrayS32FloatGrad:
03141     Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
03142     break;
03143   case NVPTXISD::Tex2DArrayU32S32:
03144     Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
03145     break;
03146   case NVPTXISD::Tex2DArrayU32Float:
03147     Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
03148     break;
03149   case NVPTXISD::Tex2DArrayU32FloatLevel:
03150     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
03151     break;
03152   case NVPTXISD::Tex2DArrayU32FloatGrad:
03153     Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
03154     break;
03155   case NVPTXISD::Tex3DFloatS32:
03156     Opc = NVPTX::TEX_3D_F32_S32;
03157     break;
03158   case NVPTXISD::Tex3DFloatFloat:
03159     Opc = NVPTX::TEX_3D_F32_F32;
03160     break;
03161   case NVPTXISD::Tex3DFloatFloatLevel:
03162     Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
03163     break;
03164   case NVPTXISD::Tex3DFloatFloatGrad:
03165     Opc = NVPTX::TEX_3D_F32_F32_GRAD;
03166     break;
03167   case NVPTXISD::Tex3DS32S32:
03168     Opc = NVPTX::TEX_3D_S32_S32;
03169     break;
03170   case NVPTXISD::Tex3DS32Float:
03171     Opc = NVPTX::TEX_3D_S32_F32;
03172     break;
03173   case NVPTXISD::Tex3DS32FloatLevel:
03174     Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
03175     break;
03176   case NVPTXISD::Tex3DS32FloatGrad:
03177     Opc = NVPTX::TEX_3D_S32_F32_GRAD;
03178     break;
03179   case NVPTXISD::Tex3DU32S32:
03180     Opc = NVPTX::TEX_3D_U32_S32;
03181     break;
03182   case NVPTXISD::Tex3DU32Float:
03183     Opc = NVPTX::TEX_3D_U32_F32;
03184     break;
03185   case NVPTXISD::Tex3DU32FloatLevel:
03186     Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
03187     break;
03188   case NVPTXISD::Tex3DU32FloatGrad:
03189     Opc = NVPTX::TEX_3D_U32_F32_GRAD;
03190     break;
03191   case NVPTXISD::TexCubeFloatFloat:
03192     Opc = NVPTX::TEX_CUBE_F32_F32;
03193     break;
03194   case NVPTXISD::TexCubeFloatFloatLevel:
03195     Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
03196     break;
03197   case NVPTXISD::TexCubeS32Float:
03198     Opc = NVPTX::TEX_CUBE_S32_F32;
03199     break;
03200   case NVPTXISD::TexCubeS32FloatLevel:
03201     Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
03202     break;
03203   case NVPTXISD::TexCubeU32Float:
03204     Opc = NVPTX::TEX_CUBE_U32_F32;
03205     break;
03206   case NVPTXISD::TexCubeU32FloatLevel:
03207     Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
03208     break;
03209   case NVPTXISD::TexCubeArrayFloatFloat:
03210     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
03211     break;
03212   case NVPTXISD::TexCubeArrayFloatFloatLevel:
03213     Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
03214     break;
03215   case NVPTXISD::TexCubeArrayS32Float:
03216     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
03217     break;
03218   case NVPTXISD::TexCubeArrayS32FloatLevel:
03219     Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
03220     break;
03221   case NVPTXISD::TexCubeArrayU32Float:
03222     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
03223     break;
03224   case NVPTXISD::TexCubeArrayU32FloatLevel:
03225     Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
03226     break;
03227   case NVPTXISD::Tld4R2DFloatFloat:
03228     Opc = NVPTX::TLD4_R_2D_F32_F32;
03229     break;
03230   case NVPTXISD::Tld4G2DFloatFloat:
03231     Opc = NVPTX::TLD4_G_2D_F32_F32;
03232     break;
03233   case NVPTXISD::Tld4B2DFloatFloat:
03234     Opc = NVPTX::TLD4_B_2D_F32_F32;
03235     break;
03236   case NVPTXISD::Tld4A2DFloatFloat:
03237     Opc = NVPTX::TLD4_A_2D_F32_F32;
03238     break;
03239   case NVPTXISD::Tld4R2DS64Float:
03240     Opc = NVPTX::TLD4_R_2D_S32_F32;
03241     break;
03242   case NVPTXISD::Tld4G2DS64Float:
03243     Opc = NVPTX::TLD4_G_2D_S32_F32;
03244     break;
03245   case NVPTXISD::Tld4B2DS64Float:
03246     Opc = NVPTX::TLD4_B_2D_S32_F32;
03247     break;
03248   case NVPTXISD::Tld4A2DS64Float:
03249     Opc = NVPTX::TLD4_A_2D_S32_F32;
03250     break;
03251   case NVPTXISD::Tld4R2DU64Float:
03252     Opc = NVPTX::TLD4_R_2D_U32_F32;
03253     break;
03254   case NVPTXISD::Tld4G2DU64Float:
03255     Opc = NVPTX::TLD4_G_2D_U32_F32;
03256     break;
03257   case NVPTXISD::Tld4B2DU64Float:
03258     Opc = NVPTX::TLD4_B_2D_U32_F32;
03259     break;
03260   case NVPTXISD::Tld4A2DU64Float:
03261     Opc = NVPTX::TLD4_A_2D_U32_F32;
03262     break;
03263   case NVPTXISD::TexUnified1DFloatS32:
03264     Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
03265     break;
03266   case NVPTXISD::TexUnified1DFloatFloat:
03267     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
03268     break;
03269   case NVPTXISD::TexUnified1DFloatFloatLevel:
03270     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
03271     break;
03272   case NVPTXISD::TexUnified1DFloatFloatGrad:
03273     Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
03274     break;
03275   case NVPTXISD::TexUnified1DS32S32:
03276     Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
03277     break;
03278   case NVPTXISD::TexUnified1DS32Float:
03279     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
03280     break;
03281   case NVPTXISD::TexUnified1DS32FloatLevel:
03282     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
03283     break;
03284   case NVPTXISD::TexUnified1DS32FloatGrad:
03285     Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
03286     break;
03287   case NVPTXISD::TexUnified1DU32S32:
03288     Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
03289     break;
03290   case NVPTXISD::TexUnified1DU32Float:
03291     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
03292     break;
03293   case NVPTXISD::TexUnified1DU32FloatLevel:
03294     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
03295     break;
03296   case NVPTXISD::TexUnified1DU32FloatGrad:
03297     Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
03298     break;
03299   case NVPTXISD::TexUnified1DArrayFloatS32:
03300     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
03301     break;
03302   case NVPTXISD::TexUnified1DArrayFloatFloat:
03303     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
03304     break;
03305   case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
03306     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
03307     break;
03308   case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
03309     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
03310     break;
03311   case NVPTXISD::TexUnified1DArrayS32S32:
03312     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
03313     break;
03314   case NVPTXISD::TexUnified1DArrayS32Float:
03315     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
03316     break;
03317   case NVPTXISD::TexUnified1DArrayS32FloatLevel:
03318     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
03319     break;
03320   case NVPTXISD::TexUnified1DArrayS32FloatGrad:
03321     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
03322     break;
03323   case NVPTXISD::TexUnified1DArrayU32S32:
03324     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
03325     break;
03326   case NVPTXISD::TexUnified1DArrayU32Float:
03327     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
03328     break;
03329   case NVPTXISD::TexUnified1DArrayU32FloatLevel:
03330     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
03331     break;
03332   case NVPTXISD::TexUnified1DArrayU32FloatGrad:
03333     Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
03334     break;
03335   case NVPTXISD::TexUnified2DFloatS32:
03336     Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
03337     break;
03338   case NVPTXISD::TexUnified2DFloatFloat:
03339     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
03340     break;
03341   case NVPTXISD::TexUnified2DFloatFloatLevel:
03342     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
03343     break;
03344   case NVPTXISD::TexUnified2DFloatFloatGrad:
03345     Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
03346     break;
03347   case NVPTXISD::TexUnified2DS32S32:
03348     Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
03349     break;
03350   case NVPTXISD::TexUnified2DS32Float:
03351     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
03352     break;
03353   case NVPTXISD::TexUnified2DS32FloatLevel:
03354     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
03355     break;
03356   case NVPTXISD::TexUnified2DS32FloatGrad:
03357     Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
03358     break;
03359   case NVPTXISD::TexUnified2DU32S32:
03360     Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
03361     break;
03362   case NVPTXISD::TexUnified2DU32Float:
03363     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
03364     break;
03365   case NVPTXISD::TexUnified2DU32FloatLevel:
03366     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
03367     break;
03368   case NVPTXISD::TexUnified2DU32FloatGrad:
03369     Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
03370     break;
03371   case NVPTXISD::TexUnified2DArrayFloatS32:
03372     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
03373     break;
03374   case NVPTXISD::TexUnified2DArrayFloatFloat:
03375     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
03376     break;
03377   case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
03378     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
03379     break;
03380   case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
03381     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
03382     break;
03383   case NVPTXISD::TexUnified2DArrayS32S32:
03384     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
03385     break;
03386   case NVPTXISD::TexUnified2DArrayS32Float:
03387     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
03388     break;
03389   case NVPTXISD::TexUnified2DArrayS32FloatLevel:
03390     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
03391     break;
03392   case NVPTXISD::TexUnified2DArrayS32FloatGrad:
03393     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
03394     break;
03395   case NVPTXISD::TexUnified2DArrayU32S32:
03396     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
03397     break;
03398   case NVPTXISD::TexUnified2DArrayU32Float:
03399     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
03400     break;
03401   case NVPTXISD::TexUnified2DArrayU32FloatLevel:
03402     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
03403     break;
03404   case NVPTXISD::TexUnified2DArrayU32FloatGrad:
03405     Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
03406     break;
03407   case NVPTXISD::TexUnified3DFloatS32:
03408     Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
03409     break;
03410   case NVPTXISD::TexUnified3DFloatFloat:
03411     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
03412     break;
03413   case NVPTXISD::TexUnified3DFloatFloatLevel:
03414     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
03415     break;
03416   case NVPTXISD::TexUnified3DFloatFloatGrad:
03417     Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
03418     break;
03419   case NVPTXISD::TexUnified3DS32S32:
03420     Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
03421     break;
03422   case NVPTXISD::TexUnified3DS32Float:
03423     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
03424     break;
03425   case NVPTXISD::TexUnified3DS32FloatLevel:
03426     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
03427     break;
03428   case NVPTXISD::TexUnified3DS32FloatGrad:
03429     Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
03430     break;
03431   case NVPTXISD::TexUnified3DU32S32:
03432     Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
03433     break;
03434   case NVPTXISD::TexUnified3DU32Float:
03435     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
03436     break;
03437   case NVPTXISD::TexUnified3DU32FloatLevel:
03438     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
03439     break;
03440   case NVPTXISD::TexUnified3DU32FloatGrad:
03441     Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
03442     break;
03443   case NVPTXISD::TexUnifiedCubeFloatFloat:
03444     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
03445     break;
03446   case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
03447     Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
03448     break;
03449   case NVPTXISD::TexUnifiedCubeS32Float:
03450     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
03451     break;
03452   case NVPTXISD::TexUnifiedCubeS32FloatLevel:
03453     Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
03454     break;
03455   case NVPTXISD::TexUnifiedCubeU32Float:
03456     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
03457     break;
03458   case NVPTXISD::TexUnifiedCubeU32FloatLevel:
03459     Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
03460     break;
03461   case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
03462     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
03463     break;
03464   case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
03465     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
03466     break;
03467   case NVPTXISD::TexUnifiedCubeArrayS32Float:
03468     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
03469     break;
03470   case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
03471     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
03472     break;
03473   case NVPTXISD::TexUnifiedCubeArrayU32Float:
03474     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
03475     break;
03476   case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
03477     Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
03478     break;
03479   case NVPTXISD::Tld4UnifiedR2DFloatFloat:
03480     Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
03481     break;
03482   case NVPTXISD::Tld4UnifiedG2DFloatFloat:
03483     Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
03484     break;
03485   case NVPTXISD::Tld4UnifiedB2DFloatFloat:
03486     Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
03487     break;
03488   case NVPTXISD::Tld4UnifiedA2DFloatFloat:
03489     Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
03490     break;
03491   case NVPTXISD::Tld4UnifiedR2DS64Float:
03492     Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
03493     break;
03494   case NVPTXISD::Tld4UnifiedG2DS64Float:
03495     Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
03496     break;
03497   case NVPTXISD::Tld4UnifiedB2DS64Float:
03498     Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
03499     break;
03500   case NVPTXISD::Tld4UnifiedA2DS64Float:
03501     Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
03502     break;
03503   case NVPTXISD::Tld4UnifiedR2DU64Float:
03504     Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
03505     break;
03506   case NVPTXISD::Tld4UnifiedG2DU64Float:
03507     Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
03508     break;
03509   case NVPTXISD::Tld4UnifiedB2DU64Float:
03510     Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
03511     break;
03512   case NVPTXISD::Tld4UnifiedA2DU64Float:
03513     Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
03514     break;
03515   }
03516 
03517   // Copy over operands
03518   for (unsigned i = 1; i < N->getNumOperands(); ++i) {
03519     Ops.push_back(N->getOperand(i));
03520   }
03521 
03522   Ops.push_back(Chain);
03523   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
03524   return Ret;
03525 }
03526 
03527 SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
03528   SDValue Chain = N->getOperand(0);
03529   SDValue TexHandle = N->getOperand(1);
03530   SDNode *Ret = nullptr;
03531   unsigned Opc = 0;
03532   SmallVector<SDValue, 8> Ops;
03533   switch (N->getOpcode()) {
03534   default: return nullptr;
03535   case NVPTXISD::Suld1DI8Clamp:
03536     Opc = NVPTX::SULD_1D_I8_CLAMP;
03537     Ops.push_back(TexHandle);
03538     Ops.push_back(N->getOperand(2));
03539     Ops.push_back(Chain);
03540     break;
03541   case NVPTXISD::Suld1DI16Clamp:
03542     Opc = NVPTX::SULD_1D_I16_CLAMP;
03543     Ops.push_back(TexHandle);
03544     Ops.push_back(N->getOperand(2));
03545     Ops.push_back(Chain);
03546     break;
03547   case NVPTXISD::Suld1DI32Clamp:
03548     Opc = NVPTX::SULD_1D_I32_CLAMP;
03549     Ops.push_back(TexHandle);
03550     Ops.push_back(N->getOperand(2));
03551     Ops.push_back(Chain);
03552     break;
03553   case NVPTXISD::Suld1DI64Clamp:
03554     Opc = NVPTX::SULD_1D_I64_CLAMP;
03555     Ops.push_back(TexHandle);
03556     Ops.push_back(N->getOperand(2));
03557     Ops.push_back(Chain);
03558     break;
03559   case NVPTXISD::Suld1DV2I8Clamp:
03560     Opc = NVPTX::SULD_1D_V2I8_CLAMP;
03561     Ops.push_back(TexHandle);
03562     Ops.push_back(N->getOperand(2));
03563     Ops.push_back(Chain);
03564     break;
03565   case NVPTXISD::Suld1DV2I16Clamp:
03566     Opc = NVPTX::SULD_1D_V2I16_CLAMP;
03567     Ops.push_back(TexHandle);
03568     Ops.push_back(N->getOperand(2));
03569     Ops.push_back(Chain);
03570     break;
03571   case NVPTXISD::Suld1DV2I32Clamp:
03572     Opc = NVPTX::SULD_1D_V2I32_CLAMP;
03573     Ops.push_back(TexHandle);
03574     Ops.push_back(N->getOperand(2));
03575     Ops.push_back(Chain);
03576     break;
03577   case NVPTXISD::Suld1DV2I64Clamp:
03578     Opc = NVPTX::SULD_1D_V2I64_CLAMP;
03579     Ops.push_back(TexHandle);
03580     Ops.push_back(N->getOperand(2));
03581     Ops.push_back(Chain);
03582     break;
03583   case NVPTXISD::Suld1DV4I8Clamp:
03584     Opc = NVPTX::SULD_1D_V4I8_CLAMP;
03585     Ops.push_back(TexHandle);
03586     Ops.push_back(N->getOperand(2));
03587     Ops.push_back(Chain);
03588     break;
03589   case NVPTXISD::Suld1DV4I16Clamp:
03590     Opc = NVPTX::SULD_1D_V4I16_CLAMP;
03591     Ops.push_back(TexHandle);
03592     Ops.push_back(N->getOperand(2));
03593     Ops.push_back(Chain);
03594     break;
03595   case NVPTXISD::Suld1DV4I32Clamp:
03596     Opc = NVPTX::SULD_1D_V4I32_CLAMP;
03597     Ops.push_back(TexHandle);
03598     Ops.push_back(N->getOperand(2));
03599     Ops.push_back(Chain);
03600     break;
03601   case NVPTXISD::Suld1DArrayI8Clamp:
03602     Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
03603     Ops.push_back(TexHandle);
03604     Ops.push_back(N->getOperand(2));
03605     Ops.push_back(N->getOperand(3));
03606     Ops.push_back(Chain);
03607     break;
03608   case NVPTXISD::Suld1DArrayI16Clamp:
03609     Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
03610     Ops.push_back(TexHandle);
03611     Ops.push_back(N->getOperand(2));
03612     Ops.push_back(N->getOperand(3));
03613     Ops.push_back(Chain);
03614     break;
03615   case NVPTXISD::Suld1DArrayI32Clamp:
03616     Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
03617     Ops.push_back(TexHandle);
03618     Ops.push_back(N->getOperand(2));
03619     Ops.push_back(N->getOperand(3));
03620     Ops.push_back(Chain);
03621     break;
03622   case NVPTXISD::Suld1DArrayI64Clamp:
03623     Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
03624     Ops.push_back(TexHandle);
03625     Ops.push_back(N->getOperand(2));
03626     Ops.push_back(N->getOperand(3));
03627     Ops.push_back(Chain);
03628     break;
03629   case NVPTXISD::Suld1DArrayV2I8Clamp:
03630     Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
03631     Ops.push_back(TexHandle);
03632     Ops.push_back(N->getOperand(2));
03633     Ops.push_back(N->getOperand(3));
03634     Ops.push_back(Chain);
03635     break;
03636   case NVPTXISD::Suld1DArrayV2I16Clamp:
03637     Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
03638     Ops.push_back(TexHandle);
03639     Ops.push_back(N->getOperand(2));
03640     Ops.push_back(N->getOperand(3));
03641     Ops.push_back(Chain);
03642     break;
03643   case NVPTXISD::Suld1DArrayV2I32Clamp:
03644     Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
03645     Ops.push_back(TexHandle);
03646     Ops.push_back(N->getOperand(2));
03647     Ops.push_back(N->getOperand(3));
03648     Ops.push_back(Chain);
03649     break;
03650   case NVPTXISD::Suld1DArrayV2I64Clamp:
03651     Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
03652     Ops.push_back(TexHandle);
03653     Ops.push_back(N->getOperand(2));
03654     Ops.push_back(N->getOperand(3));
03655     Ops.push_back(Chain);
03656     break;
03657   case NVPTXISD::Suld1DArrayV4I8Clamp:
03658     Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
03659     Ops.push_back(TexHandle);
03660     Ops.push_back(N->getOperand(2));
03661     Ops.push_back(N->getOperand(3));
03662     Ops.push_back(Chain);
03663     break;
03664   case NVPTXISD::Suld1DArrayV4I16Clamp:
03665     Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
03666     Ops.push_back(TexHandle);
03667     Ops.push_back(N->getOperand(2));
03668     Ops.push_back(N->getOperand(3));
03669     Ops.push_back(Chain);
03670     break;
03671   case NVPTXISD::Suld1DArrayV4I32Clamp:
03672     Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
03673     Ops.push_back(TexHandle);
03674     Ops.push_back(N->getOperand(2));
03675     Ops.push_back(N->getOperand(3));
03676     Ops.push_back(Chain);
03677     break;
03678   case NVPTXISD::Suld2DI8Clamp:
03679     Opc = NVPTX::SULD_2D_I8_CLAMP;
03680     Ops.push_back(TexHandle);
03681     Ops.push_back(N->getOperand(2));
03682     Ops.push_back(N->getOperand(3));
03683     Ops.push_back(Chain);
03684     break;
03685   case NVPTXISD::Suld2DI16Clamp:
03686     Opc = NVPTX::SULD_2D_I16_CLAMP;
03687     Ops.push_back(TexHandle);
03688     Ops.push_back(N->getOperand(2));
03689     Ops.push_back(N->getOperand(3));
03690     Ops.push_back(Chain);
03691     break;
03692   case NVPTXISD::Suld2DI32Clamp:
03693     Opc = NVPTX::SULD_2D_I32_CLAMP;
03694     Ops.push_back(TexHandle);
03695     Ops.push_back(N->getOperand(2));
03696     Ops.push_back(N->getOperand(3));
03697     Ops.push_back(Chain);
03698     break;
03699   case NVPTXISD::Suld2DI64Clamp:
03700     Opc = NVPTX::SULD_2D_I64_CLAMP;
03701     Ops.push_back(TexHandle);
03702     Ops.push_back(N->getOperand(2));
03703     Ops.push_back(N->getOperand(3));
03704     Ops.push_back(Chain);
03705     break;
03706   case NVPTXISD::Suld2DV2I8Clamp:
03707     Opc = NVPTX::SULD_2D_V2I8_CLAMP;
03708     Ops.push_back(TexHandle);
03709     Ops.push_back(N->getOperand(2));
03710     Ops.push_back(N->getOperand(3));
03711     Ops.push_back(Chain);
03712     break;
03713   case NVPTXISD::Suld2DV2I16Clamp:
03714     Opc = NVPTX::SULD_2D_V2I16_CLAMP;
03715     Ops.push_back(TexHandle);
03716     Ops.push_back(N->getOperand(2));
03717     Ops.push_back(N->getOperand(3));
03718     Ops.push_back(Chain);
03719     break;
03720   case NVPTXISD::Suld2DV2I32Clamp:
03721     Opc = NVPTX::SULD_2D_V2I32_CLAMP;
03722     Ops.push_back(TexHandle);
03723     Ops.push_back(N->getOperand(2));
03724     Ops.push_back(N->getOperand(3));
03725     Ops.push_back(Chain);
03726     break;
03727   case NVPTXISD::Suld2DV2I64Clamp:
03728     Opc = NVPTX::SULD_2D_V2I64_CLAMP;
03729     Ops.push_back(TexHandle);
03730     Ops.push_back(N->getOperand(2));
03731     Ops.push_back(N->getOperand(3));
03732     Ops.push_back(Chain);
03733     break;
03734   case NVPTXISD::Suld2DV4I8Clamp:
03735     Opc = NVPTX::SULD_2D_V4I8_CLAMP;
03736     Ops.push_back(TexHandle);
03737     Ops.push_back(N->getOperand(2));
03738     Ops.push_back(N->getOperand(3));
03739     Ops.push_back(Chain);
03740     break;
03741   case NVPTXISD::Suld2DV4I16Clamp:
03742     Opc = NVPTX::SULD_2D_V4I16_CLAMP;
03743     Ops.push_back(TexHandle);
03744     Ops.push_back(N->getOperand(2));
03745     Ops.push_back(N->getOperand(3));
03746     Ops.push_back(Chain);
03747     break;
03748   case NVPTXISD::Suld2DV4I32Clamp:
03749     Opc = NVPTX::SULD_2D_V4I32_CLAMP;
03750     Ops.push_back(TexHandle);
03751     Ops.push_back(N->getOperand(2));
03752     Ops.push_back(N->getOperand(3));
03753     Ops.push_back(Chain);
03754     break;
03755   case NVPTXISD::Suld2DArrayI8Clamp:
03756     Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
03757     Ops.push_back(TexHandle);
03758     Ops.push_back(N->getOperand(2));
03759     Ops.push_back(N->getOperand(3));
03760     Ops.push_back(N->getOperand(4));
03761     Ops.push_back(Chain);
03762     break;
03763   case NVPTXISD::Suld2DArrayI16Clamp:
03764     Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
03765     Ops.push_back(TexHandle);
03766     Ops.push_back(N->getOperand(2));
03767     Ops.push_back(N->getOperand(3));
03768     Ops.push_back(N->getOperand(4));
03769     Ops.push_back(Chain);
03770     break;
03771   case NVPTXISD::Suld2DArrayI32Clamp:
03772     Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
03773     Ops.push_back(TexHandle);
03774     Ops.push_back(N->getOperand(2));
03775     Ops.push_back(N->getOperand(3));
03776     Ops.push_back(N->getOperand(4));
03777     Ops.push_back(Chain);
03778     break;
03779   case NVPTXISD::Suld2DArrayI64Clamp:
03780     Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
03781     Ops.push_back(TexHandle);
03782     Ops.push_back(N->getOperand(2));
03783     Ops.push_back(N->getOperand(3));
03784     Ops.push_back(N->getOperand(4));
03785     Ops.push_back(Chain);
03786     break;
03787   case NVPTXISD::Suld2DArrayV2I8Clamp:
03788     Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
03789     Ops.push_back(TexHandle);
03790     Ops.push_back(N->getOperand(2));
03791     Ops.push_back(N->getOperand(3));
03792     Ops.push_back(N->getOperand(4));
03793     Ops.push_back(Chain);
03794     break;
03795   case NVPTXISD::Suld2DArrayV2I16Clamp:
03796     Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
03797     Ops.push_back(TexHandle);
03798     Ops.push_back(N->getOperand(2));
03799     Ops.push_back(N->getOperand(3));
03800     Ops.push_back(N->getOperand(4));
03801     Ops.push_back(Chain);
03802     break;
03803   case NVPTXISD::Suld2DArrayV2I32Clamp:
03804     Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
03805     Ops.push_back(TexHandle);
03806     Ops.push_back(N->getOperand(2));
03807     Ops.push_back(N->getOperand(3));
03808     Ops.push_back(N->getOperand(4));
03809     Ops.push_back(Chain);
03810     break;
03811   case NVPTXISD::Suld2DArrayV2I64Clamp:
03812     Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
03813     Ops.push_back(TexHandle);
03814     Ops.push_back(N->getOperand(2));
03815     Ops.push_back(N->getOperand(3));
03816     Ops.push_back(N->getOperand(4));
03817     Ops.push_back(Chain);
03818     break;
03819   case NVPTXISD::Suld2DArrayV4I8Clamp:
03820     Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
03821     Ops.push_back(TexHandle);
03822     Ops.push_back(N->getOperand(2));
03823     Ops.push_back(N->getOperand(3));
03824     Ops.push_back(N->getOperand(4));
03825     Ops.push_back(Chain);
03826     break;
03827   case NVPTXISD::Suld2DArrayV4I16Clamp:
03828     Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
03829     Ops.push_back(TexHandle);
03830     Ops.push_back(N->getOperand(2));
03831     Ops.push_back(N->getOperand(3));
03832     Ops.push_back(N->getOperand(4));
03833     Ops.push_back(Chain);
03834     break;
03835   case NVPTXISD::Suld2DArrayV4I32Clamp:
03836     Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
03837     Ops.push_back(TexHandle);
03838     Ops.push_back(N->getOperand(2));
03839     Ops.push_back(N->getOperand(3));
03840     Ops.push_back(N->getOperand(4));
03841     Ops.push_back(Chain);
03842     break;
03843   case NVPTXISD::Suld3DI8Clamp:
03844     Opc = NVPTX::SULD_3D_I8_CLAMP;
03845     Ops.push_back(TexHandle);
03846     Ops.push_back(N->getOperand(2));
03847     Ops.push_back(N->getOperand(3));
03848     Ops.push_back(N->getOperand(4));
03849     Ops.push_back(Chain);
03850     break;
03851   case NVPTXISD::Suld3DI16Clamp:
03852     Opc = NVPTX::SULD_3D_I16_CLAMP;
03853     Ops.push_back(TexHandle);
03854     Ops.push_back(N->getOperand(2));
03855     Ops.push_back(N->getOperand(3));
03856     Ops.push_back(N->getOperand(4));
03857     Ops.push_back(Chain);
03858     break;
03859   case NVPTXISD::Suld3DI32Clamp:
03860     Opc = NVPTX::SULD_3D_I32_CLAMP;
03861     Ops.push_back(TexHandle);
03862     Ops.push_back(N->getOperand(2));
03863     Ops.push_back(N->getOperand(3));
03864     Ops.push_back(N->getOperand(4));
03865     Ops.push_back(Chain);
03866     break;
03867   case NVPTXISD::Suld3DI64Clamp:
03868     Opc = NVPTX::SULD_3D_I64_CLAMP;
03869     Ops.push_back(TexHandle);
03870     Ops.push_back(N->getOperand(2));
03871     Ops.push_back(N->getOperand(3));
03872     Ops.push_back(N->getOperand(4));
03873     Ops.push_back(Chain);
03874     break;
03875   case NVPTXISD::Suld3DV2I8Clamp:
03876     Opc = NVPTX::SULD_3D_V2I8_CLAMP;
03877     Ops.push_back(TexHandle);
03878     Ops.push_back(N->getOperand(2));
03879     Ops.push_back(N->getOperand(3));
03880     Ops.push_back(N->getOperand(4));
03881     Ops.push_back(Chain);
03882     break;
03883   case NVPTXISD::Suld3DV2I16Clamp:
03884     Opc = NVPTX::SULD_3D_V2I16_CLAMP;
03885     Ops.push_back(TexHandle);
03886     Ops.push_back(N->getOperand(2));
03887     Ops.push_back(N->getOperand(3));
03888     Ops.push_back(N->getOperand(4));
03889     Ops.push_back(Chain);
03890     break;
03891   case NVPTXISD::Suld3DV2I32Clamp:
03892     Opc = NVPTX::SULD_3D_V2I32_CLAMP;
03893     Ops.push_back(TexHandle);
03894     Ops.push_back(N->getOperand(2));
03895     Ops.push_back(N->getOperand(3));
03896     Ops.push_back(N->getOperand(4));
03897     Ops.push_back(Chain);
03898     break;
03899   case NVPTXISD::Suld3DV2I64Clamp:
03900     Opc = NVPTX::SULD_3D_V2I64_CLAMP;
03901     Ops.push_back(TexHandle);
03902     Ops.push_back(N->getOperand(2));
03903     Ops.push_back(N->getOperand(3));
03904     Ops.push_back(N->getOperand(4));
03905     Ops.push_back(Chain);
03906     break;
03907   case NVPTXISD::Suld3DV4I8Clamp:
03908     Opc = NVPTX::SULD_3D_V4I8_CLAMP;
03909     Ops.push_back(TexHandle);
03910     Ops.push_back(N->getOperand(2));
03911     Ops.push_back(N->getOperand(3));
03912     Ops.push_back(N->getOperand(4));
03913     Ops.push_back(Chain);
03914     break;
03915   case NVPTXISD::Suld3DV4I16Clamp:
03916     Opc = NVPTX::SULD_3D_V4I16_CLAMP;
03917     Ops.push_back(TexHandle);
03918     Ops.push_back(N->getOperand(2));
03919     Ops.push_back(N->getOperand(3));
03920     Ops.push_back(N->getOperand(4));
03921     Ops.push_back(Chain);
03922     break;
03923   case NVPTXISD::Suld3DV4I32Clamp:
03924     Opc = NVPTX::SULD_3D_V4I32_CLAMP;
03925     Ops.push_back(TexHandle);
03926     Ops.push_back(N->getOperand(2));
03927     Ops.push_back(N->getOperand(3));
03928     Ops.push_back(N->getOperand(4));
03929     Ops.push_back(Chain);
03930     break;
03931   case NVPTXISD::Suld1DI8Trap:
03932     Opc = NVPTX::SULD_1D_I8_TRAP;
03933     Ops.push_back(TexHandle);
03934     Ops.push_back(N->getOperand(2));
03935     Ops.push_back(Chain);
03936     break;
03937   case NVPTXISD::Suld1DI16Trap:
03938     Opc = NVPTX::SULD_1D_I16_TRAP;
03939     Ops.push_back(TexHandle);
03940     Ops.push_back(N->getOperand(2));
03941     Ops.push_back(Chain);
03942     break;
03943   case NVPTXISD::Suld1DI32Trap:
03944     Opc = NVPTX::SULD_1D_I32_TRAP;
03945     Ops.push_back(TexHandle);
03946     Ops.push_back(N->getOperand(2));
03947     Ops.push_back(Chain);
03948     break;
03949   case NVPTXISD::Suld1DI64Trap:
03950     Opc = NVPTX::SULD_1D_I64_TRAP;
03951     Ops.push_back(TexHandle);
03952     Ops.push_back(N->getOperand(2));
03953     Ops.push_back(Chain);
03954     break;
03955   case NVPTXISD::Suld1DV2I8Trap:
03956     Opc = NVPTX::SULD_1D_V2I8_TRAP;
03957     Ops.push_back(TexHandle);
03958     Ops.push_back(N->getOperand(2));
03959     Ops.push_back(Chain);
03960     break;
03961   case NVPTXISD::Suld1DV2I16Trap:
03962     Opc = NVPTX::SULD_1D_V2I16_TRAP;
03963     Ops.push_back(TexHandle);
03964     Ops.push_back(N->getOperand(2));
03965     Ops.push_back(Chain);
03966     break;
03967   case NVPTXISD::Suld1DV2I32Trap:
03968     Opc = NVPTX::SULD_1D_V2I32_TRAP;
03969     Ops.push_back(TexHandle);
03970     Ops.push_back(N->getOperand(2));
03971     Ops.push_back(Chain);
03972     break;
03973   case NVPTXISD::Suld1DV2I64Trap:
03974     Opc = NVPTX::SULD_1D_V2I64_TRAP;
03975     Ops.push_back(TexHandle);
03976     Ops.push_back(N->getOperand(2));
03977     Ops.push_back(Chain);
03978     break;
03979   case NVPTXISD::Suld1DV4I8Trap:
03980     Opc = NVPTX::SULD_1D_V4I8_TRAP;
03981     Ops.push_back(TexHandle);
03982     Ops.push_back(N->getOperand(2));
03983     Ops.push_back(Chain);
03984     break;
03985   case NVPTXISD::Suld1DV4I16Trap:
03986     Opc = NVPTX::SULD_1D_V4I16_TRAP;
03987     Ops.push_back(TexHandle);
03988     Ops.push_back(N->getOperand(2));
03989     Ops.push_back(Chain);
03990     break;
03991   case NVPTXISD::Suld1DV4I32Trap:
03992     Opc = NVPTX::SULD_1D_V4I32_TRAP;
03993     Ops.push_back(TexHandle);
03994     Ops.push_back(N->getOperand(2));
03995     Ops.push_back(Chain);
03996     break;
03997   case NVPTXISD::Suld1DArrayI8Trap:
03998     Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
03999     Ops.push_back(TexHandle);
04000     Ops.push_back(N->getOperand(2));
04001     Ops.push_back(N->getOperand(3));
04002     Ops.push_back(Chain);
04003     break;
04004   case NVPTXISD::Suld1DArrayI16Trap:
04005     Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
04006     Ops.push_back(TexHandle);
04007     Ops.push_back(N->getOperand(2));
04008     Ops.push_back(N->getOperand(3));
04009     Ops.push_back(Chain);
04010     break;
04011   case NVPTXISD::Suld1DArrayI32Trap:
04012     Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
04013     Ops.push_back(TexHandle);
04014     Ops.push_back(N->getOperand(2));
04015     Ops.push_back(N->getOperand(3));
04016     Ops.push_back(Chain);
04017     break;
04018   case NVPTXISD::Suld1DArrayI64Trap:
04019     Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
04020     Ops.push_back(TexHandle);
04021     Ops.push_back(N->getOperand(2));
04022     Ops.push_back(N->getOperand(3));
04023     Ops.push_back(Chain);
04024     break;
04025   case NVPTXISD::Suld1DArrayV2I8Trap:
04026     Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
04027     Ops.push_back(TexHandle);
04028     Ops.push_back(N->getOperand(2));
04029     Ops.push_back(N->getOperand(3));
04030     Ops.push_back(Chain);
04031     break;
04032   case NVPTXISD::Suld1DArrayV2I16Trap:
04033     Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
04034     Ops.push_back(TexHandle);
04035     Ops.push_back(N->getOperand(2));
04036     Ops.push_back(N->getOperand(3));
04037     Ops.push_back(Chain);
04038     break;
04039   case NVPTXISD::Suld1DArrayV2I32Trap:
04040     Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
04041     Ops.push_back(TexHandle);
04042     Ops.push_back(N->getOperand(2));
04043     Ops.push_back(N->getOperand(3));
04044     Ops.push_back(Chain);
04045     break;
04046   case NVPTXISD::Suld1DArrayV2I64Trap:
04047     Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
04048     Ops.push_back(TexHandle);
04049     Ops.push_back(N->getOperand(2));
04050     Ops.push_back(N->getOperand(3));
04051     Ops.push_back(Chain);
04052     break;
04053   case NVPTXISD::Suld1DArrayV4I8Trap:
04054     Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
04055     Ops.push_back(TexHandle);
04056     Ops.push_back(N->getOperand(2));
04057     Ops.push_back(N->getOperand(3));
04058     Ops.push_back(Chain);
04059     break;
04060   case NVPTXISD::Suld1DArrayV4I16Trap:
04061     Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
04062     Ops.push_back(TexHandle);
04063     Ops.push_back(N->getOperand(2));
04064     Ops.push_back(N->getOperand(3));
04065     Ops.push_back(Chain);
04066     break;
04067   case NVPTXISD::Suld1DArrayV4I32Trap:
04068     Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
04069     Ops.push_back(TexHandle);
04070     Ops.push_back(N->getOperand(2));
04071     Ops.push_back(N->getOperand(3));
04072     Ops.push_back(Chain);
04073     break;
04074   case NVPTXISD::Suld2DI8Trap:
04075     Opc = NVPTX::SULD_2D_I8_TRAP;
04076     Ops.push_back(TexHandle);
04077     Ops.push_back(N->getOperand(2));
04078     Ops.push_back(N->getOperand(3));
04079     Ops.push_back(Chain);
04080     break;
04081   case NVPTXISD::Suld2DI16Trap:
04082     Opc = NVPTX::SULD_2D_I16_TRAP;
04083     Ops.push_back(TexHandle);
04084     Ops.push_back(N->getOperand(2));
04085     Ops.push_back(N->getOperand(3));
04086     Ops.push_back(Chain);
04087     break;
04088   case NVPTXISD::Suld2DI32Trap:
04089     Opc = NVPTX::SULD_2D_I32_TRAP;
04090     Ops.push_back(TexHandle);
04091     Ops.push_back(N->getOperand(2));
04092     Ops.push_back(N->getOperand(3));
04093     Ops.push_back(Chain);
04094     break;
04095   case NVPTXISD::Suld2DI64Trap:
04096     Opc = NVPTX::SULD_2D_I64_TRAP;
04097     Ops.push_back(TexHandle);
04098     Ops.push_back(N->getOperand(2));
04099     Ops.push_back(N->getOperand(3));
04100     Ops.push_back(Chain);
04101     break;
04102   case NVPTXISD::Suld2DV2I8Trap:
04103     Opc = NVPTX::SULD_2D_V2I8_TRAP;
04104     Ops.push_back(TexHandle);
04105     Ops.push_back(N->getOperand(2));
04106     Ops.push_back(N->getOperand(3));
04107     Ops.push_back(Chain);
04108     break;
04109   case NVPTXISD::Suld2DV2I16Trap:
04110     Opc = NVPTX::SULD_2D_V2I16_TRAP;
04111     Ops.push_back(TexHandle);
04112     Ops.push_back(N->getOperand(2));
04113     Ops.push_back(N->getOperand(3));
04114     Ops.push_back(Chain);
04115     break;
04116   case NVPTXISD::Suld2DV2I32Trap:
04117     Opc = NVPTX::SULD_2D_V2I32_TRAP;
04118     Ops.push_back(TexHandle);
04119     Ops.push_back(N->getOperand(2));
04120     Ops.push_back(N->getOperand(3));
04121     Ops.push_back(Chain);
04122     break;
04123   case NVPTXISD::Suld2DV2I64Trap:
04124     Opc = NVPTX::SULD_2D_V2I64_TRAP;
04125     Ops.push_back(TexHandle);
04126     Ops.push_back(N->getOperand(2));
04127     Ops.push_back(N->getOperand(3));
04128     Ops.push_back(Chain);
04129     break;
04130   case NVPTXISD::Suld2DV4I8Trap:
04131     Opc = NVPTX::SULD_2D_V4I8_TRAP;
04132     Ops.push_back(TexHandle);
04133     Ops.push_back(N->getOperand(2));
04134     Ops.push_back(N->getOperand(3));
04135     Ops.push_back(Chain);
04136     break;
04137   case NVPTXISD::Suld2DV4I16Trap:
04138     Opc = NVPTX::SULD_2D_V4I16_TRAP;
04139     Ops.push_back(TexHandle);
04140     Ops.push_back(N->getOperand(2));
04141     Ops.push_back(N->getOperand(3));
04142     Ops.push_back(Chain);
04143     break;
04144   case NVPTXISD::Suld2DV4I32Trap:
04145     Opc = NVPTX::SULD_2D_V4I32_TRAP;
04146     Ops.push_back(TexHandle);
04147     Ops.push_back(N->getOperand(2));
04148     Ops.push_back(N->getOperand(3));
04149     Ops.push_back(Chain);
04150     break;
04151   case NVPTXISD::Suld2DArrayI8Trap:
04152     Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
04153     Ops.push_back(TexHandle);
04154     Ops.push_back(N->getOperand(2));
04155     Ops.push_back(N->getOperand(3));
04156     Ops.push_back(N->getOperand(4));
04157     Ops.push_back(Chain);
04158     break;
04159   case NVPTXISD::Suld2DArrayI16Trap:
04160     Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
04161     Ops.push_back(TexHandle);
04162     Ops.push_back(N->getOperand(2));
04163     Ops.push_back(N->getOperand(3));
04164     Ops.push_back(N->getOperand(4));
04165     Ops.push_back(Chain);
04166     break;
04167   case NVPTXISD::Suld2DArrayI32Trap:
04168     Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
04169     Ops.push_back(TexHandle);
04170     Ops.push_back(N->getOperand(2));
04171     Ops.push_back(N->getOperand(3));
04172     Ops.push_back(N->getOperand(4));
04173     Ops.push_back(Chain);
04174     break;
04175   case NVPTXISD::Suld2DArrayI64Trap:
04176     Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
04177     Ops.push_back(TexHandle);
04178     Ops.push_back(N->getOperand(2));
04179     Ops.push_back(N->getOperand(3));
04180     Ops.push_back(N->getOperand(4));
04181     Ops.push_back(Chain);
04182     break;
04183   case NVPTXISD::Suld2DArrayV2I8Trap:
04184     Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
04185     Ops.push_back(TexHandle);
04186     Ops.push_back(N->getOperand(2));
04187     Ops.push_back(N->getOperand(3));
04188     Ops.push_back(N->getOperand(4));
04189     Ops.push_back(Chain);
04190     break;
04191   case NVPTXISD::Suld2DArrayV2I16Trap:
04192     Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
04193     Ops.push_back(TexHandle);
04194     Ops.push_back(N->getOperand(2));
04195     Ops.push_back(N->getOperand(3));
04196     Ops.push_back(N->getOperand(4));
04197     Ops.push_back(Chain);
04198     break;
04199   case NVPTXISD::Suld2DArrayV2I32Trap:
04200     Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
04201     Ops.push_back(TexHandle);
04202     Ops.push_back(N->getOperand(2));
04203     Ops.push_back(N->getOperand(3));
04204     Ops.push_back(N->getOperand(4));
04205     Ops.push_back(Chain);
04206     break;
04207   case NVPTXISD::Suld2DArrayV2I64Trap:
04208     Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
04209     Ops.push_back(TexHandle);
04210     Ops.push_back(N->getOperand(2));
04211     Ops.push_back(N->getOperand(3));
04212     Ops.push_back(N->getOperand(4));
04213     Ops.push_back(Chain);
04214     break;
04215   case NVPTXISD::Suld2DArrayV4I8Trap:
04216     Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
04217     Ops.push_back(TexHandle);
04218     Ops.push_back(N->getOperand(2));
04219     Ops.push_back(N->getOperand(3));
04220     Ops.push_back(N->getOperand(4));
04221     Ops.push_back(Chain);
04222     break;
04223   case NVPTXISD::Suld2DArrayV4I16Trap:
04224     Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
04225     Ops.push_back(TexHandle);
04226     Ops.push_back(N->getOperand(2));
04227     Ops.push_back(N->getOperand(3));
04228     Ops.push_back(N->getOperand(4));
04229     Ops.push_back(Chain);
04230     break;
04231   case NVPTXISD::Suld2DArrayV4I32Trap:
04232     Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
04233     Ops.push_back(TexHandle);
04234     Ops.push_back(N->getOperand(2));
04235     Ops.push_back(N->getOperand(3));
04236     Ops.push_back(N->getOperand(4));
04237     Ops.push_back(Chain);
04238     break;
04239   case NVPTXISD::Suld3DI8Trap:
04240     Opc = NVPTX::SULD_3D_I8_TRAP;
04241     Ops.push_back(TexHandle);
04242     Ops.push_back(N->getOperand(2));
04243     Ops.push_back(N->getOperand(3));
04244     Ops.push_back(N->getOperand(4));
04245     Ops.push_back(Chain);
04246     break;
04247   case NVPTXISD::Suld3DI16Trap:
04248     Opc = NVPTX::SULD_3D_I16_TRAP;
04249     Ops.push_back(TexHandle);
04250     Ops.push_back(N->getOperand(2));
04251     Ops.push_back(N->getOperand(3));
04252     Ops.push_back(N->getOperand(4));
04253     Ops.push_back(Chain);
04254     break;
04255   case NVPTXISD::Suld3DI32Trap:
04256     Opc = NVPTX::SULD_3D_I32_TRAP;
04257     Ops.push_back(TexHandle);
04258     Ops.push_back(N->getOperand(2));
04259     Ops.push_back(N->getOperand(3));
04260     Ops.push_back(N->getOperand(4));
04261     Ops.push_back(Chain);
04262     break;
04263   case NVPTXISD::Suld3DI64Trap:
04264     Opc = NVPTX::SULD_3D_I64_TRAP;
04265     Ops.push_back(TexHandle);
04266     Ops.push_back(N->getOperand(2));
04267     Ops.push_back(N->getOperand(3));
04268     Ops.push_back(N->getOperand(4));
04269     Ops.push_back(Chain);
04270     break;
04271   case NVPTXISD::Suld3DV2I8Trap:
04272     Opc = NVPTX::SULD_3D_V2I8_TRAP;
04273     Ops.push_back(TexHandle);
04274     Ops.push_back(N->getOperand(2));
04275     Ops.push_back(N->getOperand(3));
04276     Ops.push_back(N->getOperand(4));
04277     Ops.push_back(Chain);
04278     break;
04279   case NVPTXISD::Suld3DV2I16Trap:
04280     Opc = NVPTX::SULD_3D_V2I16_TRAP;
04281     Ops.push_back(TexHandle);
04282     Ops.push_back(N->getOperand(2));
04283     Ops.push_back(N->getOperand(3));
04284     Ops.push_back(N->getOperand(4));
04285     Ops.push_back(Chain);
04286     break;
04287   case NVPTXISD::Suld3DV2I32Trap:
04288     Opc = NVPTX::SULD_3D_V2I32_TRAP;
04289     Ops.push_back(TexHandle);
04290     Ops.push_back(N->getOperand(2));
04291     Ops.push_back(N->getOperand(3));
04292     Ops.push_back(N->getOperand(4));
04293     Ops.push_back(Chain);
04294     break;
04295   case NVPTXISD::Suld3DV2I64Trap:
04296     Opc = NVPTX::SULD_3D_V2I64_TRAP;
04297     Ops.push_back(TexHandle);
04298     Ops.push_back(N->getOperand(2));
04299     Ops.push_back(N->getOperand(3));
04300     Ops.push_back(N->getOperand(4));
04301     Ops.push_back(Chain);
04302     break;
04303   case NVPTXISD::Suld3DV4I8Trap:
04304     Opc = NVPTX::SULD_3D_V4I8_TRAP;
04305     Ops.push_back(TexHandle);
04306     Ops.push_back(N->getOperand(2));
04307     Ops.push_back(N->getOperand(3));
04308     Ops.push_back(N->getOperand(4));
04309     Ops.push_back(Chain);
04310     break;
04311   case NVPTXISD::Suld3DV4I16Trap:
04312     Opc = NVPTX::SULD_3D_V4I16_TRAP;
04313     Ops.push_back(TexHandle);
04314     Ops.push_back(N->getOperand(2));
04315     Ops.push_back(N->getOperand(3));
04316     Ops.push_back(N->getOperand(4));
04317     Ops.push_back(Chain);
04318     break;
04319   case NVPTXISD::Suld3DV4I32Trap:
04320     Opc = NVPTX::SULD_3D_V4I32_TRAP;
04321     Ops.push_back(TexHandle);
04322     Ops.push_back(N->getOperand(2));
04323     Ops.push_back(N->getOperand(3));
04324     Ops.push_back(N->getOperand(4));
04325     Ops.push_back(Chain);
04326     break;
04327   case NVPTXISD::Suld1DI8Zero:
04328     Opc = NVPTX::SULD_1D_I8_ZERO;
04329     Ops.push_back(TexHandle);
04330     Ops.push_back(N->getOperand(2));
04331     Ops.push_back(Chain);
04332     break;
04333   case NVPTXISD::Suld1DI16Zero:
04334     Opc = NVPTX::SULD_1D_I16_ZERO;
04335     Ops.push_back(TexHandle);
04336     Ops.push_back(N->getOperand(2));
04337     Ops.push_back(Chain);
04338     break;
04339   case NVPTXISD::Suld1DI32Zero:
04340     Opc = NVPTX::SULD_1D_I32_ZERO;
04341     Ops.push_back(TexHandle);
04342     Ops.push_back(N->getOperand(2));
04343     Ops.push_back(Chain);
04344     break;
04345   case NVPTXISD::Suld1DI64Zero:
04346     Opc = NVPTX::SULD_1D_I64_ZERO;
04347     Ops.push_back(TexHandle);
04348     Ops.push_back(N->getOperand(2));
04349     Ops.push_back(Chain);
04350     break;
04351   case NVPTXISD::Suld1DV2I8Zero:
04352     Opc = NVPTX::SULD_1D_V2I8_ZERO;
04353     Ops.push_back(TexHandle);
04354     Ops.push_back(N->getOperand(2));
04355     Ops.push_back(Chain);
04356     break;
04357   case NVPTXISD::Suld1DV2I16Zero:
04358     Opc = NVPTX::SULD_1D_V2I16_ZERO;
04359     Ops.push_back(TexHandle);
04360     Ops.push_back(N->getOperand(2));
04361     Ops.push_back(Chain);
04362     break;
04363   case NVPTXISD::Suld1DV2I32Zero:
04364     Opc = NVPTX::SULD_1D_V2I32_ZERO;
04365     Ops.push_back(TexHandle);
04366     Ops.push_back(N->getOperand(2));
04367     Ops.push_back(Chain);
04368     break;
04369   case NVPTXISD::Suld1DV2I64Zero:
04370     Opc = NVPTX::SULD_1D_V2I64_ZERO;
04371     Ops.push_back(TexHandle);
04372     Ops.push_back(N->getOperand(2));
04373     Ops.push_back(Chain);
04374     break;
04375   case NVPTXISD::Suld1DV4I8Zero:
04376     Opc = NVPTX::SULD_1D_V4I8_ZERO;
04377     Ops.push_back(TexHandle);
04378     Ops.push_back(N->getOperand(2));
04379     Ops.push_back(Chain);
04380     break;
04381   case NVPTXISD::Suld1DV4I16Zero:
04382     Opc = NVPTX::SULD_1D_V4I16_ZERO;
04383     Ops.push_back(TexHandle);
04384     Ops.push_back(N->getOperand(2));
04385     Ops.push_back(Chain);
04386     break;
04387   case NVPTXISD::Suld1DV4I32Zero:
04388     Opc = NVPTX::SULD_1D_V4I32_ZERO;
04389     Ops.push_back(TexHandle);
04390     Ops.push_back(N->getOperand(2));
04391     Ops.push_back(Chain);
04392     break;
04393   case NVPTXISD::Suld1DArrayI8Zero:
04394     Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
04395     Ops.push_back(TexHandle);
04396     Ops.push_back(N->getOperand(2));
04397     Ops.push_back(N->getOperand(3));
04398     Ops.push_back(Chain);
04399     break;
04400   case NVPTXISD::Suld1DArrayI16Zero:
04401     Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
04402     Ops.push_back(TexHandle);
04403     Ops.push_back(N->getOperand(2));
04404     Ops.push_back(N->getOperand(3));
04405     Ops.push_back(Chain);
04406     break;
04407   case NVPTXISD::Suld1DArrayI32Zero:
04408     Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
04409     Ops.push_back(TexHandle);
04410     Ops.push_back(N->getOperand(2));
04411     Ops.push_back(N->getOperand(3));
04412     Ops.push_back(Chain);
04413     break;
04414   case NVPTXISD::Suld1DArrayI64Zero:
04415     Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
04416     Ops.push_back(TexHandle);
04417     Ops.push_back(N->getOperand(2));
04418     Ops.push_back(N->getOperand(3));
04419     Ops.push_back(Chain);
04420     break;
04421   case NVPTXISD::Suld1DArrayV2I8Zero:
04422     Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
04423     Ops.push_back(TexHandle);
04424     Ops.push_back(N->getOperand(2));
04425     Ops.push_back(N->getOperand(3));
04426     Ops.push_back(Chain);
04427     break;
04428   case NVPTXISD::Suld1DArrayV2I16Zero:
04429     Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
04430     Ops.push_back(TexHandle);
04431     Ops.push_back(N->getOperand(2));
04432     Ops.push_back(N->getOperand(3));
04433     Ops.push_back(Chain);
04434     break;
04435   case NVPTXISD::Suld1DArrayV2I32Zero:
04436     Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
04437     Ops.push_back(TexHandle);
04438     Ops.push_back(N->getOperand(2));
04439     Ops.push_back(N->getOperand(3));
04440     Ops.push_back(Chain);
04441     break;
04442   case NVPTXISD::Suld1DArrayV2I64Zero:
04443     Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
04444     Ops.push_back(TexHandle);
04445     Ops.push_back(N->getOperand(2));
04446     Ops.push_back(N->getOperand(3));
04447     Ops.push_back(Chain);
04448     break;
04449   case NVPTXISD::Suld1DArrayV4I8Zero:
04450     Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
04451     Ops.push_back(TexHandle);
04452     Ops.push_back(N->getOperand(2));
04453     Ops.push_back(N->getOperand(3));
04454     Ops.push_back(Chain);
04455     break;
04456   case NVPTXISD::Suld1DArrayV4I16Zero:
04457     Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
04458     Ops.push_back(TexHandle);
04459     Ops.push_back(N->getOperand(2));
04460     Ops.push_back(N->getOperand(3));
04461     Ops.push_back(Chain);
04462     break;
04463   case NVPTXISD::Suld1DArrayV4I32Zero:
04464     Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
04465     Ops.push_back(TexHandle);
04466     Ops.push_back(N->getOperand(2));
04467     Ops.push_back(N->getOperand(3));
04468     Ops.push_back(Chain);
04469     break;
04470   case NVPTXISD::Suld2DI8Zero:
04471     Opc = NVPTX::SULD_2D_I8_ZERO;
04472     Ops.push_back(TexHandle);
04473     Ops.push_back(N->getOperand(2));
04474     Ops.push_back(N->getOperand(3));
04475     Ops.push_back(Chain);
04476     break;
04477   case NVPTXISD::Suld2DI16Zero:
04478     Opc = NVPTX::SULD_2D_I16_ZERO;
04479     Ops.push_back(TexHandle);
04480     Ops.push_back(N->getOperand(2));
04481     Ops.push_back(N->getOperand(3));
04482     Ops.push_back(Chain);
04483     break;
04484   case NVPTXISD::Suld2DI32Zero:
04485     Opc = NVPTX::SULD_2D_I32_ZERO;
04486     Ops.push_back(TexHandle);
04487     Ops.push_back(N->getOperand(2));
04488     Ops.push_back(N->getOperand(3));
04489     Ops.push_back(Chain);
04490     break;
04491   case NVPTXISD::Suld2DI64Zero:
04492     Opc = NVPTX::SULD_2D_I64_ZERO;
04493     Ops.push_back(TexHandle);
04494     Ops.push_back(N->getOperand(2));
04495     Ops.push_back(N->getOperand(3));
04496     Ops.push_back(Chain);
04497     break;
04498   case NVPTXISD::Suld2DV2I8Zero:
04499     Opc = NVPTX::SULD_2D_V2I8_ZERO;
04500     Ops.push_back(TexHandle);
04501     Ops.push_back(N->getOperand(2));
04502     Ops.push_back(N->getOperand(3));
04503     Ops.push_back(Chain);
04504     break;
04505   case NVPTXISD::Suld2DV2I16Zero:
04506     Opc = NVPTX::SULD_2D_V2I16_ZERO;
04507     Ops.push_back(TexHandle);
04508     Ops.push_back(N->getOperand(2));
04509     Ops.push_back(N->getOperand(3));
04510     Ops.push_back(Chain);
04511     break;
04512   case NVPTXISD::Suld2DV2I32Zero:
04513     Opc = NVPTX::SULD_2D_V2I32_ZERO;
04514     Ops.push_back(TexHandle);
04515     Ops.push_back(N->getOperand(2));
04516     Ops.push_back(N->getOperand(3));
04517     Ops.push_back(Chain);
04518     break;
04519   case NVPTXISD::Suld2DV2I64Zero:
04520     Opc = NVPTX::SULD_2D_V2I64_ZERO;
04521     Ops.push_back(TexHandle);
04522     Ops.push_back(N->getOperand(2));
04523     Ops.push_back(N->getOperand(3));
04524     Ops.push_back(Chain);
04525     break;
04526   case NVPTXISD::Suld2DV4I8Zero:
04527     Opc = NVPTX::SULD_2D_V4I8_ZERO;
04528     Ops.push_back(TexHandle);
04529     Ops.push_back(N->getOperand(2));
04530     Ops.push_back(N->getOperand(3));
04531     Ops.push_back(Chain);
04532     break;
04533   case NVPTXISD::Suld2DV4I16Zero:
04534     Opc = NVPTX::SULD_2D_V4I16_ZERO;
04535     Ops.push_back(TexHandle);
04536     Ops.push_back(N->getOperand(2));
04537     Ops.push_back(N->getOperand(3));
04538     Ops.push_back(Chain);
04539     break;
04540   case NVPTXISD::Suld2DV4I32Zero:
04541     Opc = NVPTX::SULD_2D_V4I32_ZERO;
04542     Ops.push_back(TexHandle);
04543     Ops.push_back(N->getOperand(2));
04544     Ops.push_back(N->getOperand(3));
04545     Ops.push_back(Chain);
04546     break;
04547   case NVPTXISD::Suld2DArrayI8Zero:
04548     Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
04549     Ops.push_back(TexHandle);
04550     Ops.push_back(N->getOperand(2));
04551     Ops.push_back(N->getOperand(3));
04552     Ops.push_back(N->getOperand(4));
04553     Ops.push_back(Chain);
04554     break;
04555   case NVPTXISD::Suld2DArrayI16Zero:
04556     Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
04557     Ops.push_back(TexHandle);
04558     Ops.push_back(N->getOperand(2));
04559     Ops.push_back(N->getOperand(3));
04560     Ops.push_back(N->getOperand(4));
04561     Ops.push_back(Chain);
04562     break;
04563   case NVPTXISD::Suld2DArrayI32Zero:
04564     Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
04565     Ops.push_back(TexHandle);
04566     Ops.push_back(N->getOperand(2));
04567     Ops.push_back(N->getOperand(3));
04568     Ops.push_back(N->getOperand(4));
04569     Ops.push_back(Chain);
04570     break;
04571   case NVPTXISD::Suld2DArrayI64Zero:
04572     Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
04573     Ops.push_back(TexHandle);
04574     Ops.push_back(N->getOperand(2));
04575     Ops.push_back(N->getOperand(3));
04576     Ops.push_back(N->getOperand(4));
04577     Ops.push_back(Chain);
04578     break;
04579   case NVPTXISD::Suld2DArrayV2I8Zero:
04580     Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
04581     Ops.push_back(TexHandle);
04582     Ops.push_back(N->getOperand(2));
04583     Ops.push_back(N->getOperand(3));
04584     Ops.push_back(N->getOperand(4));
04585     Ops.push_back(Chain);
04586     break;
04587   case NVPTXISD::Suld2DArrayV2I16Zero:
04588     Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
04589     Ops.push_back(TexHandle);
04590     Ops.push_back(N->getOperand(2));
04591     Ops.push_back(N->getOperand(3));
04592     Ops.push_back(N->getOperand(4));
04593     Ops.push_back(Chain);
04594     break;
04595   case NVPTXISD::Suld2DArrayV2I32Zero:
04596     Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
04597     Ops.push_back(TexHandle);
04598     Ops.push_back(N->getOperand(2));
04599     Ops.push_back(N->getOperand(3));
04600     Ops.push_back(N->getOperand(4));
04601     Ops.push_back(Chain);
04602     break;
04603   case NVPTXISD::Suld2DArrayV2I64Zero:
04604     Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
04605     Ops.push_back(TexHandle);
04606     Ops.push_back(N->getOperand(2));
04607     Ops.push_back(N->getOperand(3));
04608     Ops.push_back(N->getOperand(4));
04609     Ops.push_back(Chain);
04610     break;
04611   case NVPTXISD::Suld2DArrayV4I8Zero:
04612     Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
04613     Ops.push_back(TexHandle);
04614     Ops.push_back(N->getOperand(2));
04615     Ops.push_back(N->getOperand(3));
04616     Ops.push_back(N->getOperand(4));
04617     Ops.push_back(Chain);
04618     break;
04619   case NVPTXISD::Suld2DArrayV4I16Zero:
04620     Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
04621     Ops.push_back(TexHandle);
04622     Ops.push_back(N->getOperand(2));
04623     Ops.push_back(N->getOperand(3));
04624     Ops.push_back(N->getOperand(4));
04625     Ops.push_back(Chain);
04626     break;
04627   case NVPTXISD::Suld2DArrayV4I32Zero:
04628     Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
04629     Ops.push_back(TexHandle);
04630     Ops.push_back(N->getOperand(2));
04631     Ops.push_back(N->getOperand(3));
04632     Ops.push_back(N->getOperand(4));
04633     Ops.push_back(Chain);
04634     break;
04635   case NVPTXISD::Suld3DI8Zero:
04636     Opc = NVPTX::SULD_3D_I8_ZERO;
04637     Ops.push_back(TexHandle);
04638     Ops.push_back(N->getOperand(2));
04639     Ops.push_back(N->getOperand(3));
04640     Ops.push_back(N->getOperand(4));
04641     Ops.push_back(Chain);
04642     break;
04643   case NVPTXISD::Suld3DI16Zero:
04644     Opc = NVPTX::SULD_3D_I16_ZERO;
04645     Ops.push_back(TexHandle);
04646     Ops.push_back(N->getOperand(2));
04647     Ops.push_back(N->getOperand(3));
04648     Ops.push_back(N->getOperand(4));
04649     Ops.push_back(Chain);
04650     break;
04651   case NVPTXISD::Suld3DI32Zero:
04652     Opc = NVPTX::SULD_3D_I32_ZERO;
04653     Ops.push_back(TexHandle);
04654     Ops.push_back(N->getOperand(2));
04655     Ops.push_back(N->getOperand(3));
04656     Ops.push_back(N->getOperand(4));
04657     Ops.push_back(Chain);
04658     break;
04659   case NVPTXISD::Suld3DI64Zero:
04660     Opc = NVPTX::SULD_3D_I64_ZERO;
04661     Ops.push_back(TexHandle);
04662     Ops.push_back(N->getOperand(2));
04663     Ops.push_back(N->getOperand(3));
04664     Ops.push_back(N->getOperand(4));
04665     Ops.push_back(Chain);
04666     break;
04667   case NVPTXISD::Suld3DV2I8Zero:
04668     Opc = NVPTX::SULD_3D_V2I8_ZERO;
04669     Ops.push_back(TexHandle);
04670     Ops.push_back(N->getOperand(2));
04671     Ops.push_back(N->getOperand(3));
04672     Ops.push_back(N->getOperand(4));
04673     Ops.push_back(Chain);
04674     break;
04675   case NVPTXISD::Suld3DV2I16Zero:
04676     Opc = NVPTX::SULD_3D_V2I16_ZERO;
04677     Ops.push_back(TexHandle);
04678     Ops.push_back(N->getOperand(2));
04679     Ops.push_back(N->getOperand(3));
04680     Ops.push_back(N->getOperand(4));
04681     Ops.push_back(Chain);
04682     break;
04683   case NVPTXISD::Suld3DV2I32Zero:
04684     Opc = NVPTX::SULD_3D_V2I32_ZERO;
04685     Ops.push_back(TexHandle);
04686     Ops.push_back(N->getOperand(2));
04687     Ops.push_back(N->getOperand(3));
04688     Ops.push_back(N->getOperand(4));
04689     Ops.push_back(Chain);
04690     break;
04691   case NVPTXISD::Suld3DV2I64Zero:
04692     Opc = NVPTX::SULD_3D_V2I64_ZERO;
04693     Ops.push_back(TexHandle);
04694     Ops.push_back(N->getOperand(2));
04695     Ops.push_back(N->getOperand(3));
04696     Ops.push_back(N->getOperand(4));
04697     Ops.push_back(Chain);
04698     break;
04699   case NVPTXISD::Suld3DV4I8Zero:
04700     Opc = NVPTX::SULD_3D_V4I8_ZERO;
04701     Ops.push_back(TexHandle);
04702     Ops.push_back(N->getOperand(2));
04703     Ops.push_back(N->getOperand(3));
04704     Ops.push_back(N->getOperand(4));
04705     Ops.push_back(Chain);
04706     break;
04707   case NVPTXISD::Suld3DV4I16Zero:
04708     Opc = NVPTX::SULD_3D_V4I16_ZERO;
04709     Ops.push_back(TexHandle);
04710     Ops.push_back(N->getOperand(2));
04711     Ops.push_back(N->getOperand(3));
04712     Ops.push_back(N->getOperand(4));
04713     Ops.push_back(Chain);
04714     break;
04715   case NVPTXISD::Suld3DV4I32Zero:
04716     Opc = NVPTX::SULD_3D_V4I32_ZERO;
04717     Ops.push_back(TexHandle);
04718     Ops.push_back(N->getOperand(2));
04719     Ops.push_back(N->getOperand(3));
04720     Ops.push_back(N->getOperand(4));
04721     Ops.push_back(Chain);
04722     break;
04723   }
04724   Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
04725   return Ret;
04726 }
04727 
04728 
04729 /// SelectBFE - Look for instruction sequences that can be made more efficient
04730 /// by using the 'bfe' (bit-field extract) PTX instruction
04731 SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
04732   SDLoc DL(N);
04733   SDValue LHS = N->getOperand(0);
04734   SDValue RHS = N->getOperand(1);
04735   SDValue Len;
04736   SDValue Start;
04737   SDValue Val;
04738   bool IsSigned = false;
04739 
04740   if (N->getOpcode() == ISD::AND) {
04741     // Canonicalize the operands
04742     // We want 'and %val, %mask'
04743     if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
04744       std::swap(LHS, RHS);
04745     }
04746 
04747     ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
04748     if (!Mask) {
04749       // We need a constant mask on the RHS of the AND
04750       return NULL;
04751     }
04752 
04753     // Extract the mask bits
04754     uint64_t MaskVal = Mask->getZExtValue();
04755     if (!isMask_64(MaskVal)) {
04756       // We *could* handle shifted masks here, but doing so would require an
04757       // 'and' operation to fix up the low-order bits so we would trade
04758       // shr+and for bfe+and, which has the same throughput
04759       return NULL;
04760     }
04761 
04762     // How many bits are in our mask?
04763     uint64_t NumBits = countTrailingOnes(MaskVal);
04764     Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
04765 
04766     if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
04767       // We have a 'srl/and' pair, extract the effective start bit and length
04768       Val = LHS.getNode()->getOperand(0);
04769       Start = LHS.getNode()->getOperand(1);
04770       ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
04771       if (StartConst) {
04772         uint64_t StartVal = StartConst->getZExtValue();
04773         // How many "good" bits do we have left?  "good" is defined here as bits
04774         // that exist in the original value, not shifted in.
04775         uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
04776         if (NumBits > GoodBits) {
04777           // Do not handle the case where bits have been shifted in. In theory
04778           // we could handle this, but the cost is likely higher than just
04779           // emitting the srl/and pair.
04780           return NULL;
04781         }
04782         Start = CurDAG->getTargetConstant(StartVal, DL, MVT::i32);
04783       } else {
04784         // Do not handle the case where the shift amount (can be zero if no srl
04785         // was found) is not constant. We could handle this case, but it would
04786         // require run-time logic that would be more expensive than just
04787         // emitting the srl/and pair.
04788         return NULL;
04789       }
04790     } else {
04791       // Do not handle the case where the LHS of the and is not a shift. While
04792       // it would be trivial to handle this case, it would just transform
04793       // 'and' -> 'bfe', but 'and' has higher-throughput.
04794       return NULL;
04795     }
04796   } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
04797     if (LHS->getOpcode() == ISD::AND) {
04798       ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
04799       if (!ShiftCnst) {
04800         // Shift amount must be constant
04801         return NULL;
04802       }
04803 
04804       uint64_t ShiftAmt = ShiftCnst->getZExtValue();
04805 
04806       SDValue AndLHS = LHS->getOperand(0);
04807       SDValue AndRHS = LHS->getOperand(1);
04808 
04809       // Canonicalize the AND to have the mask on the RHS
04810       if (isa<ConstantSDNode>(AndLHS)) {
04811         std::swap(AndLHS, AndRHS);
04812       }
04813 
04814       ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
04815       if (!MaskCnst) {
04816         // Mask must be constant
04817         return NULL;
04818       }
04819 
04820       uint64_t MaskVal = MaskCnst->getZExtValue();
04821       uint64_t NumZeros;
04822       uint64_t NumBits;
04823       if (isMask_64(MaskVal)) {
04824         NumZeros = 0;
04825         // The number of bits in the result bitfield will be the number of
04826         // trailing ones (the AND) minus the number of bits we shift off
04827         NumBits = countTrailingOnes(MaskVal) - ShiftAmt;
04828       } else if (isShiftedMask_64(MaskVal)) {
04829         NumZeros = countTrailingZeros(MaskVal);
04830         unsigned NumOnes = countTrailingOnes(MaskVal >> NumZeros);
04831         // The number of bits in the result bitfield will be the number of
04832         // trailing zeros plus the number of set bits in the mask minus the
04833         // number of bits we shift off
04834         NumBits = NumZeros + NumOnes - ShiftAmt;
04835       } else {
04836         // This is not a mask we can handle
04837         return NULL;
04838       }
04839 
04840       if (ShiftAmt < NumZeros) {
04841         // Handling this case would require extra logic that would make this
04842         // transformation non-profitable
04843         return NULL;
04844       }
04845 
04846       Val = AndLHS;
04847       Start = CurDAG->getTargetConstant(ShiftAmt, DL, MVT::i32);
04848       Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
04849     } else if (LHS->getOpcode() == ISD::SHL) {
04850       // Here, we have a pattern like:
04851       //
04852       // (sra (shl val, NN), MM)
04853       // or
04854       // (srl (shl val, NN), MM)
04855       //
04856       // If MM >= NN, we can efficiently optimize this with bfe
04857       Val = LHS->getOperand(0);
04858 
04859       SDValue ShlRHS = LHS->getOperand(1);
04860       ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
04861       if (!ShlCnst) {
04862         // Shift amount must be constant
04863         return NULL;
04864       }
04865       uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
04866 
04867       SDValue ShrRHS = RHS;
04868       ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
04869       if (!ShrCnst) {
04870         // Shift amount must be constant
04871         return NULL;
04872       }
04873       uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
04874 
04875       // To avoid extra codegen and be profitable, we need Outer >= Inner
04876       if (OuterShiftAmt < InnerShiftAmt) {
04877         return NULL;
04878       }
04879 
04880       // If the outer shift is more than the type size, we have no bitfield to
04881       // extract (since we also check that the inner shift is <= the outer shift
04882       // then this also implies that the inner shift is < the type size)
04883       if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
04884         return NULL;
04885       }
04886 
04887       Start =
04888         CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, DL, MVT::i32);
04889       Len =
04890         CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
04891                                   OuterShiftAmt, DL, MVT::i32);
04892 
04893       if (N->getOpcode() == ISD::SRA) {
04894         // If we have a arithmetic right shift, we need to use the signed bfe
04895         // variant
04896         IsSigned = true;
04897       }
04898     } else {
04899       // No can do...
04900       return NULL;
04901     }
04902   } else {
04903     // No can do...
04904     return NULL;
04905   }
04906 
04907 
04908   unsigned Opc;
04909   // For the BFE operations we form here from "and" and "srl", always use the
04910   // unsigned variants.
04911   if (Val.getValueType() == MVT::i32) {
04912     if (IsSigned) {
04913       Opc = NVPTX::BFE_S32rii;
04914     } else {
04915       Opc = NVPTX::BFE_U32rii;
04916     }
04917   } else if (Val.getValueType() == MVT::i64) {
04918     if (IsSigned) {
04919       Opc = NVPTX::BFE_S64rii;
04920     } else {
04921       Opc = NVPTX::BFE_U64rii;
04922     }
04923   } else {
04924     // We cannot handle this type
04925     return NULL;
04926   }
04927 
04928   SDValue Ops[] = {
04929     Val, Start, Len
04930   };
04931 
04932   return CurDAG->getMachineNode(Opc, DL, N->getVTList(), Ops);
04933 }
04934 
04935 // SelectDirectAddr - Match a direct address for DAG.
04936 // A direct address could be a globaladdress or externalsymbol.
04937 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
04938   // Return true if TGA or ES.
04939   if (N.getOpcode() == ISD::TargetGlobalAddress ||
04940       N.getOpcode() == ISD::TargetExternalSymbol) {
04941     Address = N;
04942     return true;
04943   }
04944   if (N.getOpcode() == NVPTXISD::Wrapper) {
04945     Address = N.getOperand(0);
04946     return true;
04947   }
04948   if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
04949     unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
04950     if (IID == Intrinsic::nvvm_ptr_gen_to_param)
04951       if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
04952         return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
04953   }
04954   return false;
04955 }
04956 
04957 // symbol+offset
04958 bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
04959     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
04960   if (Addr.getOpcode() == ISD::ADD) {
04961     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
04962       SDValue base = Addr.getOperand(0);
04963       if (SelectDirectAddr(base, Base)) {
04964         Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
04965                                            mvt);
04966         return true;
04967       }
04968     }
04969   }
04970   return false;
04971 }
04972 
04973 // symbol+offset
04974 bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
04975                                      SDValue &Base, SDValue &Offset) {
04976   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
04977 }
04978 
04979 // symbol+offset
04980 bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
04981                                        SDValue &Base, SDValue &Offset) {
04982   return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
04983 }
04984 
04985 // register+offset
04986 bool NVPTXDAGToDAGISel::SelectADDRri_imp(
04987     SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
04988   if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
04989     Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
04990     Offset = CurDAG->getTargetConstant(0, SDLoc(OpNode), mvt);
04991     return true;
04992   }
04993   if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
04994       Addr.getOpcode() == ISD::TargetGlobalAddress)
04995     return false; // direct calls.
04996 
04997   if (Addr.getOpcode() == ISD::ADD) {
04998     if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
04999       return false;
05000     }
05001     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
05002       if (FrameIndexSDNode *FIN =
05003               dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
05004         // Constant offset from frame ref.
05005         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
05006       else
05007         Base = Addr.getOperand(0);
05008       Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
05009                                          mvt);
05010       return true;
05011     }
05012   }
05013   return false;
05014 }
05015 
05016 // register+offset
05017 bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
05018                                      SDValue &Base, SDValue &Offset) {
05019   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
05020 }
05021 
05022 // register+offset
05023 bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
05024                                        SDValue &Base, SDValue &Offset) {
05025   return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
05026 }
05027 
05028 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
05029                                                  unsigned int spN) const {
05030   const Value *Src = nullptr;
05031   if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
05032     if (spN == 0 && mN->getMemOperand()->getPseudoValue())
05033       return true;
05034     Src = mN->getMemOperand()->getValue();
05035   }
05036   if (!Src)
05037     return false;
05038   if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
05039     return (PT->getAddressSpace() == spN);
05040   return false;
05041 }
05042 
05043 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
05044 /// inline asm expressions.
05045 bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
05046     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
05047   SDValue Op0, Op1;
05048   switch (ConstraintID) {
05049   default:
05050     return true;
05051   case InlineAsm::Constraint_m: // memory
05052     if (SelectDirectAddr(Op, Op0)) {
05053       OutOps.push_back(Op0);
05054       OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
05055       return false;
05056     }
05057     if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
05058       OutOps.push_back(Op0);
05059       OutOps.push_back(Op1);
05060       return false;
05061     }
05062     break;
05063   }
05064   return true;
05065 }