/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp

Bug Summary

File:	lib/Target/NVPTX/NVPTXISelLowering.cpp
Warning:	line 1849, column 13 Called C++ object pointer is null

Annotated Source Code

//===-- NVPTXISelLowering.cpp - NVPTX DAG Lowering Implementation ---------===//

// The LLVM Compiler Infrastructure

// This file is distributed under the University of Illinois Open Source

// License. See LICENSE.TXT for details.

//===----------------------------------------------------------------------===//

// This file defines the interfaces that NVPTX uses to lower LLVM code into a

// selection DAG.

//===----------------------------------------------------------------------===//

#include "MCTargetDesc/NVPTXBaseInfo.h"

#include "NVPTX.h"

#include "NVPTXISelLowering.h"

#include "NVPTXSection.h"

#include "NVPTXSubtarget.h"

#include "NVPTXTargetMachine.h"

#include "NVPTXTargetObjectFile.h"

#include "NVPTXUtilities.h"

#include "llvm/ADT/APInt.h"

#include "llvm/ADT/SmallVector.h"

#include "llvm/ADT/StringRef.h"

#include "llvm/CodeGen/Analysis.h"

#include "llvm/CodeGen/MachineFunction.h"

#include "llvm/CodeGen/MachineMemOperand.h"

#include "llvm/CodeGen/MachineValueType.h"

#include "llvm/CodeGen/SelectionDAG.h"

#include "llvm/CodeGen/SelectionDAGNodes.h"

#include "llvm/CodeGen/ValueTypes.h"

#include "llvm/IR/Argument.h"

#include "llvm/IR/Attributes.h"

#include "llvm/IR/CallSite.h"

#include "llvm/IR/Constants.h"

#include "llvm/IR/DataLayout.h"

#include "llvm/IR/DerivedTypes.h"

#include "llvm/IR/Function.h"

#include "llvm/IR/GlobalValue.h"

#include "llvm/IR/Instruction.h"

#include "llvm/IR/Instructions.h"

#include "llvm/IR/Module.h"

#include "llvm/IR/Type.h"

#include "llvm/IR/Value.h"

#include "llvm/Support/Casting.h"

#include "llvm/Support/CodeGen.h"

#include "llvm/Support/CommandLine.h"

#include "llvm/Support/ErrorHandling.h"

#include "llvm/Support/MathExtras.h"

#include "llvm/Support/raw_ostream.h"

#include "llvm/Target/TargetCallingConv.h"

#include "llvm/Target/TargetLowering.h"

#include "llvm/Target/TargetMachine.h"

#include "llvm/Target/TargetOptions.h"

#include <algorithm>

#include <cassert>

#include <cstdint>

#include <iterator>

#include <sstream>

#include <string>

#include <utility>

#include <vector>

#undef DEBUG_TYPE"nvptx-lower"

#define DEBUG_TYPE"nvptx-lower" "nvptx-lower"

using namespace llvm;

static unsigned int uniqueCallSite = 0;

static cl::opt<bool> sched4reg(

"nvptx-sched4reg",

cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false));

static cl::opt<unsigned>

FMAContractLevelOpt("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden,

cl::desc("NVPTX Specific: FMA contraction (0: don't do it"

" 1: do it 2: do it aggressively"),

cl::init(2));

static cl::opt<int> UsePrecDivF32(

"nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,

cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"

" IEEE Compliant F32 div.rnd if available."),

cl::init(2));

static cl::opt<bool> UsePrecSqrtF32(

"nvptx-prec-sqrtf32", cl::Hidden,

cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),

cl::init(true));

static cl::opt<bool> FtzEnabled(

"nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,

cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),

cl::init(false));

int NVPTXTargetLowering::getDivF32Level() const {

if (UsePrecDivF32.getNumOccurrences() > 0) {

100

// If nvptx-prec-div32=N is used on the command-line, always honor it

101

return UsePrecDivF32;

102

} else {

103

// Otherwise, use div.approx if fast math is enabled

104

if (getTargetMachine().Options.UnsafeFPMath)

105

return 0;

106

else

107

return 2;

108

}

109

}

110

111

bool NVPTXTargetLowering::usePrecSqrtF32() const {

112

if (UsePrecSqrtF32.getNumOccurrences() > 0) {

113

// If nvptx-prec-sqrtf32 is used on the command-line, always honor it

114

return UsePrecSqrtF32;

115

} else {

116

// Otherwise, use sqrt.approx if fast math is enabled

117

return !getTargetMachine().Options.UnsafeFPMath;

118

}

119

}

120

121

bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const {

122

// TODO: Get rid of this flag; there can be only one way to do this.

123

if (FtzEnabled.getNumOccurrences() > 0) {

124

// If nvptx-f32ftz is used on the command-line, always honor it

125

return FtzEnabled;

126

} else {

127

const Function *F = MF.getFunction();

128

// Otherwise, check for an nvptx-f32ftz attribute on the function

129

if (F->hasFnAttribute("nvptx-f32ftz"))

130

return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true";

131

else

132

return false;

133

}

134

}

135

136

static bool IsPTXVectorType(MVT VT) {

137

switch (VT.SimpleTy) {

138

default:

139

return false;

140

case MVT::v2i1:

141

case MVT::v4i1:

142

case MVT::v2i8:

143

case MVT::v4i8:

144

case MVT::v2i16:

145

case MVT::v4i16:

146

case MVT::v2i32:

147

case MVT::v4i32:

148

case MVT::v2i64:

149

case MVT::v2f32:

150

case MVT::v4f32:

151

case MVT::v2f64:

152

return true;

153

}

154

}

155

156

/// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive

157

/// EVTs that compose it. Unlike ComputeValueVTs, this will break apart vectors

158

/// into their primitive components.

159

/// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the

160

/// same number of types as the Ins/Outs arrays in LowerFormalArguments,

161

/// LowerCall, and LowerReturn.

162

static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,

163

Type *Ty, SmallVectorImpl<EVT> &ValueVTs,

164

SmallVectorImpl<uint64_t> *Offsets = nullptr,

165

uint64_t StartingOffset = 0) {

166

SmallVector<EVT, 16> TempVTs;

167

SmallVector<uint64_t, 16> TempOffsets;

168

169

ComputeValueVTs(TLI, DL, Ty, TempVTs, &TempOffsets, StartingOffset);

170

for (unsigned i = 0, e = TempVTs.size(); i != e; ++i) {

171

EVT VT = TempVTs[i];

172

uint64_t Off = TempOffsets[i];

173

if (VT.isVector())

174

for (unsigned j = 0, je = VT.getVectorNumElements(); j != je; ++j) {

175

ValueVTs.push_back(VT.getVectorElementType());

176

if (Offsets)

177

Offsets->push_back(Off+j*VT.getVectorElementType().getStoreSize());

178

}

179

else {

180

ValueVTs.push_back(VT);

181

if (Offsets)

182

Offsets->push_back(Off);

183

}

184

}

185

}

186

187

// NVPTXTargetLowering Constructor.

188

NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,

189

const NVPTXSubtarget &STI)

190

: TargetLowering(TM), nvTM(&TM), STI(STI) {

191

// always lower memset, memcpy, and memmove intrinsics to load/store

192

// instructions, rather

193

// then generating calls to memset, mempcy or memmove.

194

MaxStoresPerMemset = (unsigned) 0xFFFFFFFF;

195

MaxStoresPerMemcpy = (unsigned) 0xFFFFFFFF;

196

MaxStoresPerMemmove = (unsigned) 0xFFFFFFFF;

197

198

setBooleanContents(ZeroOrNegativeOneBooleanContent);

199

setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);

200

201

// Jump is Expensive. Don't create extra control flow for 'and', 'or'

202

// condition branches.

203

setJumpIsExpensive(true);

204

205

// Wide divides are _very_ slow. Try to reduce the width of the divide if

206

// possible.

207

addBypassSlowDiv(64, 32);

208

209

// By default, use the Source scheduling

210

if (sched4reg)

211

setSchedulingPreference(Sched::RegPressure);

212

else

213

setSchedulingPreference(Sched::Source);

214

215

addRegisterClass(MVT::i1, &NVPTX::Int1RegsRegClass);

216

addRegisterClass(MVT::i16, &NVPTX::Int16RegsRegClass);

217

addRegisterClass(MVT::i32, &NVPTX::Int32RegsRegClass);

218

addRegisterClass(MVT::i64, &NVPTX::Int64RegsRegClass);

219

addRegisterClass(MVT::f32, &NVPTX::Float32RegsRegClass);

220

addRegisterClass(MVT::f64, &NVPTX::Float64RegsRegClass);

221

addRegisterClass(MVT::f16, &NVPTX::Float16RegsRegClass);

222

223

setOperationAction(ISD::SETCC, MVT::f16,

224

STI.allowFP16Math() ? Legal : Promote);

225

226

// Operations not directly supported by NVPTX.

227

setOperationAction(ISD::SELECT_CC, MVT::f16,

228

STI.allowFP16Math() ? Expand : Promote);

229

setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);

230

setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);

231

setOperationAction(ISD::SELECT_CC, MVT::i1, Expand);

232

setOperationAction(ISD::SELECT_CC, MVT::i8, Expand);

233

setOperationAction(ISD::SELECT_CC, MVT::i16, Expand);

234

setOperationAction(ISD::SELECT_CC, MVT::i32, Expand);

235

setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);

236

setOperationAction(ISD::BR_CC, MVT::f16,

237

STI.allowFP16Math() ? Expand : Promote);

238

setOperationAction(ISD::BR_CC, MVT::f32, Expand);

239

setOperationAction(ISD::BR_CC, MVT::f64, Expand);

240

setOperationAction(ISD::BR_CC, MVT::i1, Expand);

241

setOperationAction(ISD::BR_CC, MVT::i8, Expand);

242

setOperationAction(ISD::BR_CC, MVT::i16, Expand);

243

setOperationAction(ISD::BR_CC, MVT::i32, Expand);

244

setOperationAction(ISD::BR_CC, MVT::i64, Expand);

245

// Some SIGN_EXTEND_INREG can be done using cvt instruction.

246

// For others we will expand to a SHL/SRA pair.

247

setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i64, Legal);

248

setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);

249

setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);

250

setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);

251

setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);

252

253

setOperationAction(ISD::SHL_PARTS, MVT::i32 , Custom);

254

setOperationAction(ISD::SRA_PARTS, MVT::i32 , Custom);

255

setOperationAction(ISD::SRL_PARTS, MVT::i32 , Custom);

256

setOperationAction(ISD::SHL_PARTS, MVT::i64 , Custom);

257

setOperationAction(ISD::SRA_PARTS, MVT::i64 , Custom);

258

setOperationAction(ISD::SRL_PARTS, MVT::i64 , Custom);

259

260

setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);

261

setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);

262

263

if (STI.hasROT64()) {

264

setOperationAction(ISD::ROTL, MVT::i64, Legal);

265

setOperationAction(ISD::ROTR, MVT::i64, Legal);

266

} else {

267

setOperationAction(ISD::ROTL, MVT::i64, Expand);

268

setOperationAction(ISD::ROTR, MVT::i64, Expand);

269

}

270

if (STI.hasROT32()) {

271

setOperationAction(ISD::ROTL, MVT::i32, Legal);

272

setOperationAction(ISD::ROTR, MVT::i32, Legal);

273

} else {

274

setOperationAction(ISD::ROTL, MVT::i32, Expand);

275

setOperationAction(ISD::ROTR, MVT::i32, Expand);

276

}

277

278

setOperationAction(ISD::ROTL, MVT::i16, Expand);

279

setOperationAction(ISD::ROTR, MVT::i16, Expand);

280

setOperationAction(ISD::ROTL, MVT::i8, Expand);

281

setOperationAction(ISD::ROTR, MVT::i8, Expand);

282

setOperationAction(ISD::BSWAP, MVT::i16, Expand);

283

setOperationAction(ISD::BSWAP, MVT::i32, Expand);

284

setOperationAction(ISD::BSWAP, MVT::i64, Expand);

285

286

// Indirect branch is not supported.

287

// This also disables Jump Table creation.

288

setOperationAction(ISD::BR_JT, MVT::Other, Expand);

289

setOperationAction(ISD::BRIND, MVT::Other, Expand);

290

291

setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);

292

setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);

293

294

// We want to legalize constant related memmove and memcopy

295

// intrinsics.

296

setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);

297

298

// Turn FP extload into load/fpextend

299

setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);

300

setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);

301

setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);

302

setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand);

303

setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand);

304

setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand);

305

setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand);

306

setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand);

307

setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand);

308

// Turn FP truncstore into trunc + store.

309

// FIXME: vector types should also be expanded

310

setTruncStoreAction(MVT::f32, MVT::f16, Expand);

311

setTruncStoreAction(MVT::f64, MVT::f16, Expand);

312

setTruncStoreAction(MVT::f64, MVT::f32, Expand);

313

314

// PTX does not support load / store predicate registers

315

setOperationAction(ISD::LOAD, MVT::i1, Custom);

316

setOperationAction(ISD::STORE, MVT::i1, Custom);

317

318

for (MVT VT : MVT::integer_valuetypes()) {

319

setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);

320

setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);

321

setTruncStoreAction(VT, MVT::i1, Expand);

322

}

323

324

// This is legal in NVPTX

325

setOperationAction(ISD::ConstantFP, MVT::f64, Legal);

326

setOperationAction(ISD::ConstantFP, MVT::f32, Legal);

327

setOperationAction(ISD::ConstantFP, MVT::f16, Legal);

328

329

// TRAP can be lowered to PTX trap

330

setOperationAction(ISD::TRAP, MVT::Other, Legal);

331

332

setOperationAction(ISD::ADDC, MVT::i64, Expand);

333

setOperationAction(ISD::ADDE, MVT::i64, Expand);

334

335

// Register custom handling for vector loads/stores

336

for (MVT VT : MVT::vector_valuetypes()) {

337

if (IsPTXVectorType(VT)) {

338

setOperationAction(ISD::LOAD, VT, Custom);

339

setOperationAction(ISD::STORE, VT, Custom);

340

setOperationAction(ISD::INTRINSIC_W_CHAIN, VT, Custom);

341

}

342

}

343

344

// Custom handling for i8 intrinsics

345

setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);

346

347

for (const auto& Ty : {MVT::i16, MVT::i32, MVT::i64}) {

348

setOperationAction(ISD::SMIN, Ty, Legal);

349

setOperationAction(ISD::SMAX, Ty, Legal);

350

setOperationAction(ISD::UMIN, Ty, Legal);

351

setOperationAction(ISD::UMAX, Ty, Legal);

352

353

setOperationAction(ISD::CTPOP, Ty, Legal);

354

setOperationAction(ISD::CTLZ, Ty, Legal);

355

}

356

357

setOperationAction(ISD::CTTZ, MVT::i16, Expand);

358

setOperationAction(ISD::CTTZ, MVT::i32, Expand);

359

setOperationAction(ISD::CTTZ, MVT::i64, Expand);

360

361

// PTX does not directly support SELP of i1, so promote to i32 first

362

setOperationAction(ISD::SELECT, MVT::i1, Custom);

363

364

// PTX cannot multiply two i64s in a single instruction.

365

setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);

366

setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);

367

368

// We have some custom DAG combine patterns for these nodes

369

setTargetDAGCombine(ISD::ADD);

370

setTargetDAGCombine(ISD::AND);

371

setTargetDAGCombine(ISD::FADD);

372

setTargetDAGCombine(ISD::MUL);

373

setTargetDAGCombine(ISD::SHL);

374

setTargetDAGCombine(ISD::SREM);

375

setTargetDAGCombine(ISD::UREM);

376

377

if (!STI.allowFP16Math()) {

378

// Promote fp16 arithmetic if fp16 hardware isn't available or the

379

// user passed --nvptx-no-fp16-math. The flag is useful because,

380

// although sm_53+ GPUs have some sort of FP16 support in

381

// hardware, only sm_53 and sm_60 have full implementation. Others

382

// only have token amount of hardware and are likely to run faster

383

// by using fp32 units instead.

384

setOperationAction(ISD::FADD, MVT::f16, Promote);

385

setOperationAction(ISD::FMUL, MVT::f16, Promote);

386

setOperationAction(ISD::FSUB, MVT::f16, Promote);

387

setOperationAction(ISD::FMA, MVT::f16, Promote);

388

}

389

// There's no neg.f16 instruction.

390

setOperationAction(ISD::FNEG, MVT::f16, Expand);

391

392

// Library functions. These default to Expand, but we have instructions

393

// for them.

394

setOperationAction(ISD::FCEIL, MVT::f16, Legal);

395

setOperationAction(ISD::FCEIL, MVT::f32, Legal);

396

setOperationAction(ISD::FCEIL, MVT::f64, Legal);

397

setOperationAction(ISD::FFLOOR, MVT::f16, Legal);

398

setOperationAction(ISD::FFLOOR, MVT::f32, Legal);

399

setOperationAction(ISD::FFLOOR, MVT::f64, Legal);

400

setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);

401

setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);

402

setOperationAction(ISD::FRINT, MVT::f16, Legal);

403

setOperationAction(ISD::FRINT, MVT::f32, Legal);

404

setOperationAction(ISD::FRINT, MVT::f64, Legal);

405

setOperationAction(ISD::FROUND, MVT::f16, Legal);

406

setOperationAction(ISD::FROUND, MVT::f32, Legal);

407

setOperationAction(ISD::FROUND, MVT::f64, Legal);

408

setOperationAction(ISD::FTRUNC, MVT::f16, Legal);

409

setOperationAction(ISD::FTRUNC, MVT::f32, Legal);

410

setOperationAction(ISD::FTRUNC, MVT::f64, Legal);

411

setOperationAction(ISD::FMINNUM, MVT::f32, Legal);

412

setOperationAction(ISD::FMINNUM, MVT::f64, Legal);

413

setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);

414

setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);

415

416

// 'Expand' implements FCOPYSIGN without calling an external library.

417

setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);

418

setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);

419

setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);

420

421

// FP16 does not support these nodes in hardware, but we can perform

422

// these ops using single-precision hardware.

423

setOperationAction(ISD::FDIV, MVT::f16, Promote);

424

setOperationAction(ISD::FREM, MVT::f16, Promote);

425

setOperationAction(ISD::FSQRT, MVT::f16, Promote);

426

setOperationAction(ISD::FSIN, MVT::f16, Promote);

427

setOperationAction(ISD::FCOS, MVT::f16, Promote);

428

setOperationAction(ISD::FABS, MVT::f16, Promote);

429

setOperationAction(ISD::FMINNUM, MVT::f16, Promote);

430

setOperationAction(ISD::FMAXNUM, MVT::f16, Promote);

431

setOperationAction(ISD::FMINNAN, MVT::f16, Promote);

432

setOperationAction(ISD::FMAXNAN, MVT::f16, Promote);

433

434

// No FEXP2, FLOG2. The PTX ex2 and log2 functions are always approximate.

435

// No FPOW or FREM in PTX.

436

437

// Now deduce the information based on the above mentioned

438

// actions

439

computeRegisterProperties(STI.getRegisterInfo());

440

}

441

442

const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {

443

switch ((NVPTXISD::NodeType)Opcode) {

444

case NVPTXISD::FIRST_NUMBER:

445

break;

446

case NVPTXISD::CALL:

447

return "NVPTXISD::CALL";

448

case NVPTXISD::RET_FLAG:

449

return "NVPTXISD::RET_FLAG";

450

case NVPTXISD::LOAD_PARAM:

451

return "NVPTXISD::LOAD_PARAM";

452

case NVPTXISD::Wrapper:

453

return "NVPTXISD::Wrapper";

454

case NVPTXISD::DeclareParam:

455

return "NVPTXISD::DeclareParam";

456

case NVPTXISD::DeclareScalarParam:

457

return "NVPTXISD::DeclareScalarParam";

458

case NVPTXISD::DeclareRet:

459

return "NVPTXISD::DeclareRet";

460

case NVPTXISD::DeclareScalarRet:

461

return "NVPTXISD::DeclareScalarRet";

462

case NVPTXISD::DeclareRetParam:

463

return "NVPTXISD::DeclareRetParam";

464

case NVPTXISD::PrintCall:

465

return "NVPTXISD::PrintCall";

466

case NVPTXISD::PrintConvergentCall:

467

return "NVPTXISD::PrintConvergentCall";

468

case NVPTXISD::PrintCallUni:

469

return "NVPTXISD::PrintCallUni";

470

case NVPTXISD::PrintConvergentCallUni:

471

return "NVPTXISD::PrintConvergentCallUni";

472

case NVPTXISD::LoadParam:

473

return "NVPTXISD::LoadParam";

474

case NVPTXISD::LoadParamV2:

475

return "NVPTXISD::LoadParamV2";

476

case NVPTXISD::LoadParamV4:

477

return "NVPTXISD::LoadParamV4";

478

case NVPTXISD::StoreParam:

479

return "NVPTXISD::StoreParam";

480

case NVPTXISD::StoreParamV2:

481

return "NVPTXISD::StoreParamV2";

482

case NVPTXISD::StoreParamV4:

483

return "NVPTXISD::StoreParamV4";

484

case NVPTXISD::StoreParamS32:

485

return "NVPTXISD::StoreParamS32";

486

case NVPTXISD::StoreParamU32:

487

return "NVPTXISD::StoreParamU32";

488

case NVPTXISD::CallArgBegin:

489

return "NVPTXISD::CallArgBegin";

490

case NVPTXISD::CallArg:

491

return "NVPTXISD::CallArg";

492

case NVPTXISD::LastCallArg:

493

return "NVPTXISD::LastCallArg";

494

case NVPTXISD::CallArgEnd:

495

return "NVPTXISD::CallArgEnd";

496

case NVPTXISD::CallVoid:

497

return "NVPTXISD::CallVoid";

498

case NVPTXISD::CallVal:

499

return "NVPTXISD::CallVal";

500

case NVPTXISD::CallSymbol:

501

return "NVPTXISD::CallSymbol";

502

case NVPTXISD::Prototype:

503

return "NVPTXISD::Prototype";

504

case NVPTXISD::MoveParam:

505

return "NVPTXISD::MoveParam";

506

case NVPTXISD::StoreRetval:

507

return "NVPTXISD::StoreRetval";

508

case NVPTXISD::StoreRetvalV2:

509

return "NVPTXISD::StoreRetvalV2";

510

case NVPTXISD::StoreRetvalV4:

511

return "NVPTXISD::StoreRetvalV4";

512

case NVPTXISD::PseudoUseParam:

513

return "NVPTXISD::PseudoUseParam";

514

case NVPTXISD::RETURN:

515

return "NVPTXISD::RETURN";

516

case NVPTXISD::CallSeqBegin:

517

return "NVPTXISD::CallSeqBegin";

518

case NVPTXISD::CallSeqEnd:

519

return "NVPTXISD::CallSeqEnd";

520

case NVPTXISD::CallPrototype:

521

return "NVPTXISD::CallPrototype";

522

case NVPTXISD::LoadV2:

523

return "NVPTXISD::LoadV2";

524

case NVPTXISD::LoadV4:

525

return "NVPTXISD::LoadV4";

526

case NVPTXISD::LDGV2:

527

return "NVPTXISD::LDGV2";

528

case NVPTXISD::LDGV4:

529

return "NVPTXISD::LDGV4";

530

case NVPTXISD::LDUV2:

531

return "NVPTXISD::LDUV2";

532

case NVPTXISD::LDUV4:

533

return "NVPTXISD::LDUV4";

534

case NVPTXISD::StoreV2:

535

return "NVPTXISD::StoreV2";

536

case NVPTXISD::StoreV4:

537

return "NVPTXISD::StoreV4";

538

case NVPTXISD::FUN_SHFL_CLAMP:

539

return "NVPTXISD::FUN_SHFL_CLAMP";

540

case NVPTXISD::FUN_SHFR_CLAMP:

541

return "NVPTXISD::FUN_SHFR_CLAMP";

542

case NVPTXISD::IMAD:

543

return "NVPTXISD::IMAD";

544

case NVPTXISD::Dummy:

545

return "NVPTXISD::Dummy";

546

case NVPTXISD::MUL_WIDE_SIGNED:

547

return "NVPTXISD::MUL_WIDE_SIGNED";

548

case NVPTXISD::MUL_WIDE_UNSIGNED:

549

return "NVPTXISD::MUL_WIDE_UNSIGNED";

550

case NVPTXISD::Tex1DFloatS32: return "NVPTXISD::Tex1DFloatS32";

551

case NVPTXISD::Tex1DFloatFloat: return "NVPTXISD::Tex1DFloatFloat";

552

case NVPTXISD::Tex1DFloatFloatLevel:

553

return "NVPTXISD::Tex1DFloatFloatLevel";

554

case NVPTXISD::Tex1DFloatFloatGrad:

555

return "NVPTXISD::Tex1DFloatFloatGrad";

556

case NVPTXISD::Tex1DS32S32: return "NVPTXISD::Tex1DS32S32";

557

case NVPTXISD::Tex1DS32Float: return "NVPTXISD::Tex1DS32Float";

558

case NVPTXISD::Tex1DS32FloatLevel:

559

return "NVPTXISD::Tex1DS32FloatLevel";

560

case NVPTXISD::Tex1DS32FloatGrad:

561

return "NVPTXISD::Tex1DS32FloatGrad";

562

case NVPTXISD::Tex1DU32S32: return "NVPTXISD::Tex1DU32S32";

563

case NVPTXISD::Tex1DU32Float: return "NVPTXISD::Tex1DU32Float";

564

case NVPTXISD::Tex1DU32FloatLevel:

565

return "NVPTXISD::Tex1DU32FloatLevel";

566

case NVPTXISD::Tex1DU32FloatGrad:

567

return "NVPTXISD::Tex1DU32FloatGrad";

568

case NVPTXISD::Tex1DArrayFloatS32: return "NVPTXISD::Tex1DArrayFloatS32";

569

case NVPTXISD::Tex1DArrayFloatFloat: return "NVPTXISD::Tex1DArrayFloatFloat";

570

case NVPTXISD::Tex1DArrayFloatFloatLevel:

571

return "NVPTXISD::Tex1DArrayFloatFloatLevel";

572

case NVPTXISD::Tex1DArrayFloatFloatGrad:

573

return "NVPTXISD::Tex1DArrayFloatFloatGrad";

574

case NVPTXISD::Tex1DArrayS32S32: return "NVPTXISD::Tex1DArrayS32S32";

575

case NVPTXISD::Tex1DArrayS32Float: return "NVPTXISD::Tex1DArrayS32Float";

576

case NVPTXISD::Tex1DArrayS32FloatLevel:

577

return "NVPTXISD::Tex1DArrayS32FloatLevel";

578

case NVPTXISD::Tex1DArrayS32FloatGrad:

579

return "NVPTXISD::Tex1DArrayS32FloatGrad";

580

case NVPTXISD::Tex1DArrayU32S32: return "NVPTXISD::Tex1DArrayU32S32";

581

case NVPTXISD::Tex1DArrayU32Float: return "NVPTXISD::Tex1DArrayU32Float";

582

case NVPTXISD::Tex1DArrayU32FloatLevel:

583

return "NVPTXISD::Tex1DArrayU32FloatLevel";

584

case NVPTXISD::Tex1DArrayU32FloatGrad:

585

return "NVPTXISD::Tex1DArrayU32FloatGrad";

586

case NVPTXISD::Tex2DFloatS32: return "NVPTXISD::Tex2DFloatS32";

587

case NVPTXISD::Tex2DFloatFloat: return "NVPTXISD::Tex2DFloatFloat";

588

case NVPTXISD::Tex2DFloatFloatLevel:

589

return "NVPTXISD::Tex2DFloatFloatLevel";

590

case NVPTXISD::Tex2DFloatFloatGrad:

591

return "NVPTXISD::Tex2DFloatFloatGrad";

592

case NVPTXISD::Tex2DS32S32: return "NVPTXISD::Tex2DS32S32";

593

case NVPTXISD::Tex2DS32Float: return "NVPTXISD::Tex2DS32Float";

594

case NVPTXISD::Tex2DS32FloatLevel:

595

return "NVPTXISD::Tex2DS32FloatLevel";

596

case NVPTXISD::Tex2DS32FloatGrad:

597

return "NVPTXISD::Tex2DS32FloatGrad";

598

case NVPTXISD::Tex2DU32S32: return "NVPTXISD::Tex2DU32S32";

599

case NVPTXISD::Tex2DU32Float: return "NVPTXISD::Tex2DU32Float";

600

case NVPTXISD::Tex2DU32FloatLevel:

601

return "NVPTXISD::Tex2DU32FloatLevel";

602

case NVPTXISD::Tex2DU32FloatGrad:

603

return "NVPTXISD::Tex2DU32FloatGrad";

604

case NVPTXISD::Tex2DArrayFloatS32: return "NVPTXISD::Tex2DArrayFloatS32";

605

case NVPTXISD::Tex2DArrayFloatFloat: return "NVPTXISD::Tex2DArrayFloatFloat";

606

case NVPTXISD::Tex2DArrayFloatFloatLevel:

607

return "NVPTXISD::Tex2DArrayFloatFloatLevel";

608

case NVPTXISD::Tex2DArrayFloatFloatGrad:

609

return "NVPTXISD::Tex2DArrayFloatFloatGrad";

610

case NVPTXISD::Tex2DArrayS32S32: return "NVPTXISD::Tex2DArrayS32S32";

611

case NVPTXISD::Tex2DArrayS32Float: return "NVPTXISD::Tex2DArrayS32Float";

612

case NVPTXISD::Tex2DArrayS32FloatLevel:

613

return "NVPTXISD::Tex2DArrayS32FloatLevel";

614

case NVPTXISD::Tex2DArrayS32FloatGrad:

615

return "NVPTXISD::Tex2DArrayS32FloatGrad";

616

case NVPTXISD::Tex2DArrayU32S32: return "NVPTXISD::Tex2DArrayU32S32";

617

case NVPTXISD::Tex2DArrayU32Float: return "NVPTXISD::Tex2DArrayU32Float";

618

case NVPTXISD::Tex2DArrayU32FloatLevel:

619

return "NVPTXISD::Tex2DArrayU32FloatLevel";

620

case NVPTXISD::Tex2DArrayU32FloatGrad:

621

return "NVPTXISD::Tex2DArrayU32FloatGrad";

622

case NVPTXISD::Tex3DFloatS32: return "NVPTXISD::Tex3DFloatS32";

623

case NVPTXISD::Tex3DFloatFloat: return "NVPTXISD::Tex3DFloatFloat";

624

case NVPTXISD::Tex3DFloatFloatLevel:

625

return "NVPTXISD::Tex3DFloatFloatLevel";

626

case NVPTXISD::Tex3DFloatFloatGrad:

627

return "NVPTXISD::Tex3DFloatFloatGrad";

628

case NVPTXISD::Tex3DS32S32: return "NVPTXISD::Tex3DS32S32";

629

case NVPTXISD::Tex3DS32Float: return "NVPTXISD::Tex3DS32Float";

630

case NVPTXISD::Tex3DS32FloatLevel:

631

return "NVPTXISD::Tex3DS32FloatLevel";

632

case NVPTXISD::Tex3DS32FloatGrad:

633

return "NVPTXISD::Tex3DS32FloatGrad";

634

case NVPTXISD::Tex3DU32S32: return "NVPTXISD::Tex3DU32S32";

635

case NVPTXISD::Tex3DU32Float: return "NVPTXISD::Tex3DU32Float";

636

case NVPTXISD::Tex3DU32FloatLevel:

637

return "NVPTXISD::Tex3DU32FloatLevel";

638

case NVPTXISD::Tex3DU32FloatGrad:

639

return "NVPTXISD::Tex3DU32FloatGrad";

640

case NVPTXISD::TexCubeFloatFloat: return "NVPTXISD::TexCubeFloatFloat";

641

case NVPTXISD::TexCubeFloatFloatLevel:

642

return "NVPTXISD::TexCubeFloatFloatLevel";

643

case NVPTXISD::TexCubeS32Float: return "NVPTXISD::TexCubeS32Float";

644

case NVPTXISD::TexCubeS32FloatLevel:

645

return "NVPTXISD::TexCubeS32FloatLevel";

646

case NVPTXISD::TexCubeU32Float: return "NVPTXISD::TexCubeU32Float";

647

case NVPTXISD::TexCubeU32FloatLevel:

648

return "NVPTXISD::TexCubeU32FloatLevel";

649

case NVPTXISD::TexCubeArrayFloatFloat:

650

return "NVPTXISD::TexCubeArrayFloatFloat";

651

case NVPTXISD::TexCubeArrayFloatFloatLevel:

652

return "NVPTXISD::TexCubeArrayFloatFloatLevel";

653

case NVPTXISD::TexCubeArrayS32Float:

654

return "NVPTXISD::TexCubeArrayS32Float";

655

case NVPTXISD::TexCubeArrayS32FloatLevel:

656

return "NVPTXISD::TexCubeArrayS32FloatLevel";

657

case NVPTXISD::TexCubeArrayU32Float:

658

return "NVPTXISD::TexCubeArrayU32Float";

659

case NVPTXISD::TexCubeArrayU32FloatLevel:

660

return "NVPTXISD::TexCubeArrayU32FloatLevel";

661

case NVPTXISD::Tld4R2DFloatFloat:

662

return "NVPTXISD::Tld4R2DFloatFloat";

663

case NVPTXISD::Tld4G2DFloatFloat:

664

return "NVPTXISD::Tld4G2DFloatFloat";

665

case NVPTXISD::Tld4B2DFloatFloat:

666

return "NVPTXISD::Tld4B2DFloatFloat";

667

case NVPTXISD::Tld4A2DFloatFloat:

668

return "NVPTXISD::Tld4A2DFloatFloat";

669

case NVPTXISD::Tld4R2DS64Float:

670

return "NVPTXISD::Tld4R2DS64Float";

671

case NVPTXISD::Tld4G2DS64Float:

672

return "NVPTXISD::Tld4G2DS64Float";

673

case NVPTXISD::Tld4B2DS64Float:

674

return "NVPTXISD::Tld4B2DS64Float";

675

case NVPTXISD::Tld4A2DS64Float:

676

return "NVPTXISD::Tld4A2DS64Float";

677

case NVPTXISD::Tld4R2DU64Float:

678

return "NVPTXISD::Tld4R2DU64Float";

679

case NVPTXISD::Tld4G2DU64Float:

680

return "NVPTXISD::Tld4G2DU64Float";

681

case NVPTXISD::Tld4B2DU64Float:

682

return "NVPTXISD::Tld4B2DU64Float";

683

case NVPTXISD::Tld4A2DU64Float:

684

return "NVPTXISD::Tld4A2DU64Float";

685

686

case NVPTXISD::TexUnified1DFloatS32:

687

return "NVPTXISD::TexUnified1DFloatS32";

688

case NVPTXISD::TexUnified1DFloatFloat:

689

return "NVPTXISD::TexUnified1DFloatFloat";

690

case NVPTXISD::TexUnified1DFloatFloatLevel:

691

return "NVPTXISD::TexUnified1DFloatFloatLevel";

692

case NVPTXISD::TexUnified1DFloatFloatGrad:

693

return "NVPTXISD::TexUnified1DFloatFloatGrad";

694

case NVPTXISD::TexUnified1DS32S32:

695

return "NVPTXISD::TexUnified1DS32S32";

696

case NVPTXISD::TexUnified1DS32Float:

697

return "NVPTXISD::TexUnified1DS32Float";

698

case NVPTXISD::TexUnified1DS32FloatLevel:

699

return "NVPTXISD::TexUnified1DS32FloatLevel";

700

case NVPTXISD::TexUnified1DS32FloatGrad:

701

return "NVPTXISD::TexUnified1DS32FloatGrad";

702

case NVPTXISD::TexUnified1DU32S32:

703

return "NVPTXISD::TexUnified1DU32S32";

704

case NVPTXISD::TexUnified1DU32Float:

705

return "NVPTXISD::TexUnified1DU32Float";

706

case NVPTXISD::TexUnified1DU32FloatLevel:

707

return "NVPTXISD::TexUnified1DU32FloatLevel";

708

case NVPTXISD::TexUnified1DU32FloatGrad:

709

return "NVPTXISD::TexUnified1DU32FloatGrad";

710

case NVPTXISD::TexUnified1DArrayFloatS32:

711

return "NVPTXISD::TexUnified1DArrayFloatS32";

712

case NVPTXISD::TexUnified1DArrayFloatFloat:

713

return "NVPTXISD::TexUnified1DArrayFloatFloat";

714

case NVPTXISD::TexUnified1DArrayFloatFloatLevel:

715

return "NVPTXISD::TexUnified1DArrayFloatFloatLevel";

716

case NVPTXISD::TexUnified1DArrayFloatFloatGrad:

717

return "NVPTXISD::TexUnified1DArrayFloatFloatGrad";

718

case NVPTXISD::TexUnified1DArrayS32S32:

719

return "NVPTXISD::TexUnified1DArrayS32S32";

720

case NVPTXISD::TexUnified1DArrayS32Float:

721

return "NVPTXISD::TexUnified1DArrayS32Float";

722

case NVPTXISD::TexUnified1DArrayS32FloatLevel:

723

return "NVPTXISD::TexUnified1DArrayS32FloatLevel";

724

case NVPTXISD::TexUnified1DArrayS32FloatGrad:

725

return "NVPTXISD::TexUnified1DArrayS32FloatGrad";

726

case NVPTXISD::TexUnified1DArrayU32S32:

727

return "NVPTXISD::TexUnified1DArrayU32S32";

728

case NVPTXISD::TexUnified1DArrayU32Float:

729

return "NVPTXISD::TexUnified1DArrayU32Float";

730

case NVPTXISD::TexUnified1DArrayU32FloatLevel:

731

return "NVPTXISD::TexUnified1DArrayU32FloatLevel";

732

case NVPTXISD::TexUnified1DArrayU32FloatGrad:

733

return "NVPTXISD::TexUnified1DArrayU32FloatGrad";

734

case NVPTXISD::TexUnified2DFloatS32:

735

return "NVPTXISD::TexUnified2DFloatS32";

736

case NVPTXISD::TexUnified2DFloatFloat:

737

return "NVPTXISD::TexUnified2DFloatFloat";

738

case NVPTXISD::TexUnified2DFloatFloatLevel:

739

return "NVPTXISD::TexUnified2DFloatFloatLevel";

740

case NVPTXISD::TexUnified2DFloatFloatGrad:

741

return "NVPTXISD::TexUnified2DFloatFloatGrad";

742

case NVPTXISD::TexUnified2DS32S32:

743

return "NVPTXISD::TexUnified2DS32S32";

744

case NVPTXISD::TexUnified2DS32Float:

745

return "NVPTXISD::TexUnified2DS32Float";

746

case NVPTXISD::TexUnified2DS32FloatLevel:

747

return "NVPTXISD::TexUnified2DS32FloatLevel";

748

case NVPTXISD::TexUnified2DS32FloatGrad:

749

return "NVPTXISD::TexUnified2DS32FloatGrad";

750

case NVPTXISD::TexUnified2DU32S32:

751

return "NVPTXISD::TexUnified2DU32S32";

752

case NVPTXISD::TexUnified2DU32Float:

753

return "NVPTXISD::TexUnified2DU32Float";

754

case NVPTXISD::TexUnified2DU32FloatLevel:

755

return "NVPTXISD::TexUnified2DU32FloatLevel";

756

case NVPTXISD::TexUnified2DU32FloatGrad:

757

return "NVPTXISD::TexUnified2DU32FloatGrad";

758

case NVPTXISD::TexUnified2DArrayFloatS32:

759

return "NVPTXISD::TexUnified2DArrayFloatS32";

760

case NVPTXISD::TexUnified2DArrayFloatFloat:

761

return "NVPTXISD::TexUnified2DArrayFloatFloat";

762

case NVPTXISD::TexUnified2DArrayFloatFloatLevel:

763

return "NVPTXISD::TexUnified2DArrayFloatFloatLevel";

764

case NVPTXISD::TexUnified2DArrayFloatFloatGrad:

765

return "NVPTXISD::TexUnified2DArrayFloatFloatGrad";

766

case NVPTXISD::TexUnified2DArrayS32S32:

767

return "NVPTXISD::TexUnified2DArrayS32S32";

768

case NVPTXISD::TexUnified2DArrayS32Float:

769

return "NVPTXISD::TexUnified2DArrayS32Float";

770

case NVPTXISD::TexUnified2DArrayS32FloatLevel:

771

return "NVPTXISD::TexUnified2DArrayS32FloatLevel";

772

case NVPTXISD::TexUnified2DArrayS32FloatGrad:

773

return "NVPTXISD::TexUnified2DArrayS32FloatGrad";

774

case NVPTXISD::TexUnified2DArrayU32S32:

775

return "NVPTXISD::TexUnified2DArrayU32S32";

776

case NVPTXISD::TexUnified2DArrayU32Float:

777

return "NVPTXISD::TexUnified2DArrayU32Float";

778

case NVPTXISD::TexUnified2DArrayU32FloatLevel:

779

return "NVPTXISD::TexUnified2DArrayU32FloatLevel";

780

case NVPTXISD::TexUnified2DArrayU32FloatGrad:

781

return "NVPTXISD::TexUnified2DArrayU32FloatGrad";

782

case NVPTXISD::TexUnified3DFloatS32:

783

return "NVPTXISD::TexUnified3DFloatS32";

784

case NVPTXISD::TexUnified3DFloatFloat:

785

return "NVPTXISD::TexUnified3DFloatFloat";

786

case NVPTXISD::TexUnified3DFloatFloatLevel:

787

return "NVPTXISD::TexUnified3DFloatFloatLevel";

788

case NVPTXISD::TexUnified3DFloatFloatGrad:

789

return "NVPTXISD::TexUnified3DFloatFloatGrad";

790

case NVPTXISD::TexUnified3DS32S32:

791

return "NVPTXISD::TexUnified3DS32S32";

792

case NVPTXISD::TexUnified3DS32Float:

793

return "NVPTXISD::TexUnified3DS32Float";

794

case NVPTXISD::TexUnified3DS32FloatLevel:

795

return "NVPTXISD::TexUnified3DS32FloatLevel";

796

case NVPTXISD::TexUnified3DS32FloatGrad:

797

return "NVPTXISD::TexUnified3DS32FloatGrad";

798

case NVPTXISD::TexUnified3DU32S32:

799

return "NVPTXISD::TexUnified3DU32S32";

800

case NVPTXISD::TexUnified3DU32Float:

801

return "NVPTXISD::TexUnified3DU32Float";

802

case NVPTXISD::TexUnified3DU32FloatLevel:

803

return "NVPTXISD::TexUnified3DU32FloatLevel";

804

case NVPTXISD::TexUnified3DU32FloatGrad:

805

return "NVPTXISD::TexUnified3DU32FloatGrad";

806

case NVPTXISD::TexUnifiedCubeFloatFloat:

807

return "NVPTXISD::TexUnifiedCubeFloatFloat";

808

case NVPTXISD::TexUnifiedCubeFloatFloatLevel:

809

return "NVPTXISD::TexUnifiedCubeFloatFloatLevel";

810

case NVPTXISD::TexUnifiedCubeS32Float:

811

return "NVPTXISD::TexUnifiedCubeS32Float";

812

case NVPTXISD::TexUnifiedCubeS32FloatLevel:

813

return "NVPTXISD::TexUnifiedCubeS32FloatLevel";

814

case NVPTXISD::TexUnifiedCubeU32Float:

815

return "NVPTXISD::TexUnifiedCubeU32Float";

816

case NVPTXISD::TexUnifiedCubeU32FloatLevel:

817

return "NVPTXISD::TexUnifiedCubeU32FloatLevel";

818

case NVPTXISD::TexUnifiedCubeArrayFloatFloat:

819

return "NVPTXISD::TexUnifiedCubeArrayFloatFloat";

820

case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:

821

return "NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel";

822

case NVPTXISD::TexUnifiedCubeArrayS32Float:

823

return "NVPTXISD::TexUnifiedCubeArrayS32Float";

824

case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:

825

return "NVPTXISD::TexUnifiedCubeArrayS32FloatLevel";

826

case NVPTXISD::TexUnifiedCubeArrayU32Float:

827

return "NVPTXISD::TexUnifiedCubeArrayU32Float";

828

case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:

829

return "NVPTXISD::TexUnifiedCubeArrayU32FloatLevel";

830

case NVPTXISD::Tld4UnifiedR2DFloatFloat:

831

return "NVPTXISD::Tld4UnifiedR2DFloatFloat";

832

case NVPTXISD::Tld4UnifiedG2DFloatFloat:

833

return "NVPTXISD::Tld4UnifiedG2DFloatFloat";

834

case NVPTXISD::Tld4UnifiedB2DFloatFloat:

835

return "NVPTXISD::Tld4UnifiedB2DFloatFloat";

836

case NVPTXISD::Tld4UnifiedA2DFloatFloat:

837

return "NVPTXISD::Tld4UnifiedA2DFloatFloat";

838

case NVPTXISD::Tld4UnifiedR2DS64Float:

839

return "NVPTXISD::Tld4UnifiedR2DS64Float";

840

case NVPTXISD::Tld4UnifiedG2DS64Float:

841

return "NVPTXISD::Tld4UnifiedG2DS64Float";

842

case NVPTXISD::Tld4UnifiedB2DS64Float:

843

return "NVPTXISD::Tld4UnifiedB2DS64Float";

844

case NVPTXISD::Tld4UnifiedA2DS64Float:

845

return "NVPTXISD::Tld4UnifiedA2DS64Float";

846

case NVPTXISD::Tld4UnifiedR2DU64Float:

847

return "NVPTXISD::Tld4UnifiedR2DU64Float";

848

case NVPTXISD::Tld4UnifiedG2DU64Float:

849

return "NVPTXISD::Tld4UnifiedG2DU64Float";

850

case NVPTXISD::Tld4UnifiedB2DU64Float:

851

return "NVPTXISD::Tld4UnifiedB2DU64Float";

852

case NVPTXISD::Tld4UnifiedA2DU64Float:

853

return "NVPTXISD::Tld4UnifiedA2DU64Float";

854

855

case NVPTXISD::Suld1DI8Clamp: return "NVPTXISD::Suld1DI8Clamp";

856

case NVPTXISD::Suld1DI16Clamp: return "NVPTXISD::Suld1DI16Clamp";

857

case NVPTXISD::Suld1DI32Clamp: return "NVPTXISD::Suld1DI32Clamp";

858

case NVPTXISD::Suld1DI64Clamp: return "NVPTXISD::Suld1DI64Clamp";

859

case NVPTXISD::Suld1DV2I8Clamp: return "NVPTXISD::Suld1DV2I8Clamp";

860

case NVPTXISD::Suld1DV2I16Clamp: return "NVPTXISD::Suld1DV2I16Clamp";

861

case NVPTXISD::Suld1DV2I32Clamp: return "NVPTXISD::Suld1DV2I32Clamp";

862

case NVPTXISD::Suld1DV2I64Clamp: return "NVPTXISD::Suld1DV2I64Clamp";

863

case NVPTXISD::Suld1DV4I8Clamp: return "NVPTXISD::Suld1DV4I8Clamp";

864

case NVPTXISD::Suld1DV4I16Clamp: return "NVPTXISD::Suld1DV4I16Clamp";

865

case NVPTXISD::Suld1DV4I32Clamp: return "NVPTXISD::Suld1DV4I32Clamp";

866

867

case NVPTXISD::Suld1DArrayI8Clamp: return "NVPTXISD::Suld1DArrayI8Clamp";

868

case NVPTXISD::Suld1DArrayI16Clamp: return "NVPTXISD::Suld1DArrayI16Clamp";

869

case NVPTXISD::Suld1DArrayI32Clamp: return "NVPTXISD::Suld1DArrayI32Clamp";

870

case NVPTXISD::Suld1DArrayI64Clamp: return "NVPTXISD::Suld1DArrayI64Clamp";

871

case NVPTXISD::Suld1DArrayV2I8Clamp: return "NVPTXISD::Suld1DArrayV2I8Clamp";

872

case NVPTXISD::Suld1DArrayV2I16Clamp:return "NVPTXISD::Suld1DArrayV2I16Clamp";

873

case NVPTXISD::Suld1DArrayV2I32Clamp:return "NVPTXISD::Suld1DArrayV2I32Clamp";

874

case NVPTXISD::Suld1DArrayV2I64Clamp:return "NVPTXISD::Suld1DArrayV2I64Clamp";

875

case NVPTXISD::Suld1DArrayV4I8Clamp: return "NVPTXISD::Suld1DArrayV4I8Clamp";

876

case NVPTXISD::Suld1DArrayV4I16Clamp:return "NVPTXISD::Suld1DArrayV4I16Clamp";

877

case NVPTXISD::Suld1DArrayV4I32Clamp:return "NVPTXISD::Suld1DArrayV4I32Clamp";

878

879

case NVPTXISD::Suld2DI8Clamp: return "NVPTXISD::Suld2DI8Clamp";

880

case NVPTXISD::Suld2DI16Clamp: return "NVPTXISD::Suld2DI16Clamp";

881

case NVPTXISD::Suld2DI32Clamp: return "NVPTXISD::Suld2DI32Clamp";

882

case NVPTXISD::Suld2DI64Clamp: return "NVPTXISD::Suld2DI64Clamp";

883

case NVPTXISD::Suld2DV2I8Clamp: return "NVPTXISD::Suld2DV2I8Clamp";

884

case NVPTXISD::Suld2DV2I16Clamp: return "NVPTXISD::Suld2DV2I16Clamp";

885

case NVPTXISD::Suld2DV2I32Clamp: return "NVPTXISD::Suld2DV2I32Clamp";

886

case NVPTXISD::Suld2DV2I64Clamp: return "NVPTXISD::Suld2DV2I64Clamp";

887

case NVPTXISD::Suld2DV4I8Clamp: return "NVPTXISD::Suld2DV4I8Clamp";

888

case NVPTXISD::Suld2DV4I16Clamp: return "NVPTXISD::Suld2DV4I16Clamp";

889

case NVPTXISD::Suld2DV4I32Clamp: return "NVPTXISD::Suld2DV4I32Clamp";

890

891

case NVPTXISD::Suld2DArrayI8Clamp: return "NVPTXISD::Suld2DArrayI8Clamp";

892

case NVPTXISD::Suld2DArrayI16Clamp: return "NVPTXISD::Suld2DArrayI16Clamp";

893

case NVPTXISD::Suld2DArrayI32Clamp: return "NVPTXISD::Suld2DArrayI32Clamp";

894

case NVPTXISD::Suld2DArrayI64Clamp: return "NVPTXISD::Suld2DArrayI64Clamp";

895

case NVPTXISD::Suld2DArrayV2I8Clamp: return "NVPTXISD::Suld2DArrayV2I8Clamp";

896

case NVPTXISD::Suld2DArrayV2I16Clamp:return "NVPTXISD::Suld2DArrayV2I16Clamp";

897

case NVPTXISD::Suld2DArrayV2I32Clamp:return "NVPTXISD::Suld2DArrayV2I32Clamp";

898

case NVPTXISD::Suld2DArrayV2I64Clamp:return "NVPTXISD::Suld2DArrayV2I64Clamp";

899

case NVPTXISD::Suld2DArrayV4I8Clamp: return "NVPTXISD::Suld2DArrayV4I8Clamp";

900

case NVPTXISD::Suld2DArrayV4I16Clamp:return "NVPTXISD::Suld2DArrayV4I16Clamp";

901

case NVPTXISD::Suld2DArrayV4I32Clamp:return "NVPTXISD::Suld2DArrayV4I32Clamp";

902

903

case NVPTXISD::Suld3DI8Clamp: return "NVPTXISD::Suld3DI8Clamp";

904

case NVPTXISD::Suld3DI16Clamp: return "NVPTXISD::Suld3DI16Clamp";

905

case NVPTXISD::Suld3DI32Clamp: return "NVPTXISD::Suld3DI32Clamp";

906

case NVPTXISD::Suld3DI64Clamp: return "NVPTXISD::Suld3DI64Clamp";

907

case NVPTXISD::Suld3DV2I8Clamp: return "NVPTXISD::Suld3DV2I8Clamp";

908

case NVPTXISD::Suld3DV2I16Clamp: return "NVPTXISD::Suld3DV2I16Clamp";

909

case NVPTXISD::Suld3DV2I32Clamp: return "NVPTXISD::Suld3DV2I32Clamp";

910

case NVPTXISD::Suld3DV2I64Clamp: return "NVPTXISD::Suld3DV2I64Clamp";

911

case NVPTXISD::Suld3DV4I8Clamp: return "NVPTXISD::Suld3DV4I8Clamp";

912

case NVPTXISD::Suld3DV4I16Clamp: return "NVPTXISD::Suld3DV4I16Clamp";

913

case NVPTXISD::Suld3DV4I32Clamp: return "NVPTXISD::Suld3DV4I32Clamp";

914

915

case NVPTXISD::Suld1DI8Trap: return "NVPTXISD::Suld1DI8Trap";

916

case NVPTXISD::Suld1DI16Trap: return "NVPTXISD::Suld1DI16Trap";

917

case NVPTXISD::Suld1DI32Trap: return "NVPTXISD::Suld1DI32Trap";

918

case NVPTXISD::Suld1DI64Trap: return "NVPTXISD::Suld1DI64Trap";

919

case NVPTXISD::Suld1DV2I8Trap: return "NVPTXISD::Suld1DV2I8Trap";

920

case NVPTXISD::Suld1DV2I16Trap: return "NVPTXISD::Suld1DV2I16Trap";

921

case NVPTXISD::Suld1DV2I32Trap: return "NVPTXISD::Suld1DV2I32Trap";

922

case NVPTXISD::Suld1DV2I64Trap: return "NVPTXISD::Suld1DV2I64Trap";

923

case NVPTXISD::Suld1DV4I8Trap: return "NVPTXISD::Suld1DV4I8Trap";

924

case NVPTXISD::Suld1DV4I16Trap: return "NVPTXISD::Suld1DV4I16Trap";

925

case NVPTXISD::Suld1DV4I32Trap: return "NVPTXISD::Suld1DV4I32Trap";

926

927

case NVPTXISD::Suld1DArrayI8Trap: return "NVPTXISD::Suld1DArrayI8Trap";

928

case NVPTXISD::Suld1DArrayI16Trap: return "NVPTXISD::Suld1DArrayI16Trap";

929

case NVPTXISD::Suld1DArrayI32Trap: return "NVPTXISD::Suld1DArrayI32Trap";

930

case NVPTXISD::Suld1DArrayI64Trap: return "NVPTXISD::Suld1DArrayI64Trap";

931

case NVPTXISD::Suld1DArrayV2I8Trap: return "NVPTXISD::Suld1DArrayV2I8Trap";

932

case NVPTXISD::Suld1DArrayV2I16Trap: return "NVPTXISD::Suld1DArrayV2I16Trap";

933

case NVPTXISD::Suld1DArrayV2I32Trap: return "NVPTXISD::Suld1DArrayV2I32Trap";

934

case NVPTXISD::Suld1DArrayV2I64Trap: return "NVPTXISD::Suld1DArrayV2I64Trap";

935

case NVPTXISD::Suld1DArrayV4I8Trap: return "NVPTXISD::Suld1DArrayV4I8Trap";

936

case NVPTXISD::Suld1DArrayV4I16Trap: return "NVPTXISD::Suld1DArrayV4I16Trap";

937

case NVPTXISD::Suld1DArrayV4I32Trap: return "NVPTXISD::Suld1DArrayV4I32Trap";

938

939

case NVPTXISD::Suld2DI8Trap: return "NVPTXISD::Suld2DI8Trap";

940

case NVPTXISD::Suld2DI16Trap: return "NVPTXISD::Suld2DI16Trap";

941

case NVPTXISD::Suld2DI32Trap: return "NVPTXISD::Suld2DI32Trap";

942

case NVPTXISD::Suld2DI64Trap: return "NVPTXISD::Suld2DI64Trap";

943

case NVPTXISD::Suld2DV2I8Trap: return "NVPTXISD::Suld2DV2I8Trap";

944

case NVPTXISD::Suld2DV2I16Trap: return "NVPTXISD::Suld2DV2I16Trap";

945

case NVPTXISD::Suld2DV2I32Trap: return "NVPTXISD::Suld2DV2I32Trap";

946

case NVPTXISD::Suld2DV2I64Trap: return "NVPTXISD::Suld2DV2I64Trap";

947

case NVPTXISD::Suld2DV4I8Trap: return "NVPTXISD::Suld2DV4I8Trap";

948

case NVPTXISD::Suld2DV4I16Trap: return "NVPTXISD::Suld2DV4I16Trap";

949

case NVPTXISD::Suld2DV4I32Trap: return "NVPTXISD::Suld2DV4I32Trap";

950

951

case NVPTXISD::Suld2DArrayI8Trap: return "NVPTXISD::Suld2DArrayI8Trap";

952

case NVPTXISD::Suld2DArrayI16Trap: return "NVPTXISD::Suld2DArrayI16Trap";

953

case NVPTXISD::Suld2DArrayI32Trap: return "NVPTXISD::Suld2DArrayI32Trap";

954

case NVPTXISD::Suld2DArrayI64Trap: return "NVPTXISD::Suld2DArrayI64Trap";

955

case NVPTXISD::Suld2DArrayV2I8Trap: return "NVPTXISD::Suld2DArrayV2I8Trap";

956

case NVPTXISD::Suld2DArrayV2I16Trap: return "NVPTXISD::Suld2DArrayV2I16Trap";

957

case NVPTXISD::Suld2DArrayV2I32Trap: return "NVPTXISD::Suld2DArrayV2I32Trap";

958

case NVPTXISD::Suld2DArrayV2I64Trap: return "NVPTXISD::Suld2DArrayV2I64Trap";

959

case NVPTXISD::Suld2DArrayV4I8Trap: return "NVPTXISD::Suld2DArrayV4I8Trap";

960

case NVPTXISD::Suld2DArrayV4I16Trap: return "NVPTXISD::Suld2DArrayV4I16Trap";

961

case NVPTXISD::Suld2DArrayV4I32Trap: return "NVPTXISD::Suld2DArrayV4I32Trap";

962

963

case NVPTXISD::Suld3DI8Trap: return "NVPTXISD::Suld3DI8Trap";

964

case NVPTXISD::Suld3DI16Trap: return "NVPTXISD::Suld3DI16Trap";

965

case NVPTXISD::Suld3DI32Trap: return "NVPTXISD::Suld3DI32Trap";

966

case NVPTXISD::Suld3DI64Trap: return "NVPTXISD::Suld3DI64Trap";

967

case NVPTXISD::Suld3DV2I8Trap: return "NVPTXISD::Suld3DV2I8Trap";

968

case NVPTXISD::Suld3DV2I16Trap: return "NVPTXISD::Suld3DV2I16Trap";

969

case NVPTXISD::Suld3DV2I32Trap: return "NVPTXISD::Suld3DV2I32Trap";

970

case NVPTXISD::Suld3DV2I64Trap: return "NVPTXISD::Suld3DV2I64Trap";

971

case NVPTXISD::Suld3DV4I8Trap: return "NVPTXISD::Suld3DV4I8Trap";

972

case NVPTXISD::Suld3DV4I16Trap: return "NVPTXISD::Suld3DV4I16Trap";

973

case NVPTXISD::Suld3DV4I32Trap: return "NVPTXISD::Suld3DV4I32Trap";

974

975

case NVPTXISD::Suld1DI8Zero: return "NVPTXISD::Suld1DI8Zero";

976

case NVPTXISD::Suld1DI16Zero: return "NVPTXISD::Suld1DI16Zero";

977

case NVPTXISD::Suld1DI32Zero: return "NVPTXISD::Suld1DI32Zero";

978

case NVPTXISD::Suld1DI64Zero: return "NVPTXISD::Suld1DI64Zero";

979

case NVPTXISD::Suld1DV2I8Zero: return "NVPTXISD::Suld1DV2I8Zero";

980

case NVPTXISD::Suld1DV2I16Zero: return "NVPTXISD::Suld1DV2I16Zero";

981

case NVPTXISD::Suld1DV2I32Zero: return "NVPTXISD::Suld1DV2I32Zero";

982

case NVPTXISD::Suld1DV2I64Zero: return "NVPTXISD::Suld1DV2I64Zero";

983

case NVPTXISD::Suld1DV4I8Zero: return "NVPTXISD::Suld1DV4I8Zero";

984

case NVPTXISD::Suld1DV4I16Zero: return "NVPTXISD::Suld1DV4I16Zero";

985

case NVPTXISD::Suld1DV4I32Zero: return "NVPTXISD::Suld1DV4I32Zero";

986

987

case NVPTXISD::Suld1DArrayI8Zero: return "NVPTXISD::Suld1DArrayI8Zero";

988

case NVPTXISD::Suld1DArrayI16Zero: return "NVPTXISD::Suld1DArrayI16Zero";

989

case NVPTXISD::Suld1DArrayI32Zero: return "NVPTXISD::Suld1DArrayI32Zero";

990

case NVPTXISD::Suld1DArrayI64Zero: return "NVPTXISD::Suld1DArrayI64Zero";

991

case NVPTXISD::Suld1DArrayV2I8Zero: return "NVPTXISD::Suld1DArrayV2I8Zero";

992

case NVPTXISD::Suld1DArrayV2I16Zero: return "NVPTXISD::Suld1DArrayV2I16Zero";

993

case NVPTXISD::Suld1DArrayV2I32Zero: return "NVPTXISD::Suld1DArrayV2I32Zero";

994

case NVPTXISD::Suld1DArrayV2I64Zero: return "NVPTXISD::Suld1DArrayV2I64Zero";

995

case NVPTXISD::Suld1DArrayV4I8Zero: return "NVPTXISD::Suld1DArrayV4I8Zero";

996

case NVPTXISD::Suld1DArrayV4I16Zero: return "NVPTXISD::Suld1DArrayV4I16Zero";

997

case NVPTXISD::Suld1DArrayV4I32Zero: return "NVPTXISD::Suld1DArrayV4I32Zero";

998

999

case NVPTXISD::Suld2DI8Zero: return "NVPTXISD::Suld2DI8Zero";

1000

case NVPTXISD::Suld2DI16Zero: return "NVPTXISD::Suld2DI16Zero";

1001

case NVPTXISD::Suld2DI32Zero: return "NVPTXISD::Suld2DI32Zero";

1002

case NVPTXISD::Suld2DI64Zero: return "NVPTXISD::Suld2DI64Zero";

1003

case NVPTXISD::Suld2DV2I8Zero: return "NVPTXISD::Suld2DV2I8Zero";

1004

case NVPTXISD::Suld2DV2I16Zero: return "NVPTXISD::Suld2DV2I16Zero";

1005

case NVPTXISD::Suld2DV2I32Zero: return "NVPTXISD::Suld2DV2I32Zero";

1006

case NVPTXISD::Suld2DV2I64Zero: return "NVPTXISD::Suld2DV2I64Zero";

1007

case NVPTXISD::Suld2DV4I8Zero: return "NVPTXISD::Suld2DV4I8Zero";

1008

case NVPTXISD::Suld2DV4I16Zero: return "NVPTXISD::Suld2DV4I16Zero";

1009

case NVPTXISD::Suld2DV4I32Zero: return "NVPTXISD::Suld2DV4I32Zero";

1010

1011

case NVPTXISD::Suld2DArrayI8Zero: return "NVPTXISD::Suld2DArrayI8Zero";

1012

case NVPTXISD::Suld2DArrayI16Zero: return "NVPTXISD::Suld2DArrayI16Zero";

1013

case NVPTXISD::Suld2DArrayI32Zero: return "NVPTXISD::Suld2DArrayI32Zero";

1014

case NVPTXISD::Suld2DArrayI64Zero: return "NVPTXISD::Suld2DArrayI64Zero";

1015

case NVPTXISD::Suld2DArrayV2I8Zero: return "NVPTXISD::Suld2DArrayV2I8Zero";

1016

case NVPTXISD::Suld2DArrayV2I16Zero: return "NVPTXISD::Suld2DArrayV2I16Zero";

1017

case NVPTXISD::Suld2DArrayV2I32Zero: return "NVPTXISD::Suld2DArrayV2I32Zero";

1018

case NVPTXISD::Suld2DArrayV2I64Zero: return "NVPTXISD::Suld2DArrayV2I64Zero";

1019

case NVPTXISD::Suld2DArrayV4I8Zero: return "NVPTXISD::Suld2DArrayV4I8Zero";

1020

case NVPTXISD::Suld2DArrayV4I16Zero: return "NVPTXISD::Suld2DArrayV4I16Zero";

1021

case NVPTXISD::Suld2DArrayV4I32Zero: return "NVPTXISD::Suld2DArrayV4I32Zero";

1022

1023

case NVPTXISD::Suld3DI8Zero: return "NVPTXISD::Suld3DI8Zero";

1024

case NVPTXISD::Suld3DI16Zero: return "NVPTXISD::Suld3DI16Zero";

1025

case NVPTXISD::Suld3DI32Zero: return "NVPTXISD::Suld3DI32Zero";

1026

case NVPTXISD::Suld3DI64Zero: return "NVPTXISD::Suld3DI64Zero";

1027

case NVPTXISD::Suld3DV2I8Zero: return "NVPTXISD::Suld3DV2I8Zero";

1028

case NVPTXISD::Suld3DV2I16Zero: return "NVPTXISD::Suld3DV2I16Zero";

1029

case NVPTXISD::Suld3DV2I32Zero: return "NVPTXISD::Suld3DV2I32Zero";

1030

case NVPTXISD::Suld3DV2I64Zero: return "NVPTXISD::Suld3DV2I64Zero";

1031

case NVPTXISD::Suld3DV4I8Zero: return "NVPTXISD::Suld3DV4I8Zero";

1032

case NVPTXISD::Suld3DV4I16Zero: return "NVPTXISD::Suld3DV4I16Zero";

1033

case NVPTXISD::Suld3DV4I32Zero: return "NVPTXISD::Suld3DV4I32Zero";

1034

}

1035

return nullptr;

1036

}

1037

1038

TargetLoweringBase::LegalizeTypeAction

1039

NVPTXTargetLowering::getPreferredVectorAction(EVT VT) const {

1040

if (VT.getVectorNumElements() != 1 && VT.getScalarType() == MVT::i1)

1041

return TypeSplitVector;

1042

1043

return TargetLoweringBase::getPreferredVectorAction(VT);

1044

}

1045

1046

SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,

1047

int Enabled, int &ExtraSteps,

1048

bool &UseOneConst,

1049

bool Reciprocal) const {

1050

if (!(Enabled == ReciprocalEstimate::Enabled ||

1051

(Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32())))

1052

return SDValue();

1053

1054

if (ExtraSteps == ReciprocalEstimate::Unspecified)

1055

ExtraSteps = 0;

1056

1057

SDLoc DL(Operand);

1058

EVT VT = Operand.getValueType();

1059

bool Ftz = useF32FTZ(DAG.getMachineFunction());

1060

1061

auto MakeIntrinsicCall = [&](Intrinsic::ID IID) {

1062

return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,

1063

DAG.getConstant(IID, DL, MVT::i32), Operand);

1064

};

1065

1066

// The sqrt and rsqrt refinement processes assume we always start out with an

1067

// approximation of the rsqrt. Therefore, if we're going to do any refinement

1068

// (i.e. ExtraSteps > 0), we must return an rsqrt. But if we're *not* doing

1069

// any refinement, we must return a regular sqrt.

1070

if (Reciprocal || ExtraSteps > 0) {

1071

if (VT == MVT::f32)

1072

return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f

1073

: Intrinsic::nvvm_rsqrt_approx_f);

1074

else if (VT == MVT::f64)

1075

return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);

1076

else

1077

return SDValue();

1078

} else {

1079

if (VT == MVT::f32)

1080

return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f

1081

: Intrinsic::nvvm_sqrt_approx_f);

1082

else {

1083

// There's no sqrt.approx.f64 instruction, so we emit

1084

// reciprocal(rsqrt(x)). This is faster than

1085

// select(x == 0, 0, x * rsqrt(x)). (In fact, it's faster than plain

1086

// x * rsqrt(x).)

1087

return DAG.getNode(

1088

ISD::INTRINSIC_WO_CHAIN, DL, VT,

1089

DAG.getConstant(Intrinsic::nvvm_rcp_approx_ftz_d, DL, MVT::i32),

1090

MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));

1091

}

1092

}

1093

}

1094

1095

SDValue

1096

NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {

1097

SDLoc dl(Op);

1098

const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();

1099

auto PtrVT = getPointerTy(DAG.getDataLayout());

1100

Op = DAG.getTargetGlobalAddress(GV, dl, PtrVT);

1101

return DAG.getNode(NVPTXISD::Wrapper, dl, PtrVT, Op);

1102

}

1103

1104

std::string NVPTXTargetLowering::getPrototype(

1105

const DataLayout &DL, Type *retTy, const ArgListTy &Args,

1106

const SmallVectorImpl<ISD::OutputArg> &Outs, unsigned retAlignment,

1107

const ImmutableCallSite *CS) const {

1108

auto PtrVT = getPointerTy(DL);

1109

1110

bool isABI = (STI.getSmVersion() >= 20);

1111

assert(isABI && "Non-ABI compilation is not supported")((isABI && "Non-ABI compilation is not supported") ? static_cast
<void> (0) : __assert_fail ("isABI && \"Non-ABI compilation is not supported\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1111, __PRETTY_FUNCTION__));

1112

if (!isABI)

1113

return "";

1114

1115

std::stringstream O;

1116

O << "prototype_" << uniqueCallSite << " : .callprototype ";

1117

1118

if (retTy->getTypeID() == Type::VoidTyID) {

1119

O << "()";

1120

} else {

1121

O << "(";

1122

if (retTy->isFloatingPointTy() || retTy->isIntegerTy()) {

1123

unsigned size = 0;

1124

if (auto *ITy = dyn_cast<IntegerType>(retTy)) {

1125

size = ITy->getBitWidth();

1126

} else {

1127

assert(retTy->isFloatingPointTy() &&((retTy->isFloatingPointTy() && "Floating point type expected here"
) ? static_cast<void> (0) : __assert_fail ("retTy->isFloatingPointTy() && \"Floating point type expected here\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1128, __PRETTY_FUNCTION__))

1128

"Floating point type expected here")((retTy->isFloatingPointTy() && "Floating point type expected here"
) ? static_cast<void> (0) : __assert_fail ("retTy->isFloatingPointTy() && \"Floating point type expected here\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1128, __PRETTY_FUNCTION__));

1129

size = retTy->getPrimitiveSizeInBits();

1130

}

1131

// PTX ABI requires all scalar return values to be at least 32

1132

// bits in size. fp16 normally uses .b16 as its storage type in

1133

// PTX, so its size must be adjusted here, too.

1134

if (size < 32)

1135

size = 32;

1136

1137

O << ".param .b" << size << " _";

1138

} else if (isa<PointerType>(retTy)) {

1139

O << ".param .b" << PtrVT.getSizeInBits() << " _";

1140

} else if (retTy->isAggregateType() || retTy->isVectorTy()) {

1141

auto &DL = CS->getCalledFunction()->getParent()->getDataLayout();

1142

O << ".param .align " << retAlignment << " .b8 _["

1143

<< DL.getTypeAllocSize(retTy) << "]";

1144

} else {

1145

llvm_unreachable("Unknown return type")::llvm::llvm_unreachable_internal("Unknown return type", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1145);

1146

}

1147

O << ") ";

1148

}

1149

O << "_ (";

1150

1151

bool first = true;

1152

1153

unsigned OIdx = 0;

1154

for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {

1155

Type *Ty = Args[i].Ty;

1156

if (!first) {

1157

O << ", ";

1158

}

1159

first = false;

1160

1161

if (!Outs[OIdx].Flags.isByVal()) {

1162

if (Ty->isAggregateType() || Ty->isVectorTy()) {

1163

unsigned align = 0;

1164

const CallInst *CallI = cast<CallInst>(CS->getInstruction());

1165

// +1 because index 0 is reserved for return type alignment

1166

if (!getAlign(*CallI, i + 1, align))

1167

align = DL.getABITypeAlignment(Ty);

1168

unsigned sz = DL.getTypeAllocSize(Ty);

1169

O << ".param .align " << align << " .b8 ";

1170

O << "_";

1171

O << "[" << sz << "]";

1172

// update the index for Outs

1173

SmallVector<EVT, 16> vtparts;

1174

ComputeValueVTs(*this, DL, Ty, vtparts);

1175

if (unsigned len = vtparts.size())

1176

OIdx += len - 1;

1177

continue;

1178

}

1179

// i8 types in IR will be i16 types in SDAG

1180

assert((getValueType(DL, Ty) == Outs[OIdx].VT ||(((getValueType(DL, Ty) == Outs[OIdx].VT || (getValueType(DL,
Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
"type mismatch between callee prototype and arguments") ? static_cast
<void> (0) : __assert_fail ("(getValueType(DL, Ty) == Outs[OIdx].VT || (getValueType(DL, Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) && \"type mismatch between callee prototype and arguments\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1182, __PRETTY_FUNCTION__))

1181

(getValueType(DL, Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&(((getValueType(DL, Ty) == Outs[OIdx].VT || (getValueType(DL,
Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
"type mismatch between callee prototype and arguments") ? static_cast
<void> (0) : __assert_fail ("(getValueType(DL, Ty) == Outs[OIdx].VT || (getValueType(DL, Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) && \"type mismatch between callee prototype and arguments\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1182, __PRETTY_FUNCTION__))

1182

"type mismatch between callee prototype and arguments")(((getValueType(DL, Ty) == Outs[OIdx].VT || (getValueType(DL,
Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
"type mismatch between callee prototype and arguments") ? static_cast
<void> (0) : __assert_fail ("(getValueType(DL, Ty) == Outs[OIdx].VT || (getValueType(DL, Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) && \"type mismatch between callee prototype and arguments\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1182, __PRETTY_FUNCTION__));

1183

// scalar type

1184

unsigned sz = 0;

1185

if (isa<IntegerType>(Ty)) {

1186

sz = cast<IntegerType>(Ty)->getBitWidth();

1187

if (sz < 32)

1188

sz = 32;

1189

} else if (isa<PointerType>(Ty)) {

1190

sz = PtrVT.getSizeInBits();

1191

} else if (Ty->isHalfTy())

1192

// PTX ABI requires all scalar parameters to be at least 32

1193

// bits in size. fp16 normally uses .b16 as its storage type

1194

// in PTX, so its size must be adjusted here, too.

1195

sz = 32;

1196

else

1197

sz = Ty->getPrimitiveSizeInBits();

1198

O << ".param .b" << sz << " ";

1199

O << "_";

1200

continue;

1201

}

1202

auto *PTy = dyn_cast<PointerType>(Ty);

1203

assert(PTy && "Param with byval attribute should be a pointer type")((PTy && "Param with byval attribute should be a pointer type"
) ? static_cast<void> (0) : __assert_fail ("PTy && \"Param with byval attribute should be a pointer type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1203, __PRETTY_FUNCTION__));

1204

Type *ETy = PTy->getElementType();

1205

1206

unsigned align = Outs[OIdx].Flags.getByValAlign();

1207

unsigned sz = DL.getTypeAllocSize(ETy);

1208

O << ".param .align " << align << " .b8 ";

1209

O << "_";

1210

O << "[" << sz << "]";

1211

}

1212

O << ");";

1213

return O.str();

1214

}

1215

1216

unsigned NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,

1217

const ImmutableCallSite *CS,

1218

Type *Ty, unsigned Idx,

1219

const DataLayout &DL) const {

1220

if (!CS) {

1221

// CallSite is zero, fallback to ABI type alignment

1222

return DL.getABITypeAlignment(Ty);

1223

}

1224

1225

unsigned Align = 0;

1226

const Value *DirectCallee = CS->getCalledFunction();

1227

1228

if (!DirectCallee) {

1229

// We don't have a direct function symbol, but that may be because of

1230

// constant cast instructions in the call.

1231

const Instruction *CalleeI = CS->getInstruction();

1232

assert(CalleeI && "Call target is not a function or derived value?")((CalleeI && "Call target is not a function or derived value?"
) ? static_cast<void> (0) : __assert_fail ("CalleeI && \"Call target is not a function or derived value?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1232, __PRETTY_FUNCTION__));

1233

1234

// With bitcast'd call targets, the instruction will be the call

1235

if (isa<CallInst>(CalleeI)) {

1236

// Check if we have call alignment metadata

1237

if (getAlign(*cast<CallInst>(CalleeI), Idx, Align))

1238

return Align;

1239

1240

const Value *CalleeV = cast<CallInst>(CalleeI)->getCalledValue();

1241

// Ignore any bitcast instructions

1242

while (isa<ConstantExpr>(CalleeV)) {

1243

const ConstantExpr *CE = cast<ConstantExpr>(CalleeV);

1244

if (!CE->isCast())

1245

break;

1246

// Look through the bitcast

1247

CalleeV = cast<ConstantExpr>(CalleeV)->getOperand(0);

1248

}

1249

1250

// We have now looked past all of the bitcasts. Do we finally have a

1251

// Function?

1252

if (isa<Function>(CalleeV))

1253

DirectCallee = CalleeV;

1254

}

1255

}

1256

1257

// Check for function alignment information if we found that the

1258

// ultimate target is a Function

1259

if (DirectCallee)

1260

if (getAlign(*cast<Function>(DirectCallee), Idx, Align))

1261

return Align;

1262

1263

// Call is indirect or alignment information is not available, fall back to

1264

// the ABI type alignment

1265

return DL.getABITypeAlignment(Ty);

1266

}

1267

1268

SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,

1269

SmallVectorImpl<SDValue> &InVals) const {

1270

SelectionDAG &DAG = CLI.DAG;

1271

SDLoc dl = CLI.DL;

1272

SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;

1273

SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;

1274

SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;

1275

SDValue Chain = CLI.Chain;

1276

SDValue Callee = CLI.Callee;

1277

bool &isTailCall = CLI.IsTailCall;

1278

ArgListTy &Args = CLI.getArgs();

1279

Type *retTy = CLI.RetTy;

'retTy' initialized here

→

1280

ImmutableCallSite *CS = CLI.CS;

1281

1282

bool isABI = (STI.getSmVersion() >= 20);

←

Assuming the condition is true

→

1283

1284

if (!isABI)

←

Taking false branch

→

1285

return Chain;

1286

MachineFunction &MF = DAG.getMachineFunction();

1287

const Function *F = MF.getFunction();

1288

auto &DL = MF.getDataLayout();

1289

1290

SDValue tempChain = Chain;

1291

Chain = DAG.getCALLSEQ_START(Chain,

1292

DAG.getIntPtrConstant(uniqueCallSite, dl, true),

1293

dl);

1294

SDValue InFlag = Chain.getValue(1);

1295

1296

unsigned paramCount = 0;

1297

// Args.size() and Outs.size() need not match.

1298

// Outs.size() will be larger

1299

// * if there is an aggregate argument with multiple fields (each field

1300

// showing up separately in Outs)

1301

// * if there is a vector argument with more than typical vector-length

1302

// elements (generally if more than 4) where each vector element is

1303

// individually present in Outs.

1304

// So a different index should be used for indexing into Outs/OutVals.

1305

// See similar issue in LowerFormalArguments.

1306

unsigned OIdx = 0;

1307

// Declare the .params or .reg need to pass values

1308

// to the function

1309

for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {

←

Assuming 'i' is equal to 'e'

→

←

Loop condition is false. Execution continues on line 1598

→

1310

EVT VT = Outs[OIdx].VT;

1311

Type *Ty = Args[i].Ty;

1312

1313

if (!Outs[OIdx].Flags.isByVal()) {

1314

if (Ty->isAggregateType()) {

1315

// aggregate

1316

SmallVector<EVT, 16> vtparts;

1317

SmallVector<uint64_t, 16> Offsets;

1318

ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts, &Offsets,

1319

0);

1320

1321

unsigned align =

1322

getArgumentAlignment(Callee, CS, Ty, paramCount + 1, DL);

1323

// declare .param .align <align> .b8 .param<n>[<size>];

1324

unsigned sz = DL.getTypeAllocSize(Ty);

1325

SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1326

SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, dl,

1327

MVT::i32),

1328

DAG.getConstant(paramCount, dl, MVT::i32),

1329

DAG.getConstant(sz, dl, MVT::i32),

1330

InFlag };

1331

Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,

1332

DeclareParamOps);

1333

InFlag = Chain.getValue(1);

1334

for (unsigned j = 0, je = vtparts.size(); j != je; ++j) {

1335

EVT elemtype = vtparts[j];

1336

unsigned ArgAlign = GreatestCommonDivisor64(align, Offsets[j]);

1337

if (elemtype.isInteger() && (sz < 8))

1338

sz = 8;

1339

SDValue StVal = OutVals[OIdx];

1340

if (elemtype.getSizeInBits() < 16) {

1341

StVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, StVal);

1342

}

1343

SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1344

SDValue CopyParamOps[] = { Chain,

1345

DAG.getConstant(paramCount, dl, MVT::i32),

1346

DAG.getConstant(Offsets[j], dl, MVT::i32),

1347

StVal, InFlag };

1348

Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,

1349

CopyParamVTs, CopyParamOps,

1350

elemtype, MachinePointerInfo(),

1351

ArgAlign);

1352

InFlag = Chain.getValue(1);

1353

++OIdx;

1354

}

1355

if (vtparts.size() > 0)

1356

--OIdx;

1357

++paramCount;

1358

continue;

1359

}

1360

if (Ty->isVectorTy()) {

1361

EVT ObjectVT = getValueType(DL, Ty);

1362

unsigned align =

1363

getArgumentAlignment(Callee, CS, Ty, paramCount + 1, DL);

1364

// declare .param .align <align> .b8 .param<n>[<size>];

1365

unsigned sz = DL.getTypeAllocSize(Ty);

1366

SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1367

SDValue DeclareParamOps[] = { Chain,

1368

DAG.getConstant(align, dl, MVT::i32),

1369

DAG.getConstant(paramCount, dl, MVT::i32),

1370

DAG.getConstant(sz, dl, MVT::i32),

1371

InFlag };

1372

Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,

1373

DeclareParamOps);

1374

InFlag = Chain.getValue(1);

1375

unsigned NumElts = ObjectVT.getVectorNumElements();

1376

EVT EltVT = ObjectVT.getVectorElementType();

1377

EVT MemVT = EltVT;

1378

bool NeedExtend = false;

1379

if (EltVT.getSizeInBits() < 16) {

1380

NeedExtend = true;

1381

EltVT = MVT::i16;

1382

}

1383

1384

// V1 store

1385

if (NumElts == 1) {

1386

SDValue Elt = OutVals[OIdx++];

1387

if (NeedExtend)

1388

Elt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt);

1389

1390

SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1391

SDValue CopyParamOps[] = { Chain,

1392

DAG.getConstant(paramCount, dl, MVT::i32),

1393

DAG.getConstant(0, dl, MVT::i32), Elt,

1394

InFlag };

1395

Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,

1396

CopyParamVTs, CopyParamOps,

1397

MemVT, MachinePointerInfo());

1398

InFlag = Chain.getValue(1);

1399

} else if (NumElts == 2) {

1400

SDValue Elt0 = OutVals[OIdx++];

1401

SDValue Elt1 = OutVals[OIdx++];

1402

if (NeedExtend) {

1403

Elt0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt0);

1404

Elt1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt1);

1405

}

1406

1407

SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1408

SDValue CopyParamOps[] = { Chain,

1409

DAG.getConstant(paramCount, dl, MVT::i32),

1410

DAG.getConstant(0, dl, MVT::i32), Elt0,

1411

Elt1, InFlag };

1412

Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParamV2, dl,

1413

CopyParamVTs, CopyParamOps,

1414

MemVT, MachinePointerInfo());

1415

InFlag = Chain.getValue(1);

1416

} else {

1417

unsigned curOffset = 0;

1418

// V4 stores

1419

// We have at least 4 elements (<3 x Ty> expands to 4 elements) and

1420

// the

1421

// vector will be expanded to a power of 2 elements, so we know we can

1422

// always round up to the next multiple of 4 when creating the vector

1423

// stores.

1424

// e.g. 4 elem => 1 st.v4

1425

// 6 elem => 2 st.v4

1426

// 8 elem => 2 st.v4

1427

// 11 elem => 3 st.v4

1428

unsigned VecSize = 4;

1429

if (EltVT.getSizeInBits() == 64)

1430

VecSize = 2;

1431

1432

// This is potentially only part of a vector, so assume all elements

1433

// are packed together.

1434

unsigned PerStoreOffset = MemVT.getStoreSizeInBits() / 8 * VecSize;

1435

1436

for (unsigned i = 0; i < NumElts; i += VecSize) {

1437

// Get values

1438

SDValue StoreVal;

1439

SmallVector<SDValue, 8> Ops;

1440

Ops.push_back(Chain);

1441

Ops.push_back(DAG.getConstant(paramCount, dl, MVT::i32));

1442

Ops.push_back(DAG.getConstant(curOffset, dl, MVT::i32));

1443

1444

unsigned Opc = NVPTXISD::StoreParamV2;

1445

1446

StoreVal = OutVals[OIdx++];

1447

if (NeedExtend)

1448

StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);

1449

Ops.push_back(StoreVal);

1450

1451

if (i + 1 < NumElts) {

1452

StoreVal = OutVals[OIdx++];

1453

if (NeedExtend)

1454

StoreVal =

1455

DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);

1456

} else {

1457

StoreVal = DAG.getUNDEF(EltVT);

1458

}

1459

Ops.push_back(StoreVal);

1460

1461

if (VecSize == 4) {

1462

Opc = NVPTXISD::StoreParamV4;

1463

if (i + 2 < NumElts) {

1464

StoreVal = OutVals[OIdx++];

1465

if (NeedExtend)

1466

StoreVal =

1467

DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);

1468

} else {

1469

StoreVal = DAG.getUNDEF(EltVT);

1470

}

1471

Ops.push_back(StoreVal);

1472

1473

if (i + 3 < NumElts) {

1474

StoreVal = OutVals[OIdx++];

1475

if (NeedExtend)

1476

StoreVal =

1477

DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);

1478

} else {

1479

StoreVal = DAG.getUNDEF(EltVT);

1480

}

1481

Ops.push_back(StoreVal);

1482

}

1483

1484

Ops.push_back(InFlag);

1485

1486

SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1487

Chain = DAG.getMemIntrinsicNode(Opc, dl, CopyParamVTs, Ops,

1488

MemVT, MachinePointerInfo());

1489

InFlag = Chain.getValue(1);

1490

curOffset += PerStoreOffset;

1491

}

1492

}

1493

++paramCount;

1494

--OIdx;

1495

continue;

1496

}

1497

// Plain scalar

1498

// for ABI, declare .param .b<size> .param<n>;

1499

unsigned sz = VT.getSizeInBits();

1500

bool needExtend = false;

1501

if (VT.isInteger()) {

1502

if (sz < 16)

1503

needExtend = true;

1504

if (sz < 32)

1505

sz = 32;

1506

} else if (VT.isFloatingPoint() && sz < 32)

1507

// PTX ABI requires all scalar parameters to be at least 32

1508

// bits in size. fp16 normally uses .b16 as its storage type

1509

// in PTX, so its size must be adjusted here, too.

1510

sz = 32;

1511

SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1512

SDValue DeclareParamOps[] = { Chain,

1513

DAG.getConstant(paramCount, dl, MVT::i32),

1514

DAG.getConstant(sz, dl, MVT::i32),

1515

DAG.getConstant(0, dl, MVT::i32), InFlag };

1516

Chain = DAG.getNode(NVPTXISD::DeclareScalarParam, dl, DeclareParamVTs,

1517

DeclareParamOps);

1518

InFlag = Chain.getValue(1);

1519

SDValue OutV = OutVals[OIdx];

1520

if (needExtend) {

1521

// zext/sext i1 to i16

1522

unsigned opc = ISD::ZERO_EXTEND;

1523

if (Outs[OIdx].Flags.isSExt())

1524

opc = ISD::SIGN_EXTEND;

1525

OutV = DAG.getNode(opc, dl, MVT::i16, OutV);

1526

}

1527

SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1528

SDValue CopyParamOps[] = { Chain,

1529

DAG.getConstant(paramCount, dl, MVT::i32),

1530

DAG.getConstant(0, dl, MVT::i32), OutV,

1531

InFlag };

1532

1533

unsigned opcode = NVPTXISD::StoreParam;

1534

if (Outs[OIdx].Flags.isZExt() && VT.getSizeInBits() < 32)

1535

opcode = NVPTXISD::StoreParamU32;

1536

else if (Outs[OIdx].Flags.isSExt() && VT.getSizeInBits() < 32)

1537

opcode = NVPTXISD::StoreParamS32;

1538

Chain = DAG.getMemIntrinsicNode(opcode, dl, CopyParamVTs, CopyParamOps,

1539

VT, MachinePointerInfo());

1540

1541

InFlag = Chain.getValue(1);

1542

++paramCount;

1543

continue;

1544

}

1545

// struct or vector

1546

SmallVector<EVT, 16> vtparts;

1547

SmallVector<uint64_t, 16> Offsets;

1548

auto *PTy = dyn_cast<PointerType>(Args[i].Ty);

1549

assert(PTy && "Type of a byval parameter should be pointer")((PTy && "Type of a byval parameter should be pointer"
) ? static_cast<void> (0) : __assert_fail ("PTy && \"Type of a byval parameter should be pointer\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1549, __PRETTY_FUNCTION__));

1550

ComputePTXValueVTs(*this, DAG.getDataLayout(), PTy->getElementType(),

1551

vtparts, &Offsets, 0);

1552

1553

// declare .param .align <align> .b8 .param<n>[<size>];

1554

unsigned sz = Outs[OIdx].Flags.getByValSize();

1555

SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1556

unsigned ArgAlign = Outs[OIdx].Flags.getByValAlign();

1557

// The ByValAlign in the Outs[OIdx].Flags is alway set at this point,

1558

// so we don't need to worry about natural alignment or not.

1559

// See TargetLowering::LowerCallTo().

1560

1561

// Enforce minumum alignment of 4 to work around ptxas miscompile

1562

// for sm_50+. See corresponding alignment adjustment in

1563

// emitFunctionParamList() for details.

1564

if (ArgAlign < 4)

1565

ArgAlign = 4;

1566

SDValue DeclareParamOps[] = {Chain, DAG.getConstant(ArgAlign, dl, MVT::i32),

1567

DAG.getConstant(paramCount, dl, MVT::i32),

1568

DAG.getConstant(sz, dl, MVT::i32), InFlag};

1569

Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,

1570

DeclareParamOps);

1571

InFlag = Chain.getValue(1);

1572

for (unsigned j = 0, je = vtparts.size(); j != je; ++j) {

1573

EVT elemtype = vtparts[j];

1574

int curOffset = Offsets[j];

1575

unsigned PartAlign = GreatestCommonDivisor64(ArgAlign, curOffset);

1576

auto PtrVT = getPointerTy(DAG.getDataLayout());

1577

SDValue srcAddr = DAG.getNode(ISD::ADD, dl, PtrVT, OutVals[OIdx],

1578

DAG.getConstant(curOffset, dl, PtrVT));

1579

SDValue theVal = DAG.getLoad(elemtype, dl, tempChain, srcAddr,

1580

MachinePointerInfo(), PartAlign);

1581

if (elemtype.getSizeInBits() < 16) {

1582

theVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, theVal);

1583

}

1584

SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1585

SDValue CopyParamOps[] = { Chain,

1586

DAG.getConstant(paramCount, dl, MVT::i32),

1587

DAG.getConstant(curOffset, dl, MVT::i32),

1588

theVal, InFlag };

1589

Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl, CopyParamVTs,

1590

CopyParamOps, elemtype,

1591

MachinePointerInfo());

1592

1593

InFlag = Chain.getValue(1);

1594

}

1595

++paramCount;

1596

}

1597

1598

GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());

1599

unsigned retAlignment = 0;

1600

1601

// Handle Result

1602

if (Ins.size() > 0) {

←

Assuming the condition is false

→

←

Taking false branch

→

1603

SmallVector<EVT, 16> resvtparts;

1604

ComputeValueVTs(*this, DL, retTy, resvtparts);

1605

1606

// Declare

1607

// .param .align 16 .b8 retval0[<size-in-bytes>], or

1608

// .param .b<size-in-bits> retval0

1609

unsigned resultsz = DL.getTypeAllocSizeInBits(retTy);

1610

// Emit ".param .b<size-in-bits> retval0" instead of byte arrays only for

1611

// these three types to match the logic in

1612

// NVPTXAsmPrinter::printReturnValStr and NVPTXTargetLowering::getPrototype.

1613

// Plus, this behavior is consistent with nvcc's.

1614

if (retTy->isFloatingPointTy() || retTy->isIntegerTy() ||

1615

retTy->isPointerTy()) {

1616

// Scalar needs to be at least 32bit wide

1617

if (resultsz < 32)

1618

resultsz = 32;

1619

SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1620

SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),

1621

DAG.getConstant(resultsz, dl, MVT::i32),

1622

DAG.getConstant(0, dl, MVT::i32), InFlag };

1623

Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, DeclareRetVTs,

1624

DeclareRetOps);

1625

InFlag = Chain.getValue(1);

1626

} else {

1627

retAlignment = getArgumentAlignment(Callee, CS, retTy, 0, DL);

1628

SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1629

SDValue DeclareRetOps[] = { Chain,

1630

DAG.getConstant(retAlignment, dl, MVT::i32),

1631

DAG.getConstant(resultsz / 8, dl, MVT::i32),

1632

DAG.getConstant(0, dl, MVT::i32), InFlag };

1633

Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl, DeclareRetVTs,

1634

DeclareRetOps);

1635

InFlag = Chain.getValue(1);

1636

}

1637

}

1638

1639

if (!Func) {

←

Taking true branch

→

1640

// This is indirect function call case : PTX requires a prototype of the

1641

// form

1642

// proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);

1643

// to be emitted, and the label has to used as the last arg of call

1644

// instruction.

1645

// The prototype is embedded in a string and put as the operand for a

1646

// CallPrototype SDNode which will print out to the value of the string.

1647

SDVTList ProtoVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1648

std::string Proto =

1649

getPrototype(DAG.getDataLayout(), retTy, Args, Outs, retAlignment, CS);

1650

const char *ProtoStr =

1651

nvTM->getManagedStrPool()->getManagedString(Proto.c_str())->c_str();

1652

SDValue ProtoOps[] = {

1653

Chain, DAG.getTargetExternalSymbol(ProtoStr, MVT::i32), InFlag,

1654

};

1655

Chain = DAG.getNode(NVPTXISD::CallPrototype, dl, ProtoVTs, ProtoOps);

1656

InFlag = Chain.getValue(1);

1657

}

1658

// Op to just print "call"

1659

SDVTList PrintCallVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1660

SDValue PrintCallOps[] = {

1661

Chain, DAG.getConstant((Ins.size() == 0) ? 0 : 1, dl, MVT::i32), InFlag

←

Assuming the condition is false

→

←

'?' condition is false

→

1662

};

1663

// We model convergent calls as separate opcodes.

1664

unsigned Opcode = Func ? NVPTXISD::PrintCallUni : NVPTXISD::PrintCall;

←

'?' condition is false

→

1665

if (CLI.IsConvergent)

←

Assuming the condition is false

→

←

Taking false branch

→

1666

Opcode = Opcode == NVPTXISD::PrintCallUni ? NVPTXISD::PrintConvergentCallUni

1667

: NVPTXISD::PrintConvergentCall;

1668

Chain = DAG.getNode(Opcode, dl, PrintCallVTs, PrintCallOps);

1669

InFlag = Chain.getValue(1);

1670

1671

// Ops to print out the function name

1672

SDVTList CallVoidVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1673

SDValue CallVoidOps[] = { Chain, Callee, InFlag };

1674

Chain = DAG.getNode(NVPTXISD::CallVoid, dl, CallVoidVTs, CallVoidOps);

1675

InFlag = Chain.getValue(1);

1676

1677

// Ops to print out the param list

1678

SDVTList CallArgBeginVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1679

SDValue CallArgBeginOps[] = { Chain, InFlag };

1680

Chain = DAG.getNode(NVPTXISD::CallArgBegin, dl, CallArgBeginVTs,

1681

CallArgBeginOps);

1682

InFlag = Chain.getValue(1);

1683

1684

for (unsigned i = 0, e = paramCount; i != e; ++i) {

←

Loop condition is false. Execution continues on line 1696

→

1685

unsigned opcode;

1686

if (i == (e - 1))

1687

opcode = NVPTXISD::LastCallArg;

1688

else

1689

opcode = NVPTXISD::CallArg;

1690

SDVTList CallArgVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1691

SDValue CallArgOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32),

1692

DAG.getConstant(i, dl, MVT::i32), InFlag };

1693

Chain = DAG.getNode(opcode, dl, CallArgVTs, CallArgOps);

1694

InFlag = Chain.getValue(1);

1695

}

1696

SDVTList CallArgEndVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1697

SDValue CallArgEndOps[] = { Chain,

1698

DAG.getConstant(Func ? 1 : 0, dl, MVT::i32),

←

'?' condition is false

→

1699

InFlag };

1700

Chain = DAG.getNode(NVPTXISD::CallArgEnd, dl, CallArgEndVTs, CallArgEndOps);

1701

InFlag = Chain.getValue(1);

1702

1703

if (!Func) {

←

Taking true branch

→

1704

SDVTList PrototypeVTs = DAG.getVTList(MVT::Other, MVT::Glue);

1705

SDValue PrototypeOps[] = { Chain,

1706

DAG.getConstant(uniqueCallSite, dl, MVT::i32),

1707

InFlag };

1708

Chain = DAG.getNode(NVPTXISD::Prototype, dl, PrototypeVTs, PrototypeOps);

1709

InFlag = Chain.getValue(1);

1710

}

1711

1712

// Generate loads from param memory/moves from registers for result

1713

if (Ins.size() > 0) {

←

Assuming the condition is true

→

←

Taking true branch

→

1714

if (retTy && retTy->isVectorTy()) {

←

Assuming 'retTy' is null

→

1715

EVT ObjectVT = getValueType(DL, retTy);

1716

unsigned NumElts = ObjectVT.getVectorNumElements();

1717

EVT EltVT = ObjectVT.getVectorElementType();

1718

assert(STI.getTargetLowering()->getNumRegisters(F->getContext(),((STI.getTargetLowering()->getNumRegisters(F->getContext
(), ObjectVT) == NumElts && "Vector was not scalarized"
) ? static_cast<void> (0) : __assert_fail ("STI.getTargetLowering()->getNumRegisters(F->getContext(), ObjectVT) == NumElts && \"Vector was not scalarized\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1720, __PRETTY_FUNCTION__))

1719

ObjectVT) == NumElts &&((STI.getTargetLowering()->getNumRegisters(F->getContext
(), ObjectVT) == NumElts && "Vector was not scalarized"
) ? static_cast<void> (0) : __assert_fail ("STI.getTargetLowering()->getNumRegisters(F->getContext(), ObjectVT) == NumElts && \"Vector was not scalarized\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1720, __PRETTY_FUNCTION__))

1720

"Vector was not scalarized")((STI.getTargetLowering()->getNumRegisters(F->getContext
(), ObjectVT) == NumElts && "Vector was not scalarized"
) ? static_cast<void> (0) : __assert_fail ("STI.getTargetLowering()->getNumRegisters(F->getContext(), ObjectVT) == NumElts && \"Vector was not scalarized\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1720, __PRETTY_FUNCTION__));

1721

unsigned sz = EltVT.getSizeInBits();

1722

bool needTruncate = sz < 8;

1723

1724

if (NumElts == 1) {

1725

// Just a simple load

1726

SmallVector<EVT, 4> LoadRetVTs;

1727

if (EltVT == MVT::i1 || EltVT == MVT::i8) {

1728

// If loading i1/i8 result, generate

1729

// load.b8 i16

1730

// if i1

1731

// trunc i16 to i1

1732

LoadRetVTs.push_back(MVT::i16);

1733

} else

1734

LoadRetVTs.push_back(EltVT);

1735

LoadRetVTs.push_back(MVT::Other);

1736

LoadRetVTs.push_back(MVT::Glue);

1737

SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, dl, MVT::i32),

1738

DAG.getConstant(0, dl, MVT::i32), InFlag};

1739

SDValue retval = DAG.getMemIntrinsicNode(

1740

NVPTXISD::LoadParam, dl,

1741

DAG.getVTList(LoadRetVTs), LoadRetOps, EltVT, MachinePointerInfo());

1742

Chain = retval.getValue(1);

1743

InFlag = retval.getValue(2);

1744

SDValue Ret0 = retval;

1745

if (needTruncate)

1746

Ret0 = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Ret0);

1747

InVals.push_back(Ret0);

1748

} else if (NumElts == 2) {

1749

// LoadV2

1750

SmallVector<EVT, 4> LoadRetVTs;

1751

if (EltVT == MVT::i1 || EltVT == MVT::i8) {

1752

// If loading i1/i8 result, generate

1753

// load.b8 i16

1754

// if i1

1755

// trunc i16 to i1

1756

LoadRetVTs.push_back(MVT::i16);

1757

LoadRetVTs.push_back(MVT::i16);

1758

} else {

1759

LoadRetVTs.push_back(EltVT);

1760

LoadRetVTs.push_back(EltVT);

1761

}

1762

LoadRetVTs.push_back(MVT::Other);

1763

LoadRetVTs.push_back(MVT::Glue);

1764

SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, dl, MVT::i32),

1765

DAG.getConstant(0, dl, MVT::i32), InFlag};

1766

SDValue retval = DAG.getMemIntrinsicNode(

1767

NVPTXISD::LoadParamV2, dl,

1768

DAG.getVTList(LoadRetVTs), LoadRetOps, EltVT, MachinePointerInfo());

1769

Chain = retval.getValue(2);

1770

InFlag = retval.getValue(3);

1771

SDValue Ret0 = retval.getValue(0);

1772

SDValue Ret1 = retval.getValue(1);

1773

if (needTruncate) {

1774

Ret0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ret0);

1775

InVals.push_back(Ret0);

1776

Ret1 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ret1);

1777

InVals.push_back(Ret1);

1778

} else {

1779

InVals.push_back(Ret0);

1780

InVals.push_back(Ret1);

1781

}

1782

} else {

1783

// Split into N LoadV4

1784

unsigned Ofst = 0;

1785

unsigned VecSize = 4;

1786

unsigned Opc = NVPTXISD::LoadParamV4;

1787

if (EltVT.getSizeInBits() == 64) {

1788

VecSize = 2;

1789

Opc = NVPTXISD::LoadParamV2;

1790

}

1791

EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, VecSize);

1792

for (unsigned i = 0; i < NumElts; i += VecSize) {

1793

SmallVector<EVT, 8> LoadRetVTs;

1794

if (EltVT == MVT::i1 || EltVT == MVT::i8) {

1795

// If loading i1/i8 result, generate

1796

// load.b8 i16

1797

// if i1

1798

// trunc i16 to i1

1799

for (unsigned j = 0; j < VecSize; ++j)

1800

LoadRetVTs.push_back(MVT::i16);

1801

} else {

1802

for (unsigned j = 0; j < VecSize; ++j)

1803

LoadRetVTs.push_back(EltVT);

1804

}

1805

LoadRetVTs.push_back(MVT::Other);

1806

LoadRetVTs.push_back(MVT::Glue);

1807

SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, dl, MVT::i32),

1808

DAG.getConstant(Ofst, dl, MVT::i32), InFlag};

1809

SDValue retval = DAG.getMemIntrinsicNode(

1810

Opc, dl, DAG.getVTList(LoadRetVTs),

1811

LoadRetOps, EltVT, MachinePointerInfo());

1812

if (VecSize == 2) {

1813

Chain = retval.getValue(2);

1814

InFlag = retval.getValue(3);

1815

} else {

1816

Chain = retval.getValue(4);

1817

InFlag = retval.getValue(5);

1818

}

1819

1820

for (unsigned j = 0; j < VecSize; ++j) {

1821

if (i + j >= NumElts)

1822

break;

1823

SDValue Elt = retval.getValue(j);

1824

if (needTruncate)

1825

Elt = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);

1826

InVals.push_back(Elt);

1827

}

1828

Ofst += DL.getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));

1829

}

1830

}

1831

} else {

1832

SmallVector<EVT, 16> VTs;

1833

SmallVector<uint64_t, 16> Offsets;

1834

auto &DL = DAG.getDataLayout();

1835

ComputePTXValueVTs(*this, DL, retTy, VTs, &Offsets, 0);

1836

assert(VTs.size() == Ins.size() && "Bad value decomposition")((VTs.size() == Ins.size() && "Bad value decomposition"
) ? static_cast<void> (0) : __assert_fail ("VTs.size() == Ins.size() && \"Bad value decomposition\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1836, __PRETTY_FUNCTION__));

1837

unsigned RetAlign = getArgumentAlignment(Callee, CS, retTy, 0, DL);

1838

for (unsigned i = 0, e = Ins.size(); i != e; ++i) {

←

Assuming 'i' is not equal to 'e'

→

←

Loop condition is true. Entering loop body

→

1839

unsigned sz = VTs[i].getSizeInBits();

1840

unsigned AlignI = GreatestCommonDivisor64(RetAlign, Offsets[i]);

1841

bool needTruncate = false;

1842

if (VTs[i].isInteger() && sz < 8) {

←

Assuming the condition is false

→

1843

sz = 8;

1844

needTruncate = true;

1845

}

1846

1847

SmallVector<EVT, 4> LoadRetVTs;

1848

EVT TheLoadType = VTs[i];

1849

if (retTy->isIntegerTy() && DL.getTypeAllocSizeInBits(retTy) < 32) {

←

Called C++ object pointer is null

1850

// This is for integer types only, and specifically not for

1851

// aggregates.

1852

LoadRetVTs.push_back(MVT::i32);

1853

TheLoadType = MVT::i32;

1854

needTruncate = true;

1855

} else if (sz < 16) {

1856

// If loading i1/i8 result, generate

1857

// load i8 (-> i16)

1858

// trunc i16 to i1/i8

1859

1860

// FIXME: Do we need to set needTruncate to true here, too? We could

1861

// not figure out what this branch is for in D17872, so we left it

1862

// alone. The comment above about loading i1/i8 may be wrong, as the

1863

// branch above seems to cover integers of size < 32.

1864

LoadRetVTs.push_back(MVT::i16);

1865

} else

1866

LoadRetVTs.push_back(Ins[i].VT);

1867

LoadRetVTs.push_back(MVT::Other);

1868

LoadRetVTs.push_back(MVT::Glue);

1869

1870

SDValue LoadRetOps[] = {Chain, DAG.getConstant(1, dl, MVT::i32),

1871

DAG.getConstant(Offsets[i], dl, MVT::i32),

1872

InFlag};

1873

SDValue retval = DAG.getMemIntrinsicNode(

1874

NVPTXISD::LoadParam, dl,

1875

DAG.getVTList(LoadRetVTs), LoadRetOps,

1876

TheLoadType, MachinePointerInfo(), AlignI);

1877

Chain = retval.getValue(1);

1878

InFlag = retval.getValue(2);

1879

SDValue Ret0 = retval.getValue(0);

1880

if (needTruncate)

1881

Ret0 = DAG.getNode(ISD::TRUNCATE, dl, Ins[i].VT, Ret0);

1882

InVals.push_back(Ret0);

1883

}

1884

}

1885

}

1886

1887

Chain = DAG.getCALLSEQ_END(Chain,

1888

DAG.getIntPtrConstant(uniqueCallSite, dl, true),

1889

DAG.getIntPtrConstant(uniqueCallSite + 1, dl,

1890

true),

1891

InFlag, dl);

1892

uniqueCallSite++;

1893

1894

// set isTailCall to false for now, until we figure out how to express

1895

// tail call optimization in PTX

1896

isTailCall = false;

1897

return Chain;

1898

}

1899

1900

// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()

1901

// (see LegalizeDAG.cpp). This is slow and uses local memory.

1902

// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5

1903

SDValue

1904

NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {

1905

SDNode *Node = Op.getNode();

1906

SDLoc dl(Node);

1907

SmallVector<SDValue, 8> Ops;

1908

unsigned NumOperands = Node->getNumOperands();

1909

for (unsigned i = 0; i < NumOperands; ++i) {

1910

SDValue SubOp = Node->getOperand(i);

1911

EVT VVT = SubOp.getNode()->getValueType(0);

1912

EVT EltVT = VVT.getVectorElementType();

1913

unsigned NumSubElem = VVT.getVectorNumElements();

1914

for (unsigned j = 0; j < NumSubElem; ++j) {

1915

Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,

1916

DAG.getIntPtrConstant(j, dl)));

1917

}

1918

}

1919

return DAG.getBuildVector(Node->getValueType(0), dl, Ops);

1920

}

1921

1922

/// LowerShiftRightParts - Lower SRL_PARTS, SRA_PARTS, which

1923

/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift

1924

/// amount, or

1925

/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift

1926

/// amount.

1927

SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,

1928

SelectionDAG &DAG) const {

1929

assert(Op.getNumOperands() == 3 && "Not a double-shift!")((Op.getNumOperands() == 3 && "Not a double-shift!") ?
static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 3 && \"Not a double-shift!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1929, __PRETTY_FUNCTION__));

1930

assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS)((Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::
SRL_PARTS) ? static_cast<void> (0) : __assert_fail ("Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1930, __PRETTY_FUNCTION__));

1931

1932

EVT VT = Op.getValueType();

1933

unsigned VTBits = VT.getSizeInBits();

1934

SDLoc dl(Op);

1935

SDValue ShOpLo = Op.getOperand(0);

1936

SDValue ShOpHi = Op.getOperand(1);

1937

SDValue ShAmt = Op.getOperand(2);

1938

unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;

1939

1940

if (VTBits == 32 && STI.getSmVersion() >= 35) {

1941

// For 32bit and sm35, we can use the funnel shift 'shf' instruction.

1942

// {dHi, dLo} = {aHi, aLo} >> Amt

1943

// dHi = aHi >> Amt

1944

// dLo = shf.r.clamp aLo, aHi, Amt

1945

1946

SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);

1947

SDValue Lo = DAG.getNode(NVPTXISD::FUN_SHFR_CLAMP, dl, VT, ShOpLo, ShOpHi,

1948

ShAmt);

1949

1950

SDValue Ops[2] = { Lo, Hi };

1951

return DAG.getMergeValues(Ops, dl);

1952

}

1953

else {

1954

// {dHi, dLo} = {aHi, aLo} >> Amt

1955

// - if (Amt>=size) then

1956

// dLo = aHi >> (Amt-size)

1957

// dHi = aHi >> Amt (this is either all 0 or all 1)

1958

// else

1959

// dLo = (aLo >>logic Amt) | (aHi << (size-Amt))

1960

// dHi = aHi >> Amt

1961

1962

SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,

1963

DAG.getConstant(VTBits, dl, MVT::i32),

1964

ShAmt);

1965

SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);

1966

SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,

1967

DAG.getConstant(VTBits, dl, MVT::i32));

1968

SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);

1969

SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);

1970

SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);

1971

1972

SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,

1973

DAG.getConstant(VTBits, dl, MVT::i32),

1974

ISD::SETGE);

1975

SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);

1976

SDValue Lo = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);

1977

1978

SDValue Ops[2] = { Lo, Hi };

1979

return DAG.getMergeValues(Ops, dl);

1980

}

1981

}

1982

1983

/// LowerShiftLeftParts - Lower SHL_PARTS, which

1984

/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift

1985

/// amount, or

1986

/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift

1987

/// amount.

1988

SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,

1989

SelectionDAG &DAG) const {

1990

1991

assert(Op.getOpcode() == ISD::SHL_PARTS)((Op.getOpcode() == ISD::SHL_PARTS) ? static_cast<void>
(0) : __assert_fail ("Op.getOpcode() == ISD::SHL_PARTS", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 1991, __PRETTY_FUNCTION__));

1992

1993

EVT VT = Op.getValueType();

1994

unsigned VTBits = VT.getSizeInBits();

1995

SDLoc dl(Op);

1996

SDValue ShOpLo = Op.getOperand(0);

1997

SDValue ShOpHi = Op.getOperand(1);

1998

SDValue ShAmt = Op.getOperand(2);

1999

2000

if (VTBits == 32 && STI.getSmVersion() >= 35) {

2001

// For 32bit and sm35, we can use the funnel shift 'shf' instruction.

2002

// {dHi, dLo} = {aHi, aLo} << Amt

2003

// dHi = shf.l.clamp aLo, aHi, Amt

2004

// dLo = aLo << Amt

2005

2006

SDValue Hi = DAG.getNode(NVPTXISD::FUN_SHFL_CLAMP, dl, VT, ShOpLo, ShOpHi,

2007

ShAmt);

2008

SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);

2009

2010

SDValue Ops[2] = { Lo, Hi };

2011

return DAG.getMergeValues(Ops, dl);

2012

}

2013

else {

2014

// {dHi, dLo} = {aHi, aLo} << Amt

2015

// - if (Amt>=size) then

2016

// dLo = aLo << Amt (all 0)

2017

// dLo = aLo << (Amt-size)

2018

// else

2019

// dLo = aLo << Amt

2020

// dHi = (aHi << Amt) | (aLo >> (size-Amt))

2021

2022

SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,

2023

DAG.getConstant(VTBits, dl, MVT::i32),

2024

ShAmt);

2025

SDValue Tmp1 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);

2026

SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,

2027

DAG.getConstant(VTBits, dl, MVT::i32));

2028

SDValue Tmp2 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);

2029

SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);

2030

SDValue TrueVal = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);

2031

2032

SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,

2033

DAG.getConstant(VTBits, dl, MVT::i32),

2034

ISD::SETGE);

2035

SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);

2036

SDValue Hi = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);

2037

2038

SDValue Ops[2] = { Lo, Hi };

2039

return DAG.getMergeValues(Ops, dl);

2040

}

2041

}

2042

2043

SDValue

2044

NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {

2045

switch (Op.getOpcode()) {

2046

case ISD::RETURNADDR:

2047

return SDValue();

2048

case ISD::FRAMEADDR:

2049

return SDValue();

2050

case ISD::GlobalAddress:

2051

return LowerGlobalAddress(Op, DAG);

2052

case ISD::INTRINSIC_W_CHAIN:

2053

return Op;

2054

case ISD::BUILD_VECTOR:

2055

case ISD::EXTRACT_SUBVECTOR:

2056

return Op;

2057

case ISD::CONCAT_VECTORS:

2058

return LowerCONCAT_VECTORS(Op, DAG);

2059

case ISD::STORE:

2060

return LowerSTORE(Op, DAG);

2061

case ISD::LOAD:

2062

return LowerLOAD(Op, DAG);

2063

case ISD::SHL_PARTS:

2064

return LowerShiftLeftParts(Op, DAG);

2065

case ISD::SRA_PARTS:

2066

case ISD::SRL_PARTS:

2067

return LowerShiftRightParts(Op, DAG);

2068

case ISD::SELECT:

2069

return LowerSelect(Op, DAG);

2070

default:

2071

llvm_unreachable("Custom lowering not defined for operation")::llvm::llvm_unreachable_internal("Custom lowering not defined for operation"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2071);

2072

}

2073

}

2074

2075

SDValue NVPTXTargetLowering::LowerSelect(SDValue Op, SelectionDAG &DAG) const {

2076

SDValue Op0 = Op->getOperand(0);

2077

SDValue Op1 = Op->getOperand(1);

2078

SDValue Op2 = Op->getOperand(2);

2079

SDLoc DL(Op.getNode());

2080

2081

assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1")((Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType() == MVT::i1 && \"Custom lowering enabled only for i1\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2081, __PRETTY_FUNCTION__));

2082

2083

Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);

2084

Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);

2085

SDValue Select = DAG.getNode(ISD::SELECT, DL, MVT::i32, Op0, Op1, Op2);

2086

SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Select);

2087

2088

return Trunc;

2089

}

2090

2091

SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {

2092

if (Op.getValueType() == MVT::i1)

2093

return LowerLOADi1(Op, DAG);

2094

else

2095

return SDValue();

2096

}

2097

2098

// v = ld i1* addr

2099

// =>

2100

// v1 = ld i8* addr (-> i16)

2101

// v = trunc i16 to i1

2102

SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const {

2103

SDNode *Node = Op.getNode();

2104

LoadSDNode *LD = cast<LoadSDNode>(Node);

2105

SDLoc dl(Node);

2106

assert(LD->getExtensionType() == ISD::NON_EXTLOAD)((LD->getExtensionType() == ISD::NON_EXTLOAD) ? static_cast
<void> (0) : __assert_fail ("LD->getExtensionType() == ISD::NON_EXTLOAD"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2106, __PRETTY_FUNCTION__));

2107

assert(Node->getValueType(0) == MVT::i1 &&((Node->getValueType(0) == MVT::i1 && "Custom lowering for i1 load only"
) ? static_cast<void> (0) : __assert_fail ("Node->getValueType(0) == MVT::i1 && \"Custom lowering for i1 load only\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2108, __PRETTY_FUNCTION__))

2108

"Custom lowering for i1 load only")((Node->getValueType(0) == MVT::i1 && "Custom lowering for i1 load only"
) ? static_cast<void> (0) : __assert_fail ("Node->getValueType(0) == MVT::i1 && \"Custom lowering for i1 load only\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2108, __PRETTY_FUNCTION__));

2109

SDValue newLD = DAG.getLoad(MVT::i16, dl, LD->getChain(), LD->getBasePtr(),

2110

LD->getPointerInfo(), LD->getAlignment(),

2111

LD->getMemOperand()->getFlags());

2112

SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);

2113

// The legalizer (the caller) is expecting two values from the legalized

2114

// load, so we build a MergeValues node for it. See ExpandUnalignedLoad()

2115

// in LegalizeDAG.cpp which also uses MergeValues.

2116

SDValue Ops[] = { result, LD->getChain() };

2117

return DAG.getMergeValues(Ops, dl);

2118

}

2119

2120

SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {

2121

EVT ValVT = Op.getOperand(1).getValueType();

2122

switch (ValVT.getSimpleVT().SimpleTy) {

2123

case MVT::i1:

2124

return LowerSTOREi1(Op, DAG);

2125

default:

2126

if (ValVT.isVector())

2127

return LowerSTOREVector(Op, DAG);

2128

else

2129

return SDValue();

2130

}

2131

}

2132

2133

SDValue

2134

NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {

2135

SDNode *N = Op.getNode();

2136

SDValue Val = N->getOperand(1);

2137

SDLoc DL(N);

2138

EVT ValVT = Val.getValueType();

2139

2140

if (ValVT.isVector()) {

2141

// We only handle "native" vector sizes for now, e.g. <4 x double> is not

2142

// legal. We can (and should) split that into 2 stores of <2 x double> here

2143

// but I'm leaving that as a TODO for now.

2144

if (!ValVT.isSimple())

2145

return SDValue();

2146

switch (ValVT.getSimpleVT().SimpleTy) {

2147

default:

2148

return SDValue();

2149

case MVT::v2i8:

2150

case MVT::v2i16:

2151

case MVT::v2i32:

2152

case MVT::v2i64:

2153

case MVT::v2f32:

2154

case MVT::v2f64:

2155

case MVT::v4i8:

2156

case MVT::v4i16:

2157

case MVT::v4i32:

2158

case MVT::v4f32:

2159

// This is a "native" vector type

2160

break;

2161

}

2162

2163

MemSDNode *MemSD = cast<MemSDNode>(N);

2164

const DataLayout &TD = DAG.getDataLayout();

2165

2166

unsigned Align = MemSD->getAlignment();

2167

unsigned PrefAlign =

2168

TD.getPrefTypeAlignment(ValVT.getTypeForEVT(*DAG.getContext()));

2169

if (Align < PrefAlign) {

2170

// This store is not sufficiently aligned, so bail out and let this vector

2171

// store be scalarized. Note that we may still be able to emit smaller

2172

// vector stores. For example, if we are storing a <4 x float> with an

2173

// alignment of 8, this check will fail but the legalizer will try again

2174

// with 2 x <2 x float>, which will succeed with an alignment of 8.

2175

return SDValue();

2176

}

2177

2178

unsigned Opcode = 0;

2179

EVT EltVT = ValVT.getVectorElementType();

2180

unsigned NumElts = ValVT.getVectorNumElements();

2181

2182

// Since StoreV2 is a target node, we cannot rely on DAG type legalization.

2183

// Therefore, we must ensure the type is legal. For i1 and i8, we set the

2184

// stored type to i16 and propagate the "real" type as the memory type.

2185

bool NeedExt = false;

2186

if (EltVT.getSizeInBits() < 16)

2187

NeedExt = true;

2188

2189

switch (NumElts) {

2190

default:

2191

return SDValue();

2192

case 2:

2193

Opcode = NVPTXISD::StoreV2;

2194

break;

2195

case 4:

2196

Opcode = NVPTXISD::StoreV4;

2197

break;

2198

}

2199

2200

SmallVector<SDValue, 8> Ops;

2201

2202

// First is the chain

2203

Ops.push_back(N->getOperand(0));

2204

2205

// Then the split values

2206

for (unsigned i = 0; i < NumElts; ++i) {

2207

SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,

2208

DAG.getIntPtrConstant(i, DL));

2209

if (NeedExt)

2210

ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i16, ExtVal);

2211

Ops.push_back(ExtVal);

2212

}

2213

2214

// Then any remaining arguments

2215

Ops.append(N->op_begin() + 2, N->op_end());

2216

2217

SDValue NewSt = DAG.getMemIntrinsicNode(

2218

Opcode, DL, DAG.getVTList(MVT::Other), Ops,

2219

MemSD->getMemoryVT(), MemSD->getMemOperand());

2220

2221

//return DCI.CombineTo(N, NewSt, true);

2222

return NewSt;

2223

}

2224

2225

return SDValue();

2226

}

2227

2228

// st i1 v, addr

2229

// =>

2230

// v1 = zxt v to i16

2231

// st.u8 i16, addr

2232

SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {

2233

SDNode *Node = Op.getNode();

2234

SDLoc dl(Node);

2235

StoreSDNode *ST = cast<StoreSDNode>(Node);

2236

SDValue Tmp1 = ST->getChain();

2237

SDValue Tmp2 = ST->getBasePtr();

2238

SDValue Tmp3 = ST->getValue();

2239

assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only")((Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only"
) ? static_cast<void> (0) : __assert_fail ("Tmp3.getValueType() == MVT::i1 && \"Custom lowering for i1 store only\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2239, __PRETTY_FUNCTION__));

2240

Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Tmp3);

2241

SDValue Result =

2242

DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), MVT::i8,

2243

ST->getAlignment(), ST->getMemOperand()->getFlags());

2244

return Result;

2245

}

2246

2247

SDValue

2248

NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int idx, EVT v) const {

2249

std::string ParamSym;

2250

raw_string_ostream ParamStr(ParamSym);

2251

2252

ParamStr << DAG.getMachineFunction().getName() << "_param_" << idx;

2253

ParamStr.flush();

2254

2255

std::string *SavedStr =

2256

nvTM->getManagedStrPool()->getManagedString(ParamSym.c_str());

2257

return DAG.getTargetExternalSymbol(SavedStr->c_str(), v);

2258

}

2259

2260

// Check to see if the kernel argument is image*_t or sampler_t

2261

2262

static bool isImageOrSamplerVal(const Value *arg, const Module *context) {

2263

static const char *const specialTypes[] = { "struct._image2d_t",

2264

"struct._image3d_t",

2265

"struct._sampler_t" };

2266

2267

Type *Ty = arg->getType();

2268

auto *PTy = dyn_cast<PointerType>(Ty);

2269

2270

if (!PTy)

2271

return false;

2272

2273

if (!context)

2274

return false;

2275

2276

auto *STy = dyn_cast<StructType>(PTy->getElementType());

2277

if (!STy || STy->isLiteral())

2278

return false;

2279

2280

return std::find(std::begin(specialTypes), std::end(specialTypes),

2281

STy->getName()) != std::end(specialTypes);

2282

}

2283

2284

SDValue NVPTXTargetLowering::LowerFormalArguments(

2285

SDValue Chain, CallingConv::ID CallConv, bool isVarArg,

2286

const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,

2287

SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {

2288

MachineFunction &MF = DAG.getMachineFunction();

2289

const DataLayout &DL = DAG.getDataLayout();

2290

auto PtrVT = getPointerTy(DAG.getDataLayout());

2291

2292

const Function *F = MF.getFunction();

2293

const AttributeSet &PAL = F->getAttributes();

2294

const TargetLowering *TLI = STI.getTargetLowering();

2295

2296

SDValue Root = DAG.getRoot();

2297

std::vector<SDValue> OutChains;

2298

2299

bool isABI = (STI.getSmVersion() >= 20);

2300

2301

if (!isABI)

2302

return Chain;

2303

2304

std::vector<Type *> argTypes;

2305

std::vector<const Argument *> theArgs;

2306

for (const Argument &I : F->args()) {

2307

theArgs.push_back(&I);

2308

argTypes.push_back(I.getType());

2309

}

2310

// argTypes.size() (or theArgs.size()) and Ins.size() need not match.

2311

// Ins.size() will be larger

2312

// * if there is an aggregate argument with multiple fields (each field

2313

// showing up separately in Ins)

2314

// * if there is a vector argument with more than typical vector-length

2315

// elements (generally if more than 4) where each vector element is

2316

// individually present in Ins.

2317

// So a different index should be used for indexing into Ins.

2318

// See similar issue in LowerCall.

2319

unsigned InsIdx = 0;

2320

2321

int idx = 0;

2322

for (unsigned i = 0, e = theArgs.size(); i != e; ++i, ++idx, ++InsIdx) {

2323

Type *Ty = argTypes[i];

2324

2325

// If the kernel argument is image*_t or sampler_t, convert it to

2326

// a i32 constant holding the parameter position. This can later

2327

// matched in the AsmPrinter to output the correct mangled name.

2328

if (isImageOrSamplerVal(

2329

theArgs[i],

2330

(theArgs[i]->getParent() ? theArgs[i]->getParent()->getParent()

2331

: nullptr))) {

2332

assert(isKernelFunction(*F) &&((isKernelFunction(*F) && "Only kernels can have image/sampler params"
) ? static_cast<void> (0) : __assert_fail ("isKernelFunction(*F) && \"Only kernels can have image/sampler params\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2333, __PRETTY_FUNCTION__))

2333

"Only kernels can have image/sampler params")((isKernelFunction(*F) && "Only kernels can have image/sampler params"
) ? static_cast<void> (0) : __assert_fail ("isKernelFunction(*F) && \"Only kernels can have image/sampler params\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2333, __PRETTY_FUNCTION__));

2334

InVals.push_back(DAG.getConstant(i + 1, dl, MVT::i32));

2335

continue;

2336

}

2337

2338

if (theArgs[i]->use_empty()) {

2339

// argument is dead

2340

if (Ty->isAggregateType()) {

2341

SmallVector<EVT, 16> vtparts;

2342

2343

ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts);

2344

assert(vtparts.size() > 0 && "empty aggregate type not expected")((vtparts.size() > 0 && "empty aggregate type not expected"
) ? static_cast<void> (0) : __assert_fail ("vtparts.size() > 0 && \"empty aggregate type not expected\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2344, __PRETTY_FUNCTION__));

2345

for (unsigned parti = 0, parte = vtparts.size(); parti != parte;

2346

++parti) {

2347

InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));

2348

++InsIdx;

2349

}

2350

if (vtparts.size() > 0)

2351

--InsIdx;

2352

continue;

2353

}

2354

if (Ty->isVectorTy()) {

2355

EVT ObjectVT = getValueType(DL, Ty);

2356

unsigned NumRegs = TLI->getNumRegisters(F->getContext(), ObjectVT);

2357

for (unsigned parti = 0; parti < NumRegs; ++parti) {

2358

InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));

2359

++InsIdx;

2360

}

2361

if (NumRegs > 0)

2362

--InsIdx;

2363

continue;

2364

}

2365

InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));

2366

continue;

2367

}

2368

2369

// In the following cases, assign a node order of "idx+1"

2370

// to newly created nodes. The SDNodes for params have to

2371

// appear in the same order as their order of appearance

2372

// in the original function. "idx+1" holds that order.

2373

if (!PAL.hasAttribute(i + 1, Attribute::ByVal)) {

2374

if (Ty->isAggregateType()) {

2375

SmallVector<EVT, 16> vtparts;

2376

SmallVector<uint64_t, 16> offsets;

2377

2378

// NOTE: Here, we lose the ability to issue vector loads for vectors

2379

// that are a part of a struct. This should be investigated in the

2380

// future.

2381

ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts, &offsets,

2382

0);

2383

2384

bool aggregateIsPacked = false;

2385

if (StructType *STy = dyn_cast<StructType>(Ty))

2386

aggregateIsPacked = STy->isPacked();

2387

2388

SDValue Arg = getParamSymbol(DAG, idx, PtrVT);

2389

for (unsigned parti = 0, parte = vtparts.size(); parti != parte;

2390

++parti) {

2391

EVT partVT = vtparts[parti];

2392

Value *srcValue = Constant::getNullValue(

2393

PointerType::get(partVT.getTypeForEVT(F->getContext()),

2394

ADDRESS_SPACE_PARAM));

2395

SDValue srcAddr =

2396

DAG.getNode(ISD::ADD, dl, PtrVT, Arg,

2397

DAG.getConstant(offsets[parti], dl, PtrVT));

2398

unsigned partAlign = aggregateIsPacked

2399

? 1

2400

: DL.getABITypeAlignment(

2401

partVT.getTypeForEVT(F->getContext()));

2402

SDValue p;

2403

if (Ins[InsIdx].VT.getSizeInBits() > partVT.getSizeInBits()) {

2404

ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ?

2405

ISD::SEXTLOAD : ISD::ZEXTLOAD;

2406

p = DAG.getExtLoad(ExtOp, dl, Ins[InsIdx].VT, Root, srcAddr,

2407

MachinePointerInfo(srcValue), partVT, partAlign);

2408

} else {

2409

p = DAG.getLoad(partVT, dl, Root, srcAddr,

2410

MachinePointerInfo(srcValue), partAlign);

2411

}

2412

if (p.getNode())

2413

p.getNode()->setIROrder(idx + 1);

2414

InVals.push_back(p);

2415

++InsIdx;

2416

}

2417

if (vtparts.size() > 0)

2418

--InsIdx;

2419

continue;

2420

}

2421

if (Ty->isVectorTy()) {

2422

EVT ObjectVT = getValueType(DL, Ty);

2423

SDValue Arg = getParamSymbol(DAG, idx, PtrVT);

2424

unsigned NumElts = ObjectVT.getVectorNumElements();

2425

assert(TLI->getNumRegisters(F->getContext(), ObjectVT) == NumElts &&((TLI->getNumRegisters(F->getContext(), ObjectVT) == NumElts
&& "Vector was not scalarized") ? static_cast<void
> (0) : __assert_fail ("TLI->getNumRegisters(F->getContext(), ObjectVT) == NumElts && \"Vector was not scalarized\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2426, __PRETTY_FUNCTION__))

2426

"Vector was not scalarized")((TLI->getNumRegisters(F->getContext(), ObjectVT) == NumElts
&& "Vector was not scalarized") ? static_cast<void
> (0) : __assert_fail ("TLI->getNumRegisters(F->getContext(), ObjectVT) == NumElts && \"Vector was not scalarized\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2426, __PRETTY_FUNCTION__));

2427

EVT EltVT = ObjectVT.getVectorElementType();

2428

2429

// V1 load

2430

// f32 = load ...

2431

if (NumElts == 1) {

2432

// We only have one element, so just directly load it

2433

Value *SrcValue = Constant::getNullValue(PointerType::get(

2434

EltVT.getTypeForEVT(F->getContext()), ADDRESS_SPACE_PARAM));

2435

SDValue P = DAG.getLoad(

2436

EltVT, dl, Root, Arg, MachinePointerInfo(SrcValue),

2437

DL.getABITypeAlignment(EltVT.getTypeForEVT(F->getContext())),

2438

MachineMemOperand::MODereferenceable |

2439

MachineMemOperand::MOInvariant);

2440

if (P.getNode())

2441

P.getNode()->setIROrder(idx + 1);

2442

2443

if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits())

2444

P = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, P);

2445

InVals.push_back(P);

2446

++InsIdx;

2447

} else if (NumElts == 2) {

2448

// V2 load

2449

// f32,f32 = load ...

2450

EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, 2);

2451

Value *SrcValue = Constant::getNullValue(PointerType::get(

2452

VecVT.getTypeForEVT(F->getContext()), ADDRESS_SPACE_PARAM));

2453

SDValue P = DAG.getLoad(

2454

VecVT, dl, Root, Arg, MachinePointerInfo(SrcValue),

2455

DL.getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())),

2456

MachineMemOperand::MODereferenceable |

2457

MachineMemOperand::MOInvariant);

2458

if (P.getNode())

2459

P.getNode()->setIROrder(idx + 1);

2460

2461

SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,

2462

DAG.getIntPtrConstant(0, dl));

2463

SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,

2464

DAG.getIntPtrConstant(1, dl));

2465

2466

if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits()) {

2467

Elt0 = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt0);

2468

Elt1 = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt1);

2469

}

2470

2471

InVals.push_back(Elt0);

2472

InVals.push_back(Elt1);

2473

InsIdx += 2;

2474

} else {

2475

// V4 loads

2476

// We have at least 4 elements (<3 x Ty> expands to 4 elements) and

2477

// the vector will be expanded to a power of 2 elements, so we know we

2478

// can always round up to the next multiple of 4 when creating the

2479

// vector loads.

2480

// e.g. 4 elem => 1 ld.v4

2481

// 6 elem => 2 ld.v4

2482

// 8 elem => 2 ld.v4

2483

// 11 elem => 3 ld.v4

2484

unsigned VecSize = 4;

2485

if (EltVT.getSizeInBits() == 64) {

2486

VecSize = 2;

2487

}

2488

EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, VecSize);

2489

unsigned Ofst = 0;

2490

for (unsigned i = 0; i < NumElts; i += VecSize) {

2491

Value *SrcValue = Constant::getNullValue(

2492

PointerType::get(VecVT.getTypeForEVT(F->getContext()),

2493

ADDRESS_SPACE_PARAM));

2494

SDValue SrcAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Arg,

2495

DAG.getConstant(Ofst, dl, PtrVT));

2496

SDValue P = DAG.getLoad(

2497

VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue),

2498

DL.getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())),

2499

MachineMemOperand::MODereferenceable |

2500

MachineMemOperand::MOInvariant);

2501

if (P.getNode())

2502

P.getNode()->setIROrder(idx + 1);

2503

2504

for (unsigned j = 0; j < VecSize; ++j) {

2505

if (i + j >= NumElts)

2506

break;

2507

SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,

2508

DAG.getIntPtrConstant(j, dl));

2509

if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits())

2510

Elt = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt);

2511

InVals.push_back(Elt);

2512

}

2513

Ofst += DL.getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));

2514

}

2515

InsIdx += NumElts;

2516

}

2517

2518

if (NumElts > 0)

2519

--InsIdx;

2520

continue;

2521

}

2522

// A plain scalar.

2523

EVT ObjectVT = getValueType(DL, Ty);

2524

// If ABI, load from the param symbol

2525

SDValue Arg = getParamSymbol(DAG, idx, PtrVT);

2526

Value *srcValue = Constant::getNullValue(PointerType::get(

2527

ObjectVT.getTypeForEVT(F->getContext()), ADDRESS_SPACE_PARAM));

2528

SDValue p;

2529

if (ObjectVT.getSizeInBits() < Ins[InsIdx].VT.getSizeInBits()) {

2530

ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ?

2531

ISD::SEXTLOAD : ISD::ZEXTLOAD;

2532

p = DAG.getExtLoad(

2533

ExtOp, dl, Ins[InsIdx].VT, Root, Arg, MachinePointerInfo(srcValue),

2534

ObjectVT,

2535

DL.getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));

2536

} else {

2537

p = DAG.getLoad(

2538

Ins[InsIdx].VT, dl, Root, Arg, MachinePointerInfo(srcValue),

2539

DL.getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));

2540

}

2541

if (p.getNode())

2542

p.getNode()->setIROrder(idx + 1);

2543

InVals.push_back(p);

2544

continue;

2545

}

2546

2547

// Param has ByVal attribute

2548

// Return MoveParam(param symbol).

2549

// Ideally, the param symbol can be returned directly,

2550

// but when SDNode builder decides to use it in a CopyToReg(),

2551

// machine instruction fails because TargetExternalSymbol

2552

// (not lowered) is target dependent, and CopyToReg assumes

2553

// the source is lowered.

2554

EVT ObjectVT = getValueType(DL, Ty);

2555

assert(ObjectVT == Ins[InsIdx].VT &&((ObjectVT == Ins[InsIdx].VT && "Ins type did not match function type"
) ? static_cast<void> (0) : __assert_fail ("ObjectVT == Ins[InsIdx].VT && \"Ins type did not match function type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2556, __PRETTY_FUNCTION__))

2556

"Ins type did not match function type")((ObjectVT == Ins[InsIdx].VT && "Ins type did not match function type"
) ? static_cast<void> (0) : __assert_fail ("ObjectVT == Ins[InsIdx].VT && \"Ins type did not match function type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2556, __PRETTY_FUNCTION__));

2557

SDValue Arg = getParamSymbol(DAG, idx, PtrVT);

2558

SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);

2559

if (p.getNode())

2560

p.getNode()->setIROrder(idx + 1);

2561

InVals.push_back(p);

2562

}

2563

2564

// Clang will check explicit VarArg and issue error if any. However, Clang

2565

// will let code with

2566

// implicit var arg like f() pass. See bug 617733.

2567

// We treat this case as if the arg list is empty.

2568

// if (F.isVarArg()) {

2569

// assert(0 && "VarArg not supported yet!");

2570

//}

2571

2572

if (!OutChains.empty())

2573

DAG.setRoot(DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains));

2574

2575

return Chain;

2576

}

2577

2578

SDValue

2579

NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,

2580

bool isVarArg,

2581

const SmallVectorImpl<ISD::OutputArg> &Outs,

2582

const SmallVectorImpl<SDValue> &OutVals,

2583

const SDLoc &dl, SelectionDAG &DAG) const {

2584

MachineFunction &MF = DAG.getMachineFunction();

2585

const Function *F = MF.getFunction();

2586

Type *RetTy = F->getReturnType();

2587

const DataLayout &TD = DAG.getDataLayout();

2588

2589

bool isABI = (STI.getSmVersion() >= 20);

2590

2591

if (!isABI)

2592

return Chain;

2593

2594

if (VectorType *VTy = dyn_cast<VectorType>(RetTy)) {

2595

// If we have a vector type, the OutVals array will be the scalarized

2596

// components and we have combine them into 1 or more vector stores.

2597

unsigned NumElts = VTy->getNumElements();

2598

assert(NumElts == Outs.size() && "Bad scalarization of return value")((NumElts == Outs.size() && "Bad scalarization of return value"
) ? static_cast<void> (0) : __assert_fail ("NumElts == Outs.size() && \"Bad scalarization of return value\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2598, __PRETTY_FUNCTION__));

2599

2600

// const_cast can be removed in later LLVM versions

2601

EVT EltVT = getValueType(TD, RetTy).getVectorElementType();

2602

bool NeedExtend = false;

2603

if (EltVT.getSizeInBits() < 16)

2604

NeedExtend = true;

2605

2606

// V1 store

2607

if (NumElts == 1) {

2608

SDValue StoreVal = OutVals[0];

2609

// We only have one element, so just directly store it

2610

if (NeedExtend)

2611

StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);

2612

SDValue Ops[] = { Chain, DAG.getConstant(0, dl, MVT::i32), StoreVal };

2613

Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,

2614

DAG.getVTList(MVT::Other), Ops,

2615

EltVT, MachinePointerInfo());

2616

} else if (NumElts == 2) {

2617

// V2 store

2618

SDValue StoreVal0 = OutVals[0];

2619

SDValue StoreVal1 = OutVals[1];

2620

2621

if (NeedExtend) {

2622

StoreVal0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal0);

2623

StoreVal1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal1);

2624

}

2625

2626

SDValue Ops[] = { Chain, DAG.getConstant(0, dl, MVT::i32), StoreVal0,

2627

StoreVal1 };

2628

Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetvalV2, dl,

2629

DAG.getVTList(MVT::Other), Ops,

2630

EltVT, MachinePointerInfo());

2631

} else {

2632

// V4 stores

2633

// We have at least 4 elements (<3 x Ty> expands to 4 elements) and the

2634

// vector will be expanded to a power of 2 elements, so we know we can

2635

// always round up to the next multiple of 4 when creating the vector

2636

// stores.

2637

// e.g. 4 elem => 1 st.v4

2638

// 6 elem => 2 st.v4

2639

// 8 elem => 2 st.v4

2640

// 11 elem => 3 st.v4

2641

2642

unsigned VecSize = 4;

2643

if (OutVals[0].getValueSizeInBits() == 64)

2644

VecSize = 2;

2645

2646

unsigned Offset = 0;

2647

2648

EVT VecVT =

2649

EVT::getVectorVT(F->getContext(), EltVT, VecSize);

2650

unsigned PerStoreOffset =

2651

TD.getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));

2652

2653

for (unsigned i = 0; i < NumElts; i += VecSize) {

2654

// Get values

2655

SDValue StoreVal;

2656

SmallVector<SDValue, 8> Ops;

2657

Ops.push_back(Chain);

2658

Ops.push_back(DAG.getConstant(Offset, dl, MVT::i32));

2659

unsigned Opc = NVPTXISD::StoreRetvalV2;

2660

EVT ExtendedVT = (NeedExtend) ? MVT::i16 : OutVals[0].getValueType();

2661

2662

StoreVal = OutVals[i];

2663

if (NeedExtend)

2664

StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);

2665

Ops.push_back(StoreVal);

2666

2667

if (i + 1 < NumElts) {

2668

StoreVal = OutVals[i + 1];

2669

if (NeedExtend)

2670

StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);

2671

} else {

2672

StoreVal = DAG.getUNDEF(ExtendedVT);

2673

}

2674

Ops.push_back(StoreVal);

2675

2676

if (VecSize == 4) {

2677

Opc = NVPTXISD::StoreRetvalV4;

2678

if (i + 2 < NumElts) {

2679

StoreVal = OutVals[i + 2];

2680

if (NeedExtend)

2681

StoreVal =

2682

DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);

2683

} else {

2684

StoreVal = DAG.getUNDEF(ExtendedVT);

2685

}

2686

Ops.push_back(StoreVal);

2687

2688

if (i + 3 < NumElts) {

2689

StoreVal = OutVals[i + 3];

2690

if (NeedExtend)

2691

StoreVal =

2692

DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);

2693

} else {

2694

StoreVal = DAG.getUNDEF(ExtendedVT);

2695

}

2696

Ops.push_back(StoreVal);

2697

}

2698

2699

// Chain = DAG.getNode(Opc, dl, MVT::Other, &Ops[0], Ops.size());

2700

Chain =

2701

DAG.getMemIntrinsicNode(Opc, dl, DAG.getVTList(MVT::Other), Ops,

2702

EltVT, MachinePointerInfo());

2703

Offset += PerStoreOffset;

2704

}

2705

}

2706

} else {

2707

SmallVector<EVT, 16> ValVTs;

2708

SmallVector<uint64_t, 16> Offsets;

2709

ComputePTXValueVTs(*this, DAG.getDataLayout(), RetTy, ValVTs, &Offsets, 0);

2710

assert(ValVTs.size() == OutVals.size() && "Bad return value decomposition")((ValVTs.size() == OutVals.size() && "Bad return value decomposition"
) ? static_cast<void> (0) : __assert_fail ("ValVTs.size() == OutVals.size() && \"Bad return value decomposition\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 2710, __PRETTY_FUNCTION__));

2711

2712

for (unsigned i = 0, e = Outs.size(); i != e; ++i) {

2713

SDValue theVal = OutVals[i];

2714

EVT TheValType = theVal.getValueType();

2715

unsigned numElems = 1;

2716

if (TheValType.isVector())

2717

numElems = TheValType.getVectorNumElements();

2718

for (unsigned j = 0, je = numElems; j != je; ++j) {

2719

SDValue TmpVal = theVal;

2720

if (TheValType.isVector())

2721

TmpVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,

2722

TheValType.getVectorElementType(), TmpVal,

2723

DAG.getIntPtrConstant(j, dl));

2724

EVT TheStoreType = ValVTs[i];

2725

if (RetTy->isIntegerTy() && TD.getTypeAllocSizeInBits(RetTy) < 32) {

2726

// The following zero-extension is for integer types only, and

2727

// specifically not for aggregates.

2728

TmpVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, TmpVal);

2729

TheStoreType = MVT::i32;

2730

} else if (RetTy->isHalfTy()) {

2731

TheStoreType = MVT::f16;

2732

} else if (TmpVal.getValueSizeInBits() < 16)

2733

TmpVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, TmpVal);

2734

2735

SDValue Ops[] = {

2736

Chain,

2737

DAG.getConstant(Offsets[i], dl, MVT::i32),

2738

TmpVal };

2739

Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,

2740

DAG.getVTList(MVT::Other), Ops,

2741

TheStoreType,

2742

MachinePointerInfo());

2743

}

2744

}

2745

}

2746

2747

return DAG.getNode(NVPTXISD::RET_FLAG, dl, MVT::Other, Chain);

2748

}

2749

2750

void NVPTXTargetLowering::LowerAsmOperandForConstraint(

2751

SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,

2752

SelectionDAG &DAG) const {

2753

if (Constraint.length() > 1)

2754

return;

2755

else

2756

TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);

2757

}

2758

2759

static unsigned getOpcForTextureInstr(unsigned Intrinsic) {

2760

switch (Intrinsic) {

2761

default:

2762

return 0;

2763

2764

case Intrinsic::nvvm_tex_1d_v4f32_s32:

2765

return NVPTXISD::Tex1DFloatS32;

2766

case Intrinsic::nvvm_tex_1d_v4f32_f32:

2767

return NVPTXISD::Tex1DFloatFloat;

2768

case Intrinsic::nvvm_tex_1d_level_v4f32_f32:

2769

return NVPTXISD::Tex1DFloatFloatLevel;

2770

case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:

2771

return NVPTXISD::Tex1DFloatFloatGrad;

2772

case Intrinsic::nvvm_tex_1d_v4s32_s32:

2773

return NVPTXISD::Tex1DS32S32;

2774

case Intrinsic::nvvm_tex_1d_v4s32_f32:

2775

return NVPTXISD::Tex1DS32Float;

2776

case Intrinsic::nvvm_tex_1d_level_v4s32_f32:

2777

return NVPTXISD::Tex1DS32FloatLevel;

2778

case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:

2779

return NVPTXISD::Tex1DS32FloatGrad;

2780

case Intrinsic::nvvm_tex_1d_v4u32_s32:

2781

return NVPTXISD::Tex1DU32S32;

2782

case Intrinsic::nvvm_tex_1d_v4u32_f32:

2783

return NVPTXISD::Tex1DU32Float;

2784

case Intrinsic::nvvm_tex_1d_level_v4u32_f32:

2785

return NVPTXISD::Tex1DU32FloatLevel;

2786

case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:

2787

return NVPTXISD::Tex1DU32FloatGrad;

2788

2789

case Intrinsic::nvvm_tex_1d_array_v4f32_s32:

2790

return NVPTXISD::Tex1DArrayFloatS32;

2791

case Intrinsic::nvvm_tex_1d_array_v4f32_f32:

2792

return NVPTXISD::Tex1DArrayFloatFloat;

2793

case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:

2794

return NVPTXISD::Tex1DArrayFloatFloatLevel;

2795

case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:

2796

return NVPTXISD::Tex1DArrayFloatFloatGrad;

2797

case Intrinsic::nvvm_tex_1d_array_v4s32_s32:

2798

return NVPTXISD::Tex1DArrayS32S32;

2799

case Intrinsic::nvvm_tex_1d_array_v4s32_f32:

2800

return NVPTXISD::Tex1DArrayS32Float;

2801

case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:

2802

return NVPTXISD::Tex1DArrayS32FloatLevel;

2803

case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:

2804

return NVPTXISD::Tex1DArrayS32FloatGrad;

2805

case Intrinsic::nvvm_tex_1d_array_v4u32_s32:

2806

return NVPTXISD::Tex1DArrayU32S32;

2807

case Intrinsic::nvvm_tex_1d_array_v4u32_f32:

2808

return NVPTXISD::Tex1DArrayU32Float;

2809

case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:

2810

return NVPTXISD::Tex1DArrayU32FloatLevel;

2811

case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:

2812

return NVPTXISD::Tex1DArrayU32FloatGrad;

2813

2814

case Intrinsic::nvvm_tex_2d_v4f32_s32:

2815

return NVPTXISD::Tex2DFloatS32;

2816

case Intrinsic::nvvm_tex_2d_v4f32_f32:

2817

return NVPTXISD::Tex2DFloatFloat;

2818

case Intrinsic::nvvm_tex_2d_level_v4f32_f32:

2819

return NVPTXISD::Tex2DFloatFloatLevel;

2820

case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:

2821

return NVPTXISD::Tex2DFloatFloatGrad;

2822

case Intrinsic::nvvm_tex_2d_v4s32_s32:

2823

return NVPTXISD::Tex2DS32S32;

2824

case Intrinsic::nvvm_tex_2d_v4s32_f32:

2825

return NVPTXISD::Tex2DS32Float;

2826

case Intrinsic::nvvm_tex_2d_level_v4s32_f32:

2827

return NVPTXISD::Tex2DS32FloatLevel;

2828

case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:

2829

return NVPTXISD::Tex2DS32FloatGrad;

2830

case Intrinsic::nvvm_tex_2d_v4u32_s32:

2831

return NVPTXISD::Tex2DU32S32;

2832

case Intrinsic::nvvm_tex_2d_v4u32_f32:

2833

return NVPTXISD::Tex2DU32Float;

2834

case Intrinsic::nvvm_tex_2d_level_v4u32_f32:

2835

return NVPTXISD::Tex2DU32FloatLevel;

2836

case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:

2837

return NVPTXISD::Tex2DU32FloatGrad;

2838

2839

case Intrinsic::nvvm_tex_2d_array_v4f32_s32:

2840

return NVPTXISD::Tex2DArrayFloatS32;

2841

case Intrinsic::nvvm_tex_2d_array_v4f32_f32:

2842

return NVPTXISD::Tex2DArrayFloatFloat;

2843

case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:

2844

return NVPTXISD::Tex2DArrayFloatFloatLevel;

2845

case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:

2846

return NVPTXISD::Tex2DArrayFloatFloatGrad;

2847

case Intrinsic::nvvm_tex_2d_array_v4s32_s32:

2848

return NVPTXISD::Tex2DArrayS32S32;

2849

case Intrinsic::nvvm_tex_2d_array_v4s32_f32:

2850

return NVPTXISD::Tex2DArrayS32Float;

2851

case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:

2852

return NVPTXISD::Tex2DArrayS32FloatLevel;

2853

case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:

2854

return NVPTXISD::Tex2DArrayS32FloatGrad;

2855

case Intrinsic::nvvm_tex_2d_array_v4u32_s32:

2856

return NVPTXISD::Tex2DArrayU32S32;

2857

case Intrinsic::nvvm_tex_2d_array_v4u32_f32:

2858

return NVPTXISD::Tex2DArrayU32Float;

2859

case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:

2860

return NVPTXISD::Tex2DArrayU32FloatLevel;

2861

case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:

2862

return NVPTXISD::Tex2DArrayU32FloatGrad;

2863

2864

case Intrinsic::nvvm_tex_3d_v4f32_s32:

2865

return NVPTXISD::Tex3DFloatS32;

2866

case Intrinsic::nvvm_tex_3d_v4f32_f32:

2867

return NVPTXISD::Tex3DFloatFloat;

2868

case Intrinsic::nvvm_tex_3d_level_v4f32_f32:

2869

return NVPTXISD::Tex3DFloatFloatLevel;

2870

case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:

2871

return NVPTXISD::Tex3DFloatFloatGrad;

2872

case Intrinsic::nvvm_tex_3d_v4s32_s32:

2873

return NVPTXISD::Tex3DS32S32;

2874

case Intrinsic::nvvm_tex_3d_v4s32_f32:

2875

return NVPTXISD::Tex3DS32Float;

2876

case Intrinsic::nvvm_tex_3d_level_v4s32_f32:

2877

return NVPTXISD::Tex3DS32FloatLevel;

2878

case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:

2879

return NVPTXISD::Tex3DS32FloatGrad;

2880

case Intrinsic::nvvm_tex_3d_v4u32_s32:

2881

return NVPTXISD::Tex3DU32S32;

2882

case Intrinsic::nvvm_tex_3d_v4u32_f32:

2883

return NVPTXISD::Tex3DU32Float;

2884

case Intrinsic::nvvm_tex_3d_level_v4u32_f32:

2885

return NVPTXISD::Tex3DU32FloatLevel;

2886

case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:

2887

return NVPTXISD::Tex3DU32FloatGrad;

2888

2889

case Intrinsic::nvvm_tex_cube_v4f32_f32:

2890

return NVPTXISD::TexCubeFloatFloat;

2891

case Intrinsic::nvvm_tex_cube_level_v4f32_f32:

2892

return NVPTXISD::TexCubeFloatFloatLevel;

2893

case Intrinsic::nvvm_tex_cube_v4s32_f32:

2894

return NVPTXISD::TexCubeS32Float;

2895

case Intrinsic::nvvm_tex_cube_level_v4s32_f32:

2896

return NVPTXISD::TexCubeS32FloatLevel;

2897

case Intrinsic::nvvm_tex_cube_v4u32_f32:

2898

return NVPTXISD::TexCubeU32Float;

2899

case Intrinsic::nvvm_tex_cube_level_v4u32_f32:

2900

return NVPTXISD::TexCubeU32FloatLevel;

2901

2902

case Intrinsic::nvvm_tex_cube_array_v4f32_f32:

2903

return NVPTXISD::TexCubeArrayFloatFloat;

2904

case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:

2905

return NVPTXISD::TexCubeArrayFloatFloatLevel;

2906

case Intrinsic::nvvm_tex_cube_array_v4s32_f32:

2907

return NVPTXISD::TexCubeArrayS32Float;

2908

case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:

2909

return NVPTXISD::TexCubeArrayS32FloatLevel;

2910

case Intrinsic::nvvm_tex_cube_array_v4u32_f32:

2911

return NVPTXISD::TexCubeArrayU32Float;

2912

case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:

2913

return NVPTXISD::TexCubeArrayU32FloatLevel;

2914

2915

case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:

2916

return NVPTXISD::Tld4R2DFloatFloat;

2917

case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:

2918

return NVPTXISD::Tld4G2DFloatFloat;

2919

case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:

2920

return NVPTXISD::Tld4B2DFloatFloat;

2921

case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:

2922

return NVPTXISD::Tld4A2DFloatFloat;

2923

case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:

2924

return NVPTXISD::Tld4R2DS64Float;

2925

case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:

2926

return NVPTXISD::Tld4G2DS64Float;

2927

case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:

2928

return NVPTXISD::Tld4B2DS64Float;

2929

case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:

2930

return NVPTXISD::Tld4A2DS64Float;

2931

case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:

2932

return NVPTXISD::Tld4R2DU64Float;

2933

case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:

2934

return NVPTXISD::Tld4G2DU64Float;

2935

case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:

2936

return NVPTXISD::Tld4B2DU64Float;

2937

case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:

2938

return NVPTXISD::Tld4A2DU64Float;

2939

2940

case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:

2941

return NVPTXISD::TexUnified1DFloatS32;

2942

case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:

2943

return NVPTXISD::TexUnified1DFloatFloat;

2944

case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:

2945

return NVPTXISD::TexUnified1DFloatFloatLevel;

2946

case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:

2947

return NVPTXISD::TexUnified1DFloatFloatGrad;

2948

case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:

2949

return NVPTXISD::TexUnified1DS32S32;

2950

case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:

2951

return NVPTXISD::TexUnified1DS32Float;

2952

case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:

2953

return NVPTXISD::TexUnified1DS32FloatLevel;

2954

case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:

2955

return NVPTXISD::TexUnified1DS32FloatGrad;

2956

case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:

2957

return NVPTXISD::TexUnified1DU32S32;

2958

case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:

2959

return NVPTXISD::TexUnified1DU32Float;

2960

case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:

2961

return NVPTXISD::TexUnified1DU32FloatLevel;

2962

case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:

2963

return NVPTXISD::TexUnified1DU32FloatGrad;

2964

2965

case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:

2966

return NVPTXISD::TexUnified1DArrayFloatS32;

2967

case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:

2968

return NVPTXISD::TexUnified1DArrayFloatFloat;

2969

case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:

2970

return NVPTXISD::TexUnified1DArrayFloatFloatLevel;

2971

case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:

2972

return NVPTXISD::TexUnified1DArrayFloatFloatGrad;

2973

case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:

2974

return NVPTXISD::TexUnified1DArrayS32S32;

2975

case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:

2976

return NVPTXISD::TexUnified1DArrayS32Float;

2977

case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:

2978

return NVPTXISD::TexUnified1DArrayS32FloatLevel;

2979

case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:

2980

return NVPTXISD::TexUnified1DArrayS32FloatGrad;

2981

case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:

2982

return NVPTXISD::TexUnified1DArrayU32S32;

2983

case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:

2984

return NVPTXISD::TexUnified1DArrayU32Float;

2985

case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:

2986

return NVPTXISD::TexUnified1DArrayU32FloatLevel;

2987

case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:

2988

return NVPTXISD::TexUnified1DArrayU32FloatGrad;

2989

2990

case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:

2991

return NVPTXISD::TexUnified2DFloatS32;

2992

case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:

2993

return NVPTXISD::TexUnified2DFloatFloat;

2994

case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:

2995

return NVPTXISD::TexUnified2DFloatFloatLevel;

2996

case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:

2997

return NVPTXISD::TexUnified2DFloatFloatGrad;

2998

case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:

2999

return NVPTXISD::TexUnified2DS32S32;

3000

case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:

3001

return NVPTXISD::TexUnified2DS32Float;

3002

case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:

3003

return NVPTXISD::TexUnified2DS32FloatLevel;

3004

case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:

3005

return NVPTXISD::TexUnified2DS32FloatGrad;

3006

case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:

3007

return NVPTXISD::TexUnified2DU32S32;

3008

case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:

3009

return NVPTXISD::TexUnified2DU32Float;

3010

case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:

3011

return NVPTXISD::TexUnified2DU32FloatLevel;

3012

case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:

3013

return NVPTXISD::TexUnified2DU32FloatGrad;

3014

3015

case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:

3016

return NVPTXISD::TexUnified2DArrayFloatS32;

3017

case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:

3018

return NVPTXISD::TexUnified2DArrayFloatFloat;

3019

case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:

3020

return NVPTXISD::TexUnified2DArrayFloatFloatLevel;

3021

case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:

3022

return NVPTXISD::TexUnified2DArrayFloatFloatGrad;

3023

case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:

3024

return NVPTXISD::TexUnified2DArrayS32S32;

3025

case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:

3026

return NVPTXISD::TexUnified2DArrayS32Float;

3027

case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:

3028

return NVPTXISD::TexUnified2DArrayS32FloatLevel;

3029

case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:

3030

return NVPTXISD::TexUnified2DArrayS32FloatGrad;

3031

case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:

3032

return NVPTXISD::TexUnified2DArrayU32S32;

3033

case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:

3034

return NVPTXISD::TexUnified2DArrayU32Float;

3035

case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:

3036

return NVPTXISD::TexUnified2DArrayU32FloatLevel;

3037

case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:

3038

return NVPTXISD::TexUnified2DArrayU32FloatGrad;

3039

3040

case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:

3041

return NVPTXISD::TexUnified3DFloatS32;

3042

case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:

3043

return NVPTXISD::TexUnified3DFloatFloat;

3044

case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:

3045

return NVPTXISD::TexUnified3DFloatFloatLevel;

3046

case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:

3047

return NVPTXISD::TexUnified3DFloatFloatGrad;

3048

case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:

3049

return NVPTXISD::TexUnified3DS32S32;

3050

case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:

3051

return NVPTXISD::TexUnified3DS32Float;

3052

case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:

3053

return NVPTXISD::TexUnified3DS32FloatLevel;

3054

case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:

3055

return NVPTXISD::TexUnified3DS32FloatGrad;

3056

case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:

3057

return NVPTXISD::TexUnified3DU32S32;

3058

case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:

3059

return NVPTXISD::TexUnified3DU32Float;

3060

case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:

3061

return NVPTXISD::TexUnified3DU32FloatLevel;

3062

case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:

3063

return NVPTXISD::TexUnified3DU32FloatGrad;

3064

3065

case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:

3066

return NVPTXISD::TexUnifiedCubeFloatFloat;

3067

case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:

3068

return NVPTXISD::TexUnifiedCubeFloatFloatLevel;

3069

case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:

3070

return NVPTXISD::TexUnifiedCubeS32Float;

3071

case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:

3072

return NVPTXISD::TexUnifiedCubeS32FloatLevel;

3073

case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:

3074

return NVPTXISD::TexUnifiedCubeU32Float;

3075

case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:

3076

return NVPTXISD::TexUnifiedCubeU32FloatLevel;

3077

3078

case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:

3079

return NVPTXISD::TexUnifiedCubeArrayFloatFloat;

3080

case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:

3081

return NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel;

3082

case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:

3083

return NVPTXISD::TexUnifiedCubeArrayS32Float;

3084

case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:

3085

return NVPTXISD::TexUnifiedCubeArrayS32FloatLevel;

3086

case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:

3087

return NVPTXISD::TexUnifiedCubeArrayU32Float;

3088

case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:

3089

return NVPTXISD::TexUnifiedCubeArrayU32FloatLevel;

3090

3091

case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:

3092

return NVPTXISD::Tld4UnifiedR2DFloatFloat;

3093

case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:

3094

return NVPTXISD::Tld4UnifiedG2DFloatFloat;

3095

case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:

3096

return NVPTXISD::Tld4UnifiedB2DFloatFloat;

3097

case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:

3098

return NVPTXISD::Tld4UnifiedA2DFloatFloat;

3099

case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:

3100

return NVPTXISD::Tld4UnifiedR2DS64Float;

3101

case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:

3102

return NVPTXISD::Tld4UnifiedG2DS64Float;

3103

case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:

3104

return NVPTXISD::Tld4UnifiedB2DS64Float;

3105

case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:

3106

return NVPTXISD::Tld4UnifiedA2DS64Float;

3107

case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:

3108

return NVPTXISD::Tld4UnifiedR2DU64Float;

3109

case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:

3110

return NVPTXISD::Tld4UnifiedG2DU64Float;

3111

case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:

3112

return NVPTXISD::Tld4UnifiedB2DU64Float;

3113

case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:

3114

return NVPTXISD::Tld4UnifiedA2DU64Float;

3115

}

3116

}

3117

3118

static unsigned getOpcForSurfaceInstr(unsigned Intrinsic) {

3119

switch (Intrinsic) {

3120

default:

3121

return 0;

3122

case Intrinsic::nvvm_suld_1d_i8_clamp:

3123

return NVPTXISD::Suld1DI8Clamp;

3124

case Intrinsic::nvvm_suld_1d_i16_clamp:

3125

return NVPTXISD::Suld1DI16Clamp;

3126

case Intrinsic::nvvm_suld_1d_i32_clamp:

3127

return NVPTXISD::Suld1DI32Clamp;

3128

case Intrinsic::nvvm_suld_1d_i64_clamp:

3129

return NVPTXISD::Suld1DI64Clamp;

3130

case Intrinsic::nvvm_suld_1d_v2i8_clamp:

3131

return NVPTXISD::Suld1DV2I8Clamp;

3132

case Intrinsic::nvvm_suld_1d_v2i16_clamp:

3133

return NVPTXISD::Suld1DV2I16Clamp;

3134

case Intrinsic::nvvm_suld_1d_v2i32_clamp:

3135

return NVPTXISD::Suld1DV2I32Clamp;

3136

case Intrinsic::nvvm_suld_1d_v2i64_clamp:

3137

return NVPTXISD::Suld1DV2I64Clamp;

3138

case Intrinsic::nvvm_suld_1d_v4i8_clamp:

3139

return NVPTXISD::Suld1DV4I8Clamp;

3140

case Intrinsic::nvvm_suld_1d_v4i16_clamp:

3141

return NVPTXISD::Suld1DV4I16Clamp;

3142

case Intrinsic::nvvm_suld_1d_v4i32_clamp:

3143

return NVPTXISD::Suld1DV4I32Clamp;

3144

case Intrinsic::nvvm_suld_1d_array_i8_clamp:

3145

return NVPTXISD::Suld1DArrayI8Clamp;

3146

case Intrinsic::nvvm_suld_1d_array_i16_clamp:

3147

return NVPTXISD::Suld1DArrayI16Clamp;

3148

case Intrinsic::nvvm_suld_1d_array_i32_clamp:

3149

return NVPTXISD::Suld1DArrayI32Clamp;

3150

case Intrinsic::nvvm_suld_1d_array_i64_clamp:

3151

return NVPTXISD::Suld1DArrayI64Clamp;

3152

case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:

3153

return NVPTXISD::Suld1DArrayV2I8Clamp;

3154

case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:

3155

return NVPTXISD::Suld1DArrayV2I16Clamp;

3156

case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:

3157

return NVPTXISD::Suld1DArrayV2I32Clamp;

3158

case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:

3159

return NVPTXISD::Suld1DArrayV2I64Clamp;

3160

case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:

3161

return NVPTXISD::Suld1DArrayV4I8Clamp;

3162

case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:

3163

return NVPTXISD::Suld1DArrayV4I16Clamp;

3164

case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:

3165

return NVPTXISD::Suld1DArrayV4I32Clamp;

3166

case Intrinsic::nvvm_suld_2d_i8_clamp:

3167

return NVPTXISD::Suld2DI8Clamp;

3168

case Intrinsic::nvvm_suld_2d_i16_clamp:

3169

return NVPTXISD::Suld2DI16Clamp;

3170

case Intrinsic::nvvm_suld_2d_i32_clamp:

3171

return NVPTXISD::Suld2DI32Clamp;

3172

case Intrinsic::nvvm_suld_2d_i64_clamp:

3173

return NVPTXISD::Suld2DI64Clamp;

3174

case Intrinsic::nvvm_suld_2d_v2i8_clamp:

3175

return NVPTXISD::Suld2DV2I8Clamp;

3176

case Intrinsic::nvvm_suld_2d_v2i16_clamp:

3177

return NVPTXISD::Suld2DV2I16Clamp;

3178

case Intrinsic::nvvm_suld_2d_v2i32_clamp:

3179

return NVPTXISD::Suld2DV2I32Clamp;

3180

case Intrinsic::nvvm_suld_2d_v2i64_clamp:

3181

return NVPTXISD::Suld2DV2I64Clamp;

3182

case Intrinsic::nvvm_suld_2d_v4i8_clamp:

3183

return NVPTXISD::Suld2DV4I8Clamp;

3184

case Intrinsic::nvvm_suld_2d_v4i16_clamp:

3185

return NVPTXISD::Suld2DV4I16Clamp;

3186

case Intrinsic::nvvm_suld_2d_v4i32_clamp:

3187

return NVPTXISD::Suld2DV4I32Clamp;

3188

case Intrinsic::nvvm_suld_2d_array_i8_clamp:

3189

return NVPTXISD::Suld2DArrayI8Clamp;

3190

case Intrinsic::nvvm_suld_2d_array_i16_clamp:

3191

return NVPTXISD::Suld2DArrayI16Clamp;

3192

case Intrinsic::nvvm_suld_2d_array_i32_clamp:

3193

return NVPTXISD::Suld2DArrayI32Clamp;

3194

case Intrinsic::nvvm_suld_2d_array_i64_clamp:

3195

return NVPTXISD::Suld2DArrayI64Clamp;

3196

case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:

3197

return NVPTXISD::Suld2DArrayV2I8Clamp;

3198

case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:

3199

return NVPTXISD::Suld2DArrayV2I16Clamp;

3200

case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:

3201

return NVPTXISD::Suld2DArrayV2I32Clamp;

3202

case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:

3203

return NVPTXISD::Suld2DArrayV2I64Clamp;

3204

case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:

3205

return NVPTXISD::Suld2DArrayV4I8Clamp;

3206

case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:

3207

return NVPTXISD::Suld2DArrayV4I16Clamp;

3208

case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:

3209

return NVPTXISD::Suld2DArrayV4I32Clamp;

3210

case Intrinsic::nvvm_suld_3d_i8_clamp:

3211

return NVPTXISD::Suld3DI8Clamp;

3212

case Intrinsic::nvvm_suld_3d_i16_clamp:

3213

return NVPTXISD::Suld3DI16Clamp;

3214

case Intrinsic::nvvm_suld_3d_i32_clamp:

3215

return NVPTXISD::Suld3DI32Clamp;

3216

case Intrinsic::nvvm_suld_3d_i64_clamp:

3217

return NVPTXISD::Suld3DI64Clamp;

3218

case Intrinsic::nvvm_suld_3d_v2i8_clamp:

3219

return NVPTXISD::Suld3DV2I8Clamp;

3220

case Intrinsic::nvvm_suld_3d_v2i16_clamp:

3221

return NVPTXISD::Suld3DV2I16Clamp;

3222

case Intrinsic::nvvm_suld_3d_v2i32_clamp:

3223

return NVPTXISD::Suld3DV2I32Clamp;

3224

case Intrinsic::nvvm_suld_3d_v2i64_clamp:

3225

return NVPTXISD::Suld3DV2I64Clamp;

3226

case Intrinsic::nvvm_suld_3d_v4i8_clamp:

3227

return NVPTXISD::Suld3DV4I8Clamp;

3228

case Intrinsic::nvvm_suld_3d_v4i16_clamp:

3229

return NVPTXISD::Suld3DV4I16Clamp;

3230

case Intrinsic::nvvm_suld_3d_v4i32_clamp:

3231

return NVPTXISD::Suld3DV4I32Clamp;

3232

case Intrinsic::nvvm_suld_1d_i8_trap:

3233

return NVPTXISD::Suld1DI8Trap;

3234

case Intrinsic::nvvm_suld_1d_i16_trap:

3235

return NVPTXISD::Suld1DI16Trap;

3236

case Intrinsic::nvvm_suld_1d_i32_trap:

3237

return NVPTXISD::Suld1DI32Trap;

3238

case Intrinsic::nvvm_suld_1d_i64_trap:

3239

return NVPTXISD::Suld1DI64Trap;

3240

case Intrinsic::nvvm_suld_1d_v2i8_trap:

3241

return NVPTXISD::Suld1DV2I8Trap;

3242

case Intrinsic::nvvm_suld_1d_v2i16_trap:

3243

return NVPTXISD::Suld1DV2I16Trap;

3244

case Intrinsic::nvvm_suld_1d_v2i32_trap:

3245

return NVPTXISD::Suld1DV2I32Trap;

3246

case Intrinsic::nvvm_suld_1d_v2i64_trap:

3247

return NVPTXISD::Suld1DV2I64Trap;

3248

case Intrinsic::nvvm_suld_1d_v4i8_trap:

3249

return NVPTXISD::Suld1DV4I8Trap;

3250

case Intrinsic::nvvm_suld_1d_v4i16_trap:

3251

return NVPTXISD::Suld1DV4I16Trap;

3252

case Intrinsic::nvvm_suld_1d_v4i32_trap:

3253

return NVPTXISD::Suld1DV4I32Trap;

3254

case Intrinsic::nvvm_suld_1d_array_i8_trap:

3255

return NVPTXISD::Suld1DArrayI8Trap;

3256

case Intrinsic::nvvm_suld_1d_array_i16_trap:

3257

return NVPTXISD::Suld1DArrayI16Trap;

3258

case Intrinsic::nvvm_suld_1d_array_i32_trap:

3259

return NVPTXISD::Suld1DArrayI32Trap;

3260

case Intrinsic::nvvm_suld_1d_array_i64_trap:

3261

return NVPTXISD::Suld1DArrayI64Trap;

3262

case Intrinsic::nvvm_suld_1d_array_v2i8_trap:

3263

return NVPTXISD::Suld1DArrayV2I8Trap;

3264

case Intrinsic::nvvm_suld_1d_array_v2i16_trap:

3265

return NVPTXISD::Suld1DArrayV2I16Trap;

3266

case Intrinsic::nvvm_suld_1d_array_v2i32_trap:

3267

return NVPTXISD::Suld1DArrayV2I32Trap;

3268

case Intrinsic::nvvm_suld_1d_array_v2i64_trap:

3269

return NVPTXISD::Suld1DArrayV2I64Trap;

3270

case Intrinsic::nvvm_suld_1d_array_v4i8_trap:

3271

return NVPTXISD::Suld1DArrayV4I8Trap;

3272

case Intrinsic::nvvm_suld_1d_array_v4i16_trap:

3273

return NVPTXISD::Suld1DArrayV4I16Trap;

3274

case Intrinsic::nvvm_suld_1d_array_v4i32_trap:

3275

return NVPTXISD::Suld1DArrayV4I32Trap;

3276

case Intrinsic::nvvm_suld_2d_i8_trap:

3277

return NVPTXISD::Suld2DI8Trap;

3278

case Intrinsic::nvvm_suld_2d_i16_trap:

3279

return NVPTXISD::Suld2DI16Trap;

3280

case Intrinsic::nvvm_suld_2d_i32_trap:

3281

return NVPTXISD::Suld2DI32Trap;

3282

case Intrinsic::nvvm_suld_2d_i64_trap:

3283

return NVPTXISD::Suld2DI64Trap;

3284

case Intrinsic::nvvm_suld_2d_v2i8_trap:

3285

return NVPTXISD::Suld2DV2I8Trap;

3286

case Intrinsic::nvvm_suld_2d_v2i16_trap:

3287

return NVPTXISD::Suld2DV2I16Trap;

3288

case Intrinsic::nvvm_suld_2d_v2i32_trap:

3289

return NVPTXISD::Suld2DV2I32Trap;

3290

case Intrinsic::nvvm_suld_2d_v2i64_trap:

3291

return NVPTXISD::Suld2DV2I64Trap;

3292

case Intrinsic::nvvm_suld_2d_v4i8_trap:

3293

return NVPTXISD::Suld2DV4I8Trap;

3294

case Intrinsic::nvvm_suld_2d_v4i16_trap:

3295

return NVPTXISD::Suld2DV4I16Trap;

3296

case Intrinsic::nvvm_suld_2d_v4i32_trap:

3297

return NVPTXISD::Suld2DV4I32Trap;

3298

case Intrinsic::nvvm_suld_2d_array_i8_trap:

3299

return NVPTXISD::Suld2DArrayI8Trap;

3300

case Intrinsic::nvvm_suld_2d_array_i16_trap:

3301

return NVPTXISD::Suld2DArrayI16Trap;

3302

case Intrinsic::nvvm_suld_2d_array_i32_trap:

3303

return NVPTXISD::Suld2DArrayI32Trap;

3304

case Intrinsic::nvvm_suld_2d_array_i64_trap:

3305

return NVPTXISD::Suld2DArrayI64Trap;

3306

case Intrinsic::nvvm_suld_2d_array_v2i8_trap:

3307

return NVPTXISD::Suld2DArrayV2I8Trap;

3308

case Intrinsic::nvvm_suld_2d_array_v2i16_trap:

3309

return NVPTXISD::Suld2DArrayV2I16Trap;

3310

case Intrinsic::nvvm_suld_2d_array_v2i32_trap:

3311

return NVPTXISD::Suld2DArrayV2I32Trap;

3312

case Intrinsic::nvvm_suld_2d_array_v2i64_trap:

3313

return NVPTXISD::Suld2DArrayV2I64Trap;

3314

case Intrinsic::nvvm_suld_2d_array_v4i8_trap:

3315

return NVPTXISD::Suld2DArrayV4I8Trap;

3316

case Intrinsic::nvvm_suld_2d_array_v4i16_trap:

3317

return NVPTXISD::Suld2DArrayV4I16Trap;

3318

case Intrinsic::nvvm_suld_2d_array_v4i32_trap:

3319

return NVPTXISD::Suld2DArrayV4I32Trap;

3320

case Intrinsic::nvvm_suld_3d_i8_trap:

3321

return NVPTXISD::Suld3DI8Trap;

3322

case Intrinsic::nvvm_suld_3d_i16_trap:

3323

return NVPTXISD::Suld3DI16Trap;

3324

case Intrinsic::nvvm_suld_3d_i32_trap:

3325

return NVPTXISD::Suld3DI32Trap;

3326

case Intrinsic::nvvm_suld_3d_i64_trap:

3327

return NVPTXISD::Suld3DI64Trap;

3328

case Intrinsic::nvvm_suld_3d_v2i8_trap:

3329

return NVPTXISD::Suld3DV2I8Trap;

3330

case Intrinsic::nvvm_suld_3d_v2i16_trap:

3331

return NVPTXISD::Suld3DV2I16Trap;

3332

case Intrinsic::nvvm_suld_3d_v2i32_trap:

3333

return NVPTXISD::Suld3DV2I32Trap;

3334

case Intrinsic::nvvm_suld_3d_v2i64_trap:

3335

return NVPTXISD::Suld3DV2I64Trap;

3336

case Intrinsic::nvvm_suld_3d_v4i8_trap:

3337

return NVPTXISD::Suld3DV4I8Trap;

3338

case Intrinsic::nvvm_suld_3d_v4i16_trap:

3339

return NVPTXISD::Suld3DV4I16Trap;

3340

case Intrinsic::nvvm_suld_3d_v4i32_trap:

3341

return NVPTXISD::Suld3DV4I32Trap;

3342

case Intrinsic::nvvm_suld_1d_i8_zero:

3343

return NVPTXISD::Suld1DI8Zero;

3344

case Intrinsic::nvvm_suld_1d_i16_zero:

3345

return NVPTXISD::Suld1DI16Zero;

3346

case Intrinsic::nvvm_suld_1d_i32_zero:

3347

return NVPTXISD::Suld1DI32Zero;

3348

case Intrinsic::nvvm_suld_1d_i64_zero:

3349

return NVPTXISD::Suld1DI64Zero;

3350

case Intrinsic::nvvm_suld_1d_v2i8_zero:

3351

return NVPTXISD::Suld1DV2I8Zero;

3352

case Intrinsic::nvvm_suld_1d_v2i16_zero:

3353

return NVPTXISD::Suld1DV2I16Zero;

3354

case Intrinsic::nvvm_suld_1d_v2i32_zero:

3355

return NVPTXISD::Suld1DV2I32Zero;

3356

case Intrinsic::nvvm_suld_1d_v2i64_zero:

3357

return NVPTXISD::Suld1DV2I64Zero;

3358

case Intrinsic::nvvm_suld_1d_v4i8_zero:

3359

return NVPTXISD::Suld1DV4I8Zero;

3360

case Intrinsic::nvvm_suld_1d_v4i16_zero:

3361

return NVPTXISD::Suld1DV4I16Zero;

3362

case Intrinsic::nvvm_suld_1d_v4i32_zero:

3363

return NVPTXISD::Suld1DV4I32Zero;

3364

case Intrinsic::nvvm_suld_1d_array_i8_zero:

3365

return NVPTXISD::Suld1DArrayI8Zero;

3366

case Intrinsic::nvvm_suld_1d_array_i16_zero:

3367

return NVPTXISD::Suld1DArrayI16Zero;

3368

case Intrinsic::nvvm_suld_1d_array_i32_zero:

3369

return NVPTXISD::Suld1DArrayI32Zero;

3370

case Intrinsic::nvvm_suld_1d_array_i64_zero:

3371

return NVPTXISD::Suld1DArrayI64Zero;

3372

case Intrinsic::nvvm_suld_1d_array_v2i8_zero:

3373

return NVPTXISD::Suld1DArrayV2I8Zero;

3374

case Intrinsic::nvvm_suld_1d_array_v2i16_zero:

3375

return NVPTXISD::Suld1DArrayV2I16Zero;

3376

case Intrinsic::nvvm_suld_1d_array_v2i32_zero:

3377

return NVPTXISD::Suld1DArrayV2I32Zero;

3378

case Intrinsic::nvvm_suld_1d_array_v2i64_zero:

3379

return NVPTXISD::Suld1DArrayV2I64Zero;

3380

case Intrinsic::nvvm_suld_1d_array_v4i8_zero:

3381

return NVPTXISD::Suld1DArrayV4I8Zero;

3382

case Intrinsic::nvvm_suld_1d_array_v4i16_zero:

3383

return NVPTXISD::Suld1DArrayV4I16Zero;

3384

case Intrinsic::nvvm_suld_1d_array_v4i32_zero:

3385

return NVPTXISD::Suld1DArrayV4I32Zero;

3386

case Intrinsic::nvvm_suld_2d_i8_zero:

3387

return NVPTXISD::Suld2DI8Zero;

3388

case Intrinsic::nvvm_suld_2d_i16_zero:

3389

return NVPTXISD::Suld2DI16Zero;

3390

case Intrinsic::nvvm_suld_2d_i32_zero:

3391

return NVPTXISD::Suld2DI32Zero;

3392

case Intrinsic::nvvm_suld_2d_i64_zero:

3393

return NVPTXISD::Suld2DI64Zero;

3394

case Intrinsic::nvvm_suld_2d_v2i8_zero:

3395

return NVPTXISD::Suld2DV2I8Zero;

3396

case Intrinsic::nvvm_suld_2d_v2i16_zero:

3397

return NVPTXISD::Suld2DV2I16Zero;

3398

case Intrinsic::nvvm_suld_2d_v2i32_zero:

3399

return NVPTXISD::Suld2DV2I32Zero;

3400

case Intrinsic::nvvm_suld_2d_v2i64_zero:

3401

return NVPTXISD::Suld2DV2I64Zero;

3402

case Intrinsic::nvvm_suld_2d_v4i8_zero:

3403

return NVPTXISD::Suld2DV4I8Zero;

3404

case Intrinsic::nvvm_suld_2d_v4i16_zero:

3405

return NVPTXISD::Suld2DV4I16Zero;

3406

case Intrinsic::nvvm_suld_2d_v4i32_zero:

3407

return NVPTXISD::Suld2DV4I32Zero;

3408

case Intrinsic::nvvm_suld_2d_array_i8_zero:

3409

return NVPTXISD::Suld2DArrayI8Zero;

3410

case Intrinsic::nvvm_suld_2d_array_i16_zero:

3411

return NVPTXISD::Suld2DArrayI16Zero;

3412

case Intrinsic::nvvm_suld_2d_array_i32_zero:

3413

return NVPTXISD::Suld2DArrayI32Zero;

3414

case Intrinsic::nvvm_suld_2d_array_i64_zero:

3415

return NVPTXISD::Suld2DArrayI64Zero;

3416

case Intrinsic::nvvm_suld_2d_array_v2i8_zero:

3417

return NVPTXISD::Suld2DArrayV2I8Zero;

3418

case Intrinsic::nvvm_suld_2d_array_v2i16_zero:

3419

return NVPTXISD::Suld2DArrayV2I16Zero;

3420

case Intrinsic::nvvm_suld_2d_array_v2i32_zero:

3421

return NVPTXISD::Suld2DArrayV2I32Zero;

3422

case Intrinsic::nvvm_suld_2d_array_v2i64_zero:

3423

return NVPTXISD::Suld2DArrayV2I64Zero;

3424

case Intrinsic::nvvm_suld_2d_array_v4i8_zero:

3425

return NVPTXISD::Suld2DArrayV4I8Zero;

3426

case Intrinsic::nvvm_suld_2d_array_v4i16_zero:

3427

return NVPTXISD::Suld2DArrayV4I16Zero;

3428

case Intrinsic::nvvm_suld_2d_array_v4i32_zero:

3429

return NVPTXISD::Suld2DArrayV4I32Zero;

3430

case Intrinsic::nvvm_suld_3d_i8_zero:

3431

return NVPTXISD::Suld3DI8Zero;

3432

case Intrinsic::nvvm_suld_3d_i16_zero:

3433

return NVPTXISD::Suld3DI16Zero;

3434

case Intrinsic::nvvm_suld_3d_i32_zero:

3435

return NVPTXISD::Suld3DI32Zero;

3436

case Intrinsic::nvvm_suld_3d_i64_zero:

3437

return NVPTXISD::Suld3DI64Zero;

3438

case Intrinsic::nvvm_suld_3d_v2i8_zero:

3439

return NVPTXISD::Suld3DV2I8Zero;

3440

case Intrinsic::nvvm_suld_3d_v2i16_zero:

3441

return NVPTXISD::Suld3DV2I16Zero;

3442

case Intrinsic::nvvm_suld_3d_v2i32_zero:

3443

return NVPTXISD::Suld3DV2I32Zero;

3444

case Intrinsic::nvvm_suld_3d_v2i64_zero:

3445

return NVPTXISD::Suld3DV2I64Zero;

3446

case Intrinsic::nvvm_suld_3d_v4i8_zero:

3447

return NVPTXISD::Suld3DV4I8Zero;

3448

case Intrinsic::nvvm_suld_3d_v4i16_zero:

3449

return NVPTXISD::Suld3DV4I16Zero;

3450

case Intrinsic::nvvm_suld_3d_v4i32_zero:

3451

return NVPTXISD::Suld3DV4I32Zero;

3452

}

3453

}

3454

3455

// llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as

3456

// TgtMemIntrinsic

3457

// because we need the information that is only available in the "Value" type

3458

// of destination

3459

// pointer. In particular, the address space information.

3460

bool NVPTXTargetLowering::getTgtMemIntrinsic(

3461

IntrinsicInfo &Info, const CallInst &I, unsigned Intrinsic) const {

3462

switch (Intrinsic) {

3463

default:

3464

return false;

3465

3466

case Intrinsic::nvvm_atomic_load_add_f32:

3467

case Intrinsic::nvvm_atomic_load_inc_32:

3468

case Intrinsic::nvvm_atomic_load_dec_32:

3469

3470

case Intrinsic::nvvm_atomic_add_gen_f_cta:

3471

case Intrinsic::nvvm_atomic_add_gen_f_sys:

3472

case Intrinsic::nvvm_atomic_add_gen_i_cta:

3473

case Intrinsic::nvvm_atomic_add_gen_i_sys:

3474

case Intrinsic::nvvm_atomic_and_gen_i_cta:

3475

case Intrinsic::nvvm_atomic_and_gen_i_sys:

3476

case Intrinsic::nvvm_atomic_cas_gen_i_cta:

3477

case Intrinsic::nvvm_atomic_cas_gen_i_sys:

3478

case Intrinsic::nvvm_atomic_dec_gen_i_cta:

3479

case Intrinsic::nvvm_atomic_dec_gen_i_sys:

3480

case Intrinsic::nvvm_atomic_inc_gen_i_cta:

3481

case Intrinsic::nvvm_atomic_inc_gen_i_sys:

3482

case Intrinsic::nvvm_atomic_max_gen_i_cta:

3483

case Intrinsic::nvvm_atomic_max_gen_i_sys:

3484

case Intrinsic::nvvm_atomic_min_gen_i_cta:

3485

case Intrinsic::nvvm_atomic_min_gen_i_sys:

3486

case Intrinsic::nvvm_atomic_or_gen_i_cta:

3487

case Intrinsic::nvvm_atomic_or_gen_i_sys:

3488

case Intrinsic::nvvm_atomic_exch_gen_i_cta:

3489

case Intrinsic::nvvm_atomic_exch_gen_i_sys:

3490

case Intrinsic::nvvm_atomic_xor_gen_i_cta:

3491

case Intrinsic::nvvm_atomic_xor_gen_i_sys: {

3492

auto &DL = I.getModule()->getDataLayout();

3493

Info.opc = ISD::INTRINSIC_W_CHAIN;

3494

Info.memVT = getValueType(DL, I.getType());

3495

Info.ptrVal = I.getArgOperand(0);

3496

Info.offset = 0;

3497

Info.vol = false;

3498

Info.readMem = true;

3499

Info.writeMem = true;

3500

Info.align = 0;

3501

return true;

3502

}

3503

3504

case Intrinsic::nvvm_ldu_global_i:

3505

case Intrinsic::nvvm_ldu_global_f:

3506

case Intrinsic::nvvm_ldu_global_p: {

3507

auto &DL = I.getModule()->getDataLayout();

3508

Info.opc = ISD::INTRINSIC_W_CHAIN;

3509

if (Intrinsic == Intrinsic::nvvm_ldu_global_i)

3510

Info.memVT = getValueType(DL, I.getType());

3511

else if(Intrinsic == Intrinsic::nvvm_ldu_global_p)

3512

Info.memVT = getPointerTy(DL);

3513

else

3514

Info.memVT = getValueType(DL, I.getType());

3515

Info.ptrVal = I.getArgOperand(0);

3516

Info.offset = 0;

3517

Info.vol = false;

3518

Info.readMem = true;

3519

Info.writeMem = false;

3520

Info.align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();

3521

3522

return true;

3523

}

3524

case Intrinsic::nvvm_ldg_global_i:

3525

case Intrinsic::nvvm_ldg_global_f:

3526

case Intrinsic::nvvm_ldg_global_p: {

3527

auto &DL = I.getModule()->getDataLayout();

3528

3529

Info.opc = ISD::INTRINSIC_W_CHAIN;

3530

if (Intrinsic == Intrinsic::nvvm_ldg_global_i)

3531

Info.memVT = getValueType(DL, I.getType());

3532

else if(Intrinsic == Intrinsic::nvvm_ldg_global_p)

3533

Info.memVT = getPointerTy(DL);

3534

else

3535

Info.memVT = getValueType(DL, I.getType());

3536

Info.ptrVal = I.getArgOperand(0);

3537

Info.offset = 0;

3538

Info.vol = false;

3539

Info.readMem = true;

3540

Info.writeMem = false;

3541

Info.align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();

3542

3543

return true;

3544

}

3545

3546

case Intrinsic::nvvm_tex_1d_v4f32_s32:

3547

case Intrinsic::nvvm_tex_1d_v4f32_f32:

3548

case Intrinsic::nvvm_tex_1d_level_v4f32_f32:

3549

case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:

3550

case Intrinsic::nvvm_tex_1d_array_v4f32_s32:

3551

case Intrinsic::nvvm_tex_1d_array_v4f32_f32:

3552

case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:

3553

case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:

3554

case Intrinsic::nvvm_tex_2d_v4f32_s32:

3555

case Intrinsic::nvvm_tex_2d_v4f32_f32:

3556

case Intrinsic::nvvm_tex_2d_level_v4f32_f32:

3557

case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:

3558

case Intrinsic::nvvm_tex_2d_array_v4f32_s32:

3559

case Intrinsic::nvvm_tex_2d_array_v4f32_f32:

3560

case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:

3561

case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:

3562

case Intrinsic::nvvm_tex_3d_v4f32_s32:

3563

case Intrinsic::nvvm_tex_3d_v4f32_f32:

3564

case Intrinsic::nvvm_tex_3d_level_v4f32_f32:

3565

case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:

3566

case Intrinsic::nvvm_tex_cube_v4f32_f32:

3567

case Intrinsic::nvvm_tex_cube_level_v4f32_f32:

3568

case Intrinsic::nvvm_tex_cube_array_v4f32_f32:

3569

case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:

3570

case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:

3571

case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:

3572

case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:

3573

case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:

3574

case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:

3575

case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:

3576

case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:

3577

case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:

3578

case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:

3579

case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:

3580

case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:

3581

case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:

3582

case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:

3583

case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:

3584

case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:

3585

case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:

3586

case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:

3587

case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:

3588

case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:

3589

case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:

3590

case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:

3591

case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:

3592

case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:

3593

case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:

3594

case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:

3595

case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:

3596

case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:

3597

case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:

3598

case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:

3599

case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:

3600

case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:

3601

case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:

3602

Info.opc = getOpcForTextureInstr(Intrinsic);

3603

Info.memVT = MVT::v4f32;

3604

Info.ptrVal = nullptr;

3605

Info.offset = 0;

3606

Info.vol = false;

3607

Info.readMem = true;

3608

Info.writeMem = false;

3609

Info.align = 16;

3610

return true;

3611

3612

case Intrinsic::nvvm_tex_1d_v4s32_s32:

3613

case Intrinsic::nvvm_tex_1d_v4s32_f32:

3614

case Intrinsic::nvvm_tex_1d_level_v4s32_f32:

3615

case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:

3616

case Intrinsic::nvvm_tex_1d_array_v4s32_s32:

3617

case Intrinsic::nvvm_tex_1d_array_v4s32_f32:

3618

case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:

3619

case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:

3620

case Intrinsic::nvvm_tex_2d_v4s32_s32:

3621

case Intrinsic::nvvm_tex_2d_v4s32_f32:

3622

case Intrinsic::nvvm_tex_2d_level_v4s32_f32:

3623

case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:

3624

case Intrinsic::nvvm_tex_2d_array_v4s32_s32:

3625

case Intrinsic::nvvm_tex_2d_array_v4s32_f32:

3626

case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:

3627

case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:

3628

case Intrinsic::nvvm_tex_3d_v4s32_s32:

3629

case Intrinsic::nvvm_tex_3d_v4s32_f32:

3630

case Intrinsic::nvvm_tex_3d_level_v4s32_f32:

3631

case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:

3632

case Intrinsic::nvvm_tex_cube_v4s32_f32:

3633

case Intrinsic::nvvm_tex_cube_level_v4s32_f32:

3634

case Intrinsic::nvvm_tex_cube_array_v4s32_f32:

3635

case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:

3636

case Intrinsic::nvvm_tex_cube_v4u32_f32:

3637

case Intrinsic::nvvm_tex_cube_level_v4u32_f32:

3638

case Intrinsic::nvvm_tex_cube_array_v4u32_f32:

3639

case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:

3640

case Intrinsic::nvvm_tex_1d_v4u32_s32:

3641

case Intrinsic::nvvm_tex_1d_v4u32_f32:

3642

case Intrinsic::nvvm_tex_1d_level_v4u32_f32:

3643

case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:

3644

case Intrinsic::nvvm_tex_1d_array_v4u32_s32:

3645

case Intrinsic::nvvm_tex_1d_array_v4u32_f32:

3646

case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:

3647

case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:

3648

case Intrinsic::nvvm_tex_2d_v4u32_s32:

3649

case Intrinsic::nvvm_tex_2d_v4u32_f32:

3650

case Intrinsic::nvvm_tex_2d_level_v4u32_f32:

3651

case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:

3652

case Intrinsic::nvvm_tex_2d_array_v4u32_s32:

3653

case Intrinsic::nvvm_tex_2d_array_v4u32_f32:

3654

case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:

3655

case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:

3656

case Intrinsic::nvvm_tex_3d_v4u32_s32:

3657

case Intrinsic::nvvm_tex_3d_v4u32_f32:

3658

case Intrinsic::nvvm_tex_3d_level_v4u32_f32:

3659

case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:

3660

case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:

3661

case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:

3662

case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:

3663

case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:

3664

case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:

3665

case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:

3666

case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:

3667

case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:

3668

case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:

3669

case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:

3670

case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:

3671

case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:

3672

case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:

3673

case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:

3674

case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:

3675

case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:

3676

case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:

3677

case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:

3678

case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:

3679

case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:

3680

case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:

3681

case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:

3682

case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:

3683

case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:

3684

case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:

3685

case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:

3686

case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:

3687

case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:

3688

case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:

3689

case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:

3690

case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:

3691

case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:

3692

case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:

3693

case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:

3694

case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:

3695

case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:

3696

case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:

3697

case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:

3698

case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:

3699

case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:

3700

case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:

3701

case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:

3702

case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:

3703

case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:

3704

case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:

3705

case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:

3706

case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:

3707

case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:

3708

case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:

3709

case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:

3710

case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:

3711

case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:

3712

case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:

3713

case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:

3714

case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:

3715

case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:

3716

case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:

3717

case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:

3718

case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:

3719

case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:

3720

case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:

3721

case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:

3722

case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:

3723

case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:

3724

Info.opc = getOpcForTextureInstr(Intrinsic);

3725

Info.memVT = MVT::v4i32;

3726

Info.ptrVal = nullptr;

3727

Info.offset = 0;

3728

Info.vol = false;

3729

Info.readMem = true;

3730

Info.writeMem = false;

3731

Info.align = 16;

3732

return true;

3733

3734

case Intrinsic::nvvm_suld_1d_i8_clamp:

3735

case Intrinsic::nvvm_suld_1d_v2i8_clamp:

3736

case Intrinsic::nvvm_suld_1d_v4i8_clamp:

3737

case Intrinsic::nvvm_suld_1d_array_i8_clamp:

3738

case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:

3739

case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:

3740

case Intrinsic::nvvm_suld_2d_i8_clamp:

3741

case Intrinsic::nvvm_suld_2d_v2i8_clamp:

3742

case Intrinsic::nvvm_suld_2d_v4i8_clamp:

3743

case Intrinsic::nvvm_suld_2d_array_i8_clamp:

3744

case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:

3745

case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:

3746

case Intrinsic::nvvm_suld_3d_i8_clamp:

3747

case Intrinsic::nvvm_suld_3d_v2i8_clamp:

3748

case Intrinsic::nvvm_suld_3d_v4i8_clamp:

3749

case Intrinsic::nvvm_suld_1d_i8_trap:

3750

case Intrinsic::nvvm_suld_1d_v2i8_trap:

3751

case Intrinsic::nvvm_suld_1d_v4i8_trap:

3752

case Intrinsic::nvvm_suld_1d_array_i8_trap:

3753

case Intrinsic::nvvm_suld_1d_array_v2i8_trap:

3754

case Intrinsic::nvvm_suld_1d_array_v4i8_trap:

3755

case Intrinsic::nvvm_suld_2d_i8_trap:

3756

case Intrinsic::nvvm_suld_2d_v2i8_trap:

3757

case Intrinsic::nvvm_suld_2d_v4i8_trap:

3758

case Intrinsic::nvvm_suld_2d_array_i8_trap:

3759

case Intrinsic::nvvm_suld_2d_array_v2i8_trap:

3760

case Intrinsic::nvvm_suld_2d_array_v4i8_trap:

3761

case Intrinsic::nvvm_suld_3d_i8_trap:

3762

case Intrinsic::nvvm_suld_3d_v2i8_trap:

3763

case Intrinsic::nvvm_suld_3d_v4i8_trap:

3764

case Intrinsic::nvvm_suld_1d_i8_zero:

3765

case Intrinsic::nvvm_suld_1d_v2i8_zero:

3766

case Intrinsic::nvvm_suld_1d_v4i8_zero:

3767

case Intrinsic::nvvm_suld_1d_array_i8_zero:

3768

case Intrinsic::nvvm_suld_1d_array_v2i8_zero:

3769

case Intrinsic::nvvm_suld_1d_array_v4i8_zero:

3770

case Intrinsic::nvvm_suld_2d_i8_zero:

3771

case Intrinsic::nvvm_suld_2d_v2i8_zero:

3772

case Intrinsic::nvvm_suld_2d_v4i8_zero:

3773

case Intrinsic::nvvm_suld_2d_array_i8_zero:

3774

case Intrinsic::nvvm_suld_2d_array_v2i8_zero:

3775

case Intrinsic::nvvm_suld_2d_array_v4i8_zero:

3776

case Intrinsic::nvvm_suld_3d_i8_zero:

3777

case Intrinsic::nvvm_suld_3d_v2i8_zero:

3778

case Intrinsic::nvvm_suld_3d_v4i8_zero:

3779

Info.opc = getOpcForSurfaceInstr(Intrinsic);

3780

Info.memVT = MVT::i8;

3781

Info.ptrVal = nullptr;

3782

Info.offset = 0;

3783

Info.vol = false;

3784

Info.readMem = true;

3785

Info.writeMem = false;

3786

Info.align = 16;

3787

return true;

3788

3789

case Intrinsic::nvvm_suld_1d_i16_clamp:

3790

case Intrinsic::nvvm_suld_1d_v2i16_clamp:

3791

case Intrinsic::nvvm_suld_1d_v4i16_clamp:

3792

case Intrinsic::nvvm_suld_1d_array_i16_clamp:

3793

case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:

3794

case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:

3795

case Intrinsic::nvvm_suld_2d_i16_clamp:

3796

case Intrinsic::nvvm_suld_2d_v2i16_clamp:

3797

case Intrinsic::nvvm_suld_2d_v4i16_clamp:

3798

case Intrinsic::nvvm_suld_2d_array_i16_clamp:

3799

case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:

3800

case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:

3801

case Intrinsic::nvvm_suld_3d_i16_clamp:

3802

case Intrinsic::nvvm_suld_3d_v2i16_clamp:

3803

case Intrinsic::nvvm_suld_3d_v4i16_clamp:

3804

case Intrinsic::nvvm_suld_1d_i16_trap:

3805

case Intrinsic::nvvm_suld_1d_v2i16_trap:

3806

case Intrinsic::nvvm_suld_1d_v4i16_trap:

3807

case Intrinsic::nvvm_suld_1d_array_i16_trap:

3808

case Intrinsic::nvvm_suld_1d_array_v2i16_trap:

3809

case Intrinsic::nvvm_suld_1d_array_v4i16_trap:

3810

case Intrinsic::nvvm_suld_2d_i16_trap:

3811

case Intrinsic::nvvm_suld_2d_v2i16_trap:

3812

case Intrinsic::nvvm_suld_2d_v4i16_trap:

3813

case Intrinsic::nvvm_suld_2d_array_i16_trap:

3814

case Intrinsic::nvvm_suld_2d_array_v2i16_trap:

3815

case Intrinsic::nvvm_suld_2d_array_v4i16_trap:

3816

case Intrinsic::nvvm_suld_3d_i16_trap:

3817

case Intrinsic::nvvm_suld_3d_v2i16_trap:

3818

case Intrinsic::nvvm_suld_3d_v4i16_trap:

3819

case Intrinsic::nvvm_suld_1d_i16_zero:

3820

case Intrinsic::nvvm_suld_1d_v2i16_zero:

3821

case Intrinsic::nvvm_suld_1d_v4i16_zero:

3822

case Intrinsic::nvvm_suld_1d_array_i16_zero:

3823

case Intrinsic::nvvm_suld_1d_array_v2i16_zero:

3824

case Intrinsic::nvvm_suld_1d_array_v4i16_zero:

3825

case Intrinsic::nvvm_suld_2d_i16_zero:

3826

case Intrinsic::nvvm_suld_2d_v2i16_zero:

3827

case Intrinsic::nvvm_suld_2d_v4i16_zero:

3828

case Intrinsic::nvvm_suld_2d_array_i16_zero:

3829

case Intrinsic::nvvm_suld_2d_array_v2i16_zero:

3830

case Intrinsic::nvvm_suld_2d_array_v4i16_zero:

3831

case Intrinsic::nvvm_suld_3d_i16_zero:

3832

case Intrinsic::nvvm_suld_3d_v2i16_zero:

3833

case Intrinsic::nvvm_suld_3d_v4i16_zero:

3834

Info.opc = getOpcForSurfaceInstr(Intrinsic);

3835

Info.memVT = MVT::i16;

3836

Info.ptrVal = nullptr;

3837

Info.offset = 0;

3838

Info.vol = false;

3839

Info.readMem = true;

3840

Info.writeMem = false;

3841

Info.align = 16;

3842

return true;

3843

3844

case Intrinsic::nvvm_suld_1d_i32_clamp:

3845

case Intrinsic::nvvm_suld_1d_v2i32_clamp:

3846

case Intrinsic::nvvm_suld_1d_v4i32_clamp:

3847

case Intrinsic::nvvm_suld_1d_array_i32_clamp:

3848

case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:

3849

case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:

3850

case Intrinsic::nvvm_suld_2d_i32_clamp:

3851

case Intrinsic::nvvm_suld_2d_v2i32_clamp:

3852

case Intrinsic::nvvm_suld_2d_v4i32_clamp:

3853

case Intrinsic::nvvm_suld_2d_array_i32_clamp:

3854

case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:

3855

case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:

3856

case Intrinsic::nvvm_suld_3d_i32_clamp:

3857

case Intrinsic::nvvm_suld_3d_v2i32_clamp:

3858

case Intrinsic::nvvm_suld_3d_v4i32_clamp:

3859

case Intrinsic::nvvm_suld_1d_i32_trap:

3860

case Intrinsic::nvvm_suld_1d_v2i32_trap:

3861

case Intrinsic::nvvm_suld_1d_v4i32_trap:

3862

case Intrinsic::nvvm_suld_1d_array_i32_trap:

3863

case Intrinsic::nvvm_suld_1d_array_v2i32_trap:

3864

case Intrinsic::nvvm_suld_1d_array_v4i32_trap:

3865

case Intrinsic::nvvm_suld_2d_i32_trap:

3866

case Intrinsic::nvvm_suld_2d_v2i32_trap:

3867

case Intrinsic::nvvm_suld_2d_v4i32_trap:

3868

case Intrinsic::nvvm_suld_2d_array_i32_trap:

3869

case Intrinsic::nvvm_suld_2d_array_v2i32_trap:

3870

case Intrinsic::nvvm_suld_2d_array_v4i32_trap:

3871

case Intrinsic::nvvm_suld_3d_i32_trap:

3872

case Intrinsic::nvvm_suld_3d_v2i32_trap:

3873

case Intrinsic::nvvm_suld_3d_v4i32_trap:

3874

case Intrinsic::nvvm_suld_1d_i32_zero:

3875

case Intrinsic::nvvm_suld_1d_v2i32_zero:

3876

case Intrinsic::nvvm_suld_1d_v4i32_zero:

3877

case Intrinsic::nvvm_suld_1d_array_i32_zero:

3878

case Intrinsic::nvvm_suld_1d_array_v2i32_zero:

3879

case Intrinsic::nvvm_suld_1d_array_v4i32_zero:

3880

case Intrinsic::nvvm_suld_2d_i32_zero:

3881

case Intrinsic::nvvm_suld_2d_v2i32_zero:

3882

case Intrinsic::nvvm_suld_2d_v4i32_zero:

3883

case Intrinsic::nvvm_suld_2d_array_i32_zero:

3884

case Intrinsic::nvvm_suld_2d_array_v2i32_zero:

3885

case Intrinsic::nvvm_suld_2d_array_v4i32_zero:

3886

case Intrinsic::nvvm_suld_3d_i32_zero:

3887

case Intrinsic::nvvm_suld_3d_v2i32_zero:

3888

case Intrinsic::nvvm_suld_3d_v4i32_zero:

3889

Info.opc = getOpcForSurfaceInstr(Intrinsic);

3890

Info.memVT = MVT::i32;

3891

Info.ptrVal = nullptr;

3892

Info.offset = 0;

3893

Info.vol = false;

3894

Info.readMem = true;

3895

Info.writeMem = false;

3896

Info.align = 16;

3897

return true;

3898

3899

case Intrinsic::nvvm_suld_1d_i64_clamp:

3900

case Intrinsic::nvvm_suld_1d_v2i64_clamp:

3901

case Intrinsic::nvvm_suld_1d_array_i64_clamp:

3902

case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:

3903

case Intrinsic::nvvm_suld_2d_i64_clamp:

3904

case Intrinsic::nvvm_suld_2d_v2i64_clamp:

3905

case Intrinsic::nvvm_suld_2d_array_i64_clamp:

3906

case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:

3907

case Intrinsic::nvvm_suld_3d_i64_clamp:

3908

case Intrinsic::nvvm_suld_3d_v2i64_clamp:

3909

case Intrinsic::nvvm_suld_1d_i64_trap:

3910

case Intrinsic::nvvm_suld_1d_v2i64_trap:

3911

case Intrinsic::nvvm_suld_1d_array_i64_trap:

3912

case Intrinsic::nvvm_suld_1d_array_v2i64_trap:

3913

case Intrinsic::nvvm_suld_2d_i64_trap:

3914

case Intrinsic::nvvm_suld_2d_v2i64_trap:

3915

case Intrinsic::nvvm_suld_2d_array_i64_trap:

3916

case Intrinsic::nvvm_suld_2d_array_v2i64_trap:

3917

case Intrinsic::nvvm_suld_3d_i64_trap:

3918

case Intrinsic::nvvm_suld_3d_v2i64_trap:

3919

case Intrinsic::nvvm_suld_1d_i64_zero:

3920

case Intrinsic::nvvm_suld_1d_v2i64_zero:

3921

case Intrinsic::nvvm_suld_1d_array_i64_zero:

3922

case Intrinsic::nvvm_suld_1d_array_v2i64_zero:

3923

case Intrinsic::nvvm_suld_2d_i64_zero:

3924

case Intrinsic::nvvm_suld_2d_v2i64_zero:

3925

case Intrinsic::nvvm_suld_2d_array_i64_zero:

3926

case Intrinsic::nvvm_suld_2d_array_v2i64_zero:

3927

case Intrinsic::nvvm_suld_3d_i64_zero:

3928

case Intrinsic::nvvm_suld_3d_v2i64_zero:

3929

Info.opc = getOpcForSurfaceInstr(Intrinsic);

3930

Info.memVT = MVT::i64;

3931

Info.ptrVal = nullptr;

3932

Info.offset = 0;

3933

Info.vol = false;

3934

Info.readMem = true;

3935

Info.writeMem = false;

3936

Info.align = 16;

3937

return true;

3938

}

3939

return false;

3940

}

3941

3942

/// isLegalAddressingMode - Return true if the addressing mode represented

3943

/// by AM is legal for this target, for a load/store of the specified type.

3944

/// Used to guide target specific optimizations, like loop strength reduction

3945

/// (LoopStrengthReduce.cpp) and memory optimization for address mode

3946

/// (CodeGenPrepare.cpp)

3947

bool NVPTXTargetLowering::isLegalAddressingMode(const DataLayout &DL,

3948

const AddrMode &AM, Type *Ty,

3949

unsigned AS) const {

3950

// AddrMode - This represents an addressing mode of:

3951

// BaseGV + BaseOffs + BaseReg + Scale*ScaleReg

3952

3953

// The legal address modes are

3954

// - [avar]

3955

// - [areg]

3956

// - [areg+immoff]

3957

// - [immAddr]

3958

3959

if (AM.BaseGV) {

3960

return !AM.BaseOffs && !AM.HasBaseReg && !AM.Scale;

3961

}

3962

3963

switch (AM.Scale) {

3964

case 0: // "r", "r+i" or "i" is allowed

3965

break;

3966

case 1:

3967

if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed.

3968

return false;

3969

// Otherwise we have r+i.

3970

break;

3971

default:

3972

// No scale > 1 is allowed

3973

return false;

3974

}

3975

return true;

3976

}

3977

3978

//===----------------------------------------------------------------------===//

3979

// NVPTX Inline Assembly Support

3980

//===----------------------------------------------------------------------===//

3981

3982

/// getConstraintType - Given a constraint letter, return the type of

3983

/// constraint it is for this target.

3984

NVPTXTargetLowering::ConstraintType

3985

NVPTXTargetLowering::getConstraintType(StringRef Constraint) const {

3986

if (Constraint.size() == 1) {

3987

switch (Constraint[0]) {

3988

default:

3989

break;

3990

case 'b':

3991

case 'r':

3992

case 'h':

3993

case 'c':

3994

case 'l':

3995

case 'f':

3996

case 'd':

3997

case '0':

3998

case 'N':

3999

return C_RegisterClass;

4000

}

4001

}

4002

return TargetLowering::getConstraintType(Constraint);

4003

}

4004

4005

std::pair<unsigned, const TargetRegisterClass *>

4006

NVPTXTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,

4007

StringRef Constraint,

4008

MVT VT) const {

4009

if (Constraint.size() == 1) {

4010

switch (Constraint[0]) {

4011

case 'b':

4012

return std::make_pair(0U, &NVPTX::Int1RegsRegClass);

4013

case 'c':

4014

return std::make_pair(0U, &NVPTX::Int16RegsRegClass);

4015

case 'h':

4016

return std::make_pair(0U, &NVPTX::Int16RegsRegClass);

4017

case 'r':

4018

return std::make_pair(0U, &NVPTX::Int32RegsRegClass);

4019

case 'l':

4020

case 'N':

4021

return std::make_pair(0U, &NVPTX::Int64RegsRegClass);

4022

case 'f':

4023

return std::make_pair(0U, &NVPTX::Float32RegsRegClass);

4024

case 'd':

4025

return std::make_pair(0U, &NVPTX::Float64RegsRegClass);

4026

}

4027

}

4028

return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);

4029

}

4030

4031

//===----------------------------------------------------------------------===//

4032

// NVPTX DAG Combining

4033

//===----------------------------------------------------------------------===//

4034

4035

bool NVPTXTargetLowering::allowFMA(MachineFunction &MF,

4036

CodeGenOpt::Level OptLevel) const {

4037

// Always honor command-line argument

4038

if (FMAContractLevelOpt.getNumOccurrences() > 0)

4039

return FMAContractLevelOpt > 0;

4040

4041

// Do not contract if we're not optimizing the code.

4042

if (OptLevel == 0)

4043

return false;

4044

4045

// Honor TargetOptions flags that explicitly say fusion is okay.

4046

if (MF.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast)

4047

return true;

4048

4049

return allowUnsafeFPMath(MF);

4050

}

4051

4052

bool NVPTXTargetLowering::allowUnsafeFPMath(MachineFunction &MF) const {

4053

// Honor TargetOptions flags that explicitly say unsafe math is okay.

4054

if (MF.getTarget().Options.UnsafeFPMath)

4055

return true;

4056

4057

// Allow unsafe math if unsafe-fp-math attribute explicitly says so.

4058

const Function *F = MF.getFunction();

4059

if (F->hasFnAttribute("unsafe-fp-math")) {

4060

Attribute Attr = F->getFnAttribute("unsafe-fp-math");

4061

StringRef Val = Attr.getValueAsString();

4062

if (Val == "true")

4063

return true;

4064

}

4065

4066

return false;

4067

}

4068

4069

/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with

4070

/// operands N0 and N1. This is a helper for PerformADDCombine that is

4071

/// called with the default operands, and if that fails, with commuted

4072

/// operands.

4073

static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,

4074

TargetLowering::DAGCombinerInfo &DCI,

4075

const NVPTXSubtarget &Subtarget,

4076

CodeGenOpt::Level OptLevel) {

4077

SelectionDAG &DAG = DCI.DAG;

4078

// Skip non-integer, non-scalar case

4079

EVT VT=N0.getValueType();

4080

if (VT.isVector())

4081

return SDValue();

4082

4083

// fold (add (mul a, b), c) -> (mad a, b, c)

4084

4085

if (N0.getOpcode() == ISD::MUL) {

4086

assert (VT.isInteger())((VT.isInteger()) ? static_cast<void> (0) : __assert_fail
("VT.isInteger()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 4086, __PRETTY_FUNCTION__));

4087

// For integer:

4088

// Since integer multiply-add costs the same as integer multiply

4089

// but is more costly than integer add, do the fusion only when

4090

// the mul is only used in the add.

4091

if (OptLevel==CodeGenOpt::None || VT != MVT::i32 ||

4092

!N0.getNode()->hasOneUse())

4093

return SDValue();

4094

4095

// Do the folding

4096

return DAG.getNode(NVPTXISD::IMAD, SDLoc(N), VT,

4097

N0.getOperand(0), N0.getOperand(1), N1);

4098

}

4099

else if (N0.getOpcode() == ISD::FMUL) {

4100

if (VT == MVT::f32 || VT == MVT::f64) {

4101

const auto *TLI = static_cast<const NVPTXTargetLowering *>(

4102

&DAG.getTargetLoweringInfo());

4103

if (!TLI->allowFMA(DAG.getMachineFunction(), OptLevel))

4104

return SDValue();

4105

4106

// For floating point:

4107

// Do the fusion only when the mul has less than 5 uses and all

4108

// are add.

4109

// The heuristic is that if a use is not an add, then that use

4110

// cannot be fused into fma, therefore mul is still needed anyway.

4111

// If there are more than 4 uses, even if they are all add, fusing

4112

// them will increase register pressue.

4113

4114

int numUses = 0;

4115

int nonAddCount = 0;

4116

for (SDNode::use_iterator UI = N0.getNode()->use_begin(),

4117

UE = N0.getNode()->use_end();

4118

UI != UE; ++UI) {

4119

numUses++;

4120

SDNode *User = *UI;

4121

if (User->getOpcode() != ISD::FADD)

4122

++nonAddCount;

4123

}

4124

if (numUses >= 5)

4125

return SDValue();

4126

if (nonAddCount) {

4127

int orderNo = N->getIROrder();

4128

int orderNo2 = N0.getNode()->getIROrder();

4129

// simple heuristics here for considering potential register

4130

// pressure, the logics here is that the differnce are used

4131

// to measure the distance between def and use, the longer distance

4132

// more likely cause register pressure.

4133

if (orderNo - orderNo2 < 500)

4134

return SDValue();

4135

4136

// Now, check if at least one of the FMUL's operands is live beyond the node N,

4137

// which guarantees that the FMA will not increase register pressure at node N.

4138

bool opIsLive = false;

4139

const SDNode *left = N0.getOperand(0).getNode();

4140

const SDNode *right = N0.getOperand(1).getNode();

4141

4142

if (isa<ConstantSDNode>(left) || isa<ConstantSDNode>(right))

4143

opIsLive = true;

4144

4145

if (!opIsLive)

4146

for (SDNode::use_iterator UI = left->use_begin(), UE = left->use_end(); UI != UE; ++UI) {

4147

SDNode *User = *UI;

4148

int orderNo3 = User->getIROrder();

4149

if (orderNo3 > orderNo) {

4150

opIsLive = true;

4151

break;

4152

}

4153

}

4154

4155

if (!opIsLive)

4156

for (SDNode::use_iterator UI = right->use_begin(), UE = right->use_end(); UI != UE; ++UI) {

4157

SDNode *User = *UI;

4158

int orderNo3 = User->getIROrder();

4159

if (orderNo3 > orderNo) {

4160

opIsLive = true;

4161

break;

4162

}

4163

}

4164

4165

if (!opIsLive)

4166

return SDValue();

4167

}

4168

4169

return DAG.getNode(ISD::FMA, SDLoc(N), VT,

4170

N0.getOperand(0), N0.getOperand(1), N1);

4171

}

4172

}

4173

4174

return SDValue();

4175

}

4176

4177

/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.

4178

///

4179

static SDValue PerformADDCombine(SDNode *N,

4180

TargetLowering::DAGCombinerInfo &DCI,

4181

const NVPTXSubtarget &Subtarget,

4182

CodeGenOpt::Level OptLevel) {

4183

SDValue N0 = N->getOperand(0);

4184

SDValue N1 = N->getOperand(1);

4185

4186

// First try with the default operand order.

4187

if (SDValue Result =

4188

PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget, OptLevel))

4189

return Result;

4190

4191

// If that didn't work, try again with the operands commuted.

4192

return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget, OptLevel);

4193

}

4194

4195

static SDValue PerformANDCombine(SDNode *N,

4196

TargetLowering::DAGCombinerInfo &DCI) {

4197

// The type legalizer turns a vector load of i8 values into a zextload to i16

4198

// registers, optionally ANY_EXTENDs it (if target type is integer),

4199

// and ANDs off the high 8 bits. Since we turn this load into a

4200

// target-specific DAG node, the DAG combiner fails to eliminate these AND

4201

// nodes. Do that here.

4202

SDValue Val = N->getOperand(0);

4203

SDValue Mask = N->getOperand(1);

4204

4205

if (isa<ConstantSDNode>(Val)) {

4206

std::swap(Val, Mask);

4207

}

4208

4209

SDValue AExt;

4210

// Generally, we will see zextload -> IMOV16rr -> ANY_EXTEND -> and

4211

if (Val.getOpcode() == ISD::ANY_EXTEND) {

4212

AExt = Val;

4213

Val = Val->getOperand(0);

4214

}

4215

4216

if (Val->isMachineOpcode() && Val->getMachineOpcode() == NVPTX::IMOV16rr) {

4217

Val = Val->getOperand(0);

4218

}

4219

4220

if (Val->getOpcode() == NVPTXISD::LoadV2 ||

4221

Val->getOpcode() == NVPTXISD::LoadV4) {

4222

ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(Mask);

4223

if (!MaskCnst) {

4224

// Not an AND with a constant

4225

return SDValue();

4226

}

4227

4228

uint64_t MaskVal = MaskCnst->getZExtValue();

4229

if (MaskVal != 0xff) {

4230

// Not an AND that chops off top 8 bits

4231

return SDValue();

4232

}

4233

4234

MemSDNode *Mem = dyn_cast<MemSDNode>(Val);

4235

if (!Mem) {

4236

// Not a MemSDNode?!?

4237

return SDValue();

4238

}

4239

4240

EVT MemVT = Mem->getMemoryVT();

4241

if (MemVT != MVT::v2i8 && MemVT != MVT::v4i8) {

4242

// We only handle the i8 case

4243

return SDValue();

4244

}

4245

4246

unsigned ExtType =

4247

cast<ConstantSDNode>(Val->getOperand(Val->getNumOperands()-1))->

4248

getZExtValue();

4249

if (ExtType == ISD::SEXTLOAD) {

4250

// If for some reason the load is a sextload, the and is needed to zero

4251

// out the high 8 bits

4252

return SDValue();

4253

}

4254

4255

bool AddTo = false;

4256

if (AExt.getNode() != nullptr) {

4257

// Re-insert the ext as a zext.

4258

Val = DCI.DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),

4259

AExt.getValueType(), Val);

4260

AddTo = true;

4261

}

4262

4263

// If we get here, the AND is unnecessary. Just replace it with the load

4264

DCI.CombineTo(N, Val, AddTo);

4265

}

4266

4267

return SDValue();

4268

}

4269

4270

static SDValue PerformREMCombine(SDNode *N,

4271

TargetLowering::DAGCombinerInfo &DCI,

4272

CodeGenOpt::Level OptLevel) {

4273

assert(N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM)((N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::
UREM) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 4273, __PRETTY_FUNCTION__));

4274

4275

// Don't do anything at less than -O2.

4276

if (OptLevel < CodeGenOpt::Default)

4277

return SDValue();

4278

4279

SelectionDAG &DAG = DCI.DAG;

4280

SDLoc DL(N);

4281

EVT VT = N->getValueType(0);

4282

bool IsSigned = N->getOpcode() == ISD::SREM;

4283

unsigned DivOpc = IsSigned ? ISD::SDIV : ISD::UDIV;

4284

4285

const SDValue &Num = N->getOperand(0);

4286

const SDValue &Den = N->getOperand(1);

4287

4288

for (const SDNode *U : Num->uses()) {

4289

if (U->getOpcode() == DivOpc && U->getOperand(0) == Num &&

4290

U->getOperand(1) == Den) {

4291

// Num % Den -> Num - (Num / Den) * Den

4292

return DAG.getNode(ISD::SUB, DL, VT, Num,

4293

DAG.getNode(ISD::MUL, DL, VT,

4294

DAG.getNode(DivOpc, DL, VT, Num, Den),

4295

Den));

4296

}

4297

}

4298

return SDValue();

4299

}

4300

4301

enum OperandSignedness {

4302

Signed = 0,

4303

Unsigned,

4304

Unknown

4305

};

4306

4307

/// IsMulWideOperandDemotable - Checks if the provided DAG node is an operand

4308

/// that can be demoted to \p OptSize bits without loss of information. The

4309

/// signedness of the operand, if determinable, is placed in \p S.

4310

static bool IsMulWideOperandDemotable(SDValue Op,

4311

unsigned OptSize,

4312

OperandSignedness &S) {

4313

S = Unknown;

4314

4315

if (Op.getOpcode() == ISD::SIGN_EXTEND ||

4316

Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {

4317

EVT OrigVT = Op.getOperand(0).getValueType();

4318

if (OrigVT.getSizeInBits() <= OptSize) {

4319

S = Signed;

4320

return true;

4321

}

4322

} else if (Op.getOpcode() == ISD::ZERO_EXTEND) {

4323

EVT OrigVT = Op.getOperand(0).getValueType();

4324

if (OrigVT.getSizeInBits() <= OptSize) {

4325

S = Unsigned;

4326

return true;

4327

}

4328

}

4329

4330

return false;

4331

}

4332

4333

/// AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can

4334

/// be demoted to \p OptSize bits without loss of information. If the operands

4335

/// contain a constant, it should appear as the RHS operand. The signedness of

4336

/// the operands is placed in \p IsSigned.

4337

static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,

4338

unsigned OptSize,

4339

bool &IsSigned) {

4340

OperandSignedness LHSSign;

4341

4342

// The LHS operand must be a demotable op

4343

if (!IsMulWideOperandDemotable(LHS, OptSize, LHSSign))

4344

return false;

4345

4346

// We should have been able to determine the signedness from the LHS

4347

if (LHSSign == Unknown)

4348

return false;

4349

4350

IsSigned = (LHSSign == Signed);

4351

4352

// The RHS can be a demotable op or a constant

4353

if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(RHS)) {

4354

const APInt &Val = CI->getAPIntValue();

4355

if (LHSSign == Unsigned) {

4356

return Val.isIntN(OptSize);

4357

} else {

4358

return Val.isSignedIntN(OptSize);

4359

}

4360

} else {

4361

OperandSignedness RHSSign;

4362

if (!IsMulWideOperandDemotable(RHS, OptSize, RHSSign))

4363

return false;

4364

4365

return LHSSign == RHSSign;

4366

}

4367

}

4368

4369

/// TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply

4370

/// of M/2 bits that produces an M-bit result (i.e. mul.wide). This transform

4371

/// works on both multiply DAG nodes and SHL DAG nodes with a constant shift

4372

/// amount.

4373

static SDValue TryMULWIDECombine(SDNode *N,

4374

TargetLowering::DAGCombinerInfo &DCI) {

4375

EVT MulType = N->getValueType(0);

4376

if (MulType != MVT::i32 && MulType != MVT::i64) {

4377

return SDValue();

4378

}

4379

4380

SDLoc DL(N);

4381

unsigned OptSize = MulType.getSizeInBits() >> 1;

4382

SDValue LHS = N->getOperand(0);

4383

SDValue RHS = N->getOperand(1);

4384

4385

// Canonicalize the multiply so the constant (if any) is on the right

4386

if (N->getOpcode() == ISD::MUL) {

4387

if (isa<ConstantSDNode>(LHS)) {

4388

std::swap(LHS, RHS);

4389

}

4390

}

4391

4392

// If we have a SHL, determine the actual multiply amount

4393

if (N->getOpcode() == ISD::SHL) {

4394

ConstantSDNode *ShlRHS = dyn_cast<ConstantSDNode>(RHS);

4395

if (!ShlRHS) {

4396

return SDValue();

4397

}

4398

4399

APInt ShiftAmt = ShlRHS->getAPIntValue();

4400

unsigned BitWidth = MulType.getSizeInBits();

4401

if (ShiftAmt.sge(0) && ShiftAmt.slt(BitWidth)) {

4402

APInt MulVal = APInt(BitWidth, 1) << ShiftAmt;

4403

RHS = DCI.DAG.getConstant(MulVal, DL, MulType);

4404

} else {

4405

return SDValue();

4406

}

4407

}

4408

4409

bool Signed;

4410

// Verify that our operands are demotable

4411

if (!AreMulWideOperandsDemotable(LHS, RHS, OptSize, Signed)) {

4412

return SDValue();

4413

}

4414

4415

EVT DemotedVT;

4416

if (MulType == MVT::i32) {

4417

DemotedVT = MVT::i16;

4418

} else {

4419

DemotedVT = MVT::i32;

4420

}

4421

4422

// Truncate the operands to the correct size. Note that these are just for

4423

// type consistency and will (likely) be eliminated in later phases.

4424

SDValue TruncLHS =

4425

DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, LHS);

4426

SDValue TruncRHS =

4427

DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, RHS);

4428

4429

unsigned Opc;

4430

if (Signed) {

4431

Opc = NVPTXISD::MUL_WIDE_SIGNED;

4432

} else {

4433

Opc = NVPTXISD::MUL_WIDE_UNSIGNED;

4434

}

4435

4436

return DCI.DAG.getNode(Opc, DL, MulType, TruncLHS, TruncRHS);

4437

}

4438

4439

/// PerformMULCombine - Runs PTX-specific DAG combine patterns on MUL nodes.

4440

static SDValue PerformMULCombine(SDNode *N,

4441

TargetLowering::DAGCombinerInfo &DCI,

4442

CodeGenOpt::Level OptLevel) {

4443

if (OptLevel > 0) {

4444

// Try mul.wide combining at OptLevel > 0

4445

if (SDValue Ret = TryMULWIDECombine(N, DCI))

4446

return Ret;

4447

}

4448

4449

return SDValue();

4450

}

4451

4452

/// PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes.

4453

static SDValue PerformSHLCombine(SDNode *N,

4454

TargetLowering::DAGCombinerInfo &DCI,

4455

CodeGenOpt::Level OptLevel) {

4456

if (OptLevel > 0) {

4457

// Try mul.wide combining at OptLevel > 0

4458

if (SDValue Ret = TryMULWIDECombine(N, DCI))

4459

return Ret;

4460

}

4461

4462

return SDValue();

4463

}

4464

4465

SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,

4466

DAGCombinerInfo &DCI) const {

4467

CodeGenOpt::Level OptLevel = getTargetMachine().getOptLevel();

4468

switch (N->getOpcode()) {

4469

default: break;

4470

case ISD::ADD:

4471

case ISD::FADD:

4472

return PerformADDCombine(N, DCI, STI, OptLevel);

4473

case ISD::MUL:

4474

return PerformMULCombine(N, DCI, OptLevel);

4475

case ISD::SHL:

4476

return PerformSHLCombine(N, DCI, OptLevel);

4477

case ISD::AND:

4478

return PerformANDCombine(N, DCI);

4479

case ISD::UREM:

4480

case ISD::SREM:

4481

return PerformREMCombine(N, DCI, OptLevel);

4482

}

4483

return SDValue();

4484

}

4485

4486

/// ReplaceVectorLoad - Convert vector loads into multi-output scalar loads.

4487

static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG,

4488

SmallVectorImpl<SDValue> &Results) {

4489

EVT ResVT = N->getValueType(0);

4490

SDLoc DL(N);

4491

4492

assert(ResVT.isVector() && "Vector load must have vector type")((ResVT.isVector() && "Vector load must have vector type"
) ? static_cast<void> (0) : __assert_fail ("ResVT.isVector() && \"Vector load must have vector type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 4492, __PRETTY_FUNCTION__));

4493

4494

// We only handle "native" vector sizes for now, e.g. <4 x double> is not

4495

// legal. We can (and should) split that into 2 loads of <2 x double> here

4496

// but I'm leaving that as a TODO for now.

4497

assert(ResVT.isSimple() && "Can only handle simple types")((ResVT.isSimple() && "Can only handle simple types")
? static_cast<void> (0) : __assert_fail ("ResVT.isSimple() && \"Can only handle simple types\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 4497, __PRETTY_FUNCTION__));

4498

switch (ResVT.getSimpleVT().SimpleTy) {

4499

default:

4500

return;

4501

case MVT::v2i8:

4502

case MVT::v2i16:

4503

case MVT::v2i32:

4504

case MVT::v2i64:

4505

case MVT::v2f32:

4506

case MVT::v2f64:

4507

case MVT::v4i8:

4508

case MVT::v4i16:

4509

case MVT::v4i32:

4510

case MVT::v4f32:

4511

// This is a "native" vector type

4512

break;

4513

}

4514

4515

LoadSDNode *LD = cast<LoadSDNode>(N);

4516

4517

unsigned Align = LD->getAlignment();

4518

auto &TD = DAG.getDataLayout();

4519

unsigned PrefAlign =

4520

TD.getPrefTypeAlignment(ResVT.getTypeForEVT(*DAG.getContext()));

4521

if (Align < PrefAlign) {

4522

// This load is not sufficiently aligned, so bail out and let this vector

4523

// load be scalarized. Note that we may still be able to emit smaller

4524

// vector loads. For example, if we are loading a <4 x float> with an

4525

// alignment of 8, this check will fail but the legalizer will try again

4526

// with 2 x <2 x float>, which will succeed with an alignment of 8.

4527

return;

4528

}

4529

4530

EVT EltVT = ResVT.getVectorElementType();

4531

unsigned NumElts = ResVT.getVectorNumElements();

4532

4533

// Since LoadV2 is a target node, we cannot rely on DAG type legalization.

4534

// Therefore, we must ensure the type is legal. For i1 and i8, we set the

4535

// loaded type to i16 and propagate the "real" type as the memory type.

4536

bool NeedTrunc = false;

4537

if (EltVT.getSizeInBits() < 16) {

4538

EltVT = MVT::i16;

4539

NeedTrunc = true;

4540

}

4541

4542

unsigned Opcode = 0;

4543

SDVTList LdResVTs;

4544

4545

switch (NumElts) {

4546

default:

4547

return;

4548

case 2:

4549

Opcode = NVPTXISD::LoadV2;

4550

LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);

4551

break;

4552

case 4: {

4553

Opcode = NVPTXISD::LoadV4;

4554

EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };

4555

LdResVTs = DAG.getVTList(ListVTs);

4556

break;

4557

}

4558

}

4559

4560

// Copy regular operands

4561

SmallVector<SDValue, 8> OtherOps(N->op_begin(), N->op_end());

4562

4563

// The select routine does not have access to the LoadSDNode instance, so

4564

// pass along the extension information

4565

OtherOps.push_back(DAG.getIntPtrConstant(LD->getExtensionType(), DL));

4566

4567

SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,

4568

LD->getMemoryVT(),

4569

LD->getMemOperand());

4570

4571

SmallVector<SDValue, 4> ScalarRes;

4572

4573

for (unsigned i = 0; i < NumElts; ++i) {

4574

SDValue Res = NewLD.getValue(i);

4575

if (NeedTrunc)

4576

Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);

4577

ScalarRes.push_back(Res);

4578

}

4579

4580

SDValue LoadChain = NewLD.getValue(NumElts);

4581

4582

SDValue BuildVec = DAG.getBuildVector(ResVT, DL, ScalarRes);

4583

4584

Results.push_back(BuildVec);

4585

Results.push_back(LoadChain);

4586

}

4587

4588

static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,

4589

SmallVectorImpl<SDValue> &Results) {

4590

SDValue Chain = N->getOperand(0);

4591

SDValue Intrin = N->getOperand(1);

4592

SDLoc DL(N);

4593

4594

// Get the intrinsic ID

4595

unsigned IntrinNo = cast<ConstantSDNode>(Intrin.getNode())->getZExtValue();

4596

switch (IntrinNo) {

4597

default:

4598

return;

4599

case Intrinsic::nvvm_ldg_global_i:

4600

case Intrinsic::nvvm_ldg_global_f:

4601

case Intrinsic::nvvm_ldg_global_p:

4602

case Intrinsic::nvvm_ldu_global_i:

4603

case Intrinsic::nvvm_ldu_global_f:

4604

case Intrinsic::nvvm_ldu_global_p: {

4605

EVT ResVT = N->getValueType(0);

4606

4607

if (ResVT.isVector()) {

4608

// Vector LDG/LDU

4609

4610

unsigned NumElts = ResVT.getVectorNumElements();

4611

EVT EltVT = ResVT.getVectorElementType();

4612

4613

// Since LDU/LDG are target nodes, we cannot rely on DAG type

4614

// legalization.

4615

// Therefore, we must ensure the type is legal. For i1 and i8, we set the

4616

// loaded type to i16 and propagate the "real" type as the memory type.

4617

bool NeedTrunc = false;

4618

if (EltVT.getSizeInBits() < 16) {

4619

EltVT = MVT::i16;

4620

NeedTrunc = true;

4621

}

4622

4623

unsigned Opcode = 0;

4624

SDVTList LdResVTs;

4625

4626

switch (NumElts) {

4627

default:

4628

return;

4629

case 2:

4630

switch (IntrinNo) {

4631

default:

4632

return;

4633

case Intrinsic::nvvm_ldg_global_i:

4634

case Intrinsic::nvvm_ldg_global_f:

4635

case Intrinsic::nvvm_ldg_global_p:

4636

Opcode = NVPTXISD::LDGV2;

4637

break;

4638

case Intrinsic::nvvm_ldu_global_i:

4639

case Intrinsic::nvvm_ldu_global_f:

4640

case Intrinsic::nvvm_ldu_global_p:

4641

Opcode = NVPTXISD::LDUV2;

4642

break;

4643

}

4644

LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);

4645

break;

4646

case 4: {

4647

switch (IntrinNo) {

4648

default:

4649

return;

4650

case Intrinsic::nvvm_ldg_global_i:

4651

case Intrinsic::nvvm_ldg_global_f:

4652

case Intrinsic::nvvm_ldg_global_p:

4653

Opcode = NVPTXISD::LDGV4;

4654

break;

4655

case Intrinsic::nvvm_ldu_global_i:

4656

case Intrinsic::nvvm_ldu_global_f:

4657

case Intrinsic::nvvm_ldu_global_p:

4658

Opcode = NVPTXISD::LDUV4;

4659

break;

4660

}

4661

EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };

4662

LdResVTs = DAG.getVTList(ListVTs);

4663

break;

4664

}

4665

}

4666

4667

SmallVector<SDValue, 8> OtherOps;

4668

4669

// Copy regular operands

4670

4671

OtherOps.push_back(Chain); // Chain

4672

// Skip operand 1 (intrinsic ID)

4673

// Others

4674

OtherOps.append(N->op_begin() + 2, N->op_end());

4675

4676

MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);

4677

4678

SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,

4679

MemSD->getMemoryVT(),

4680

MemSD->getMemOperand());

4681

4682

SmallVector<SDValue, 4> ScalarRes;

4683

4684

for (unsigned i = 0; i < NumElts; ++i) {

4685

SDValue Res = NewLD.getValue(i);

4686

if (NeedTrunc)

4687

Res =

4688

DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);

4689

ScalarRes.push_back(Res);

4690

}

4691

4692

SDValue LoadChain = NewLD.getValue(NumElts);

4693

4694

SDValue BuildVec =

4695

DAG.getBuildVector(ResVT, DL, ScalarRes);

4696

4697

Results.push_back(BuildVec);

4698

Results.push_back(LoadChain);

4699

} else {

4700

// i8 LDG/LDU

4701

assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 &&((ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy ==
MVT::i8 && "Custom handling of non-i8 ldu/ldg?") ? static_cast
<void> (0) : __assert_fail ("ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 && \"Custom handling of non-i8 ldu/ldg?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 4702, __PRETTY_FUNCTION__))

4702

"Custom handling of non-i8 ldu/ldg?")((ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy ==
MVT::i8 && "Custom handling of non-i8 ldu/ldg?") ? static_cast
<void> (0) : __assert_fail ("ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 && \"Custom handling of non-i8 ldu/ldg?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn295712/lib/Target/NVPTX/NVPTXISelLowering.cpp"
, 4702, __PRETTY_FUNCTION__));

4703

4704

// Just copy all operands as-is

4705

SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end());

4706

4707

// Force output to i16

4708

SDVTList LdResVTs = DAG.getVTList(MVT::i16, MVT::Other);

4709

4710

MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);

4711

4712

// We make sure the memory type is i8, which will be used during isel

4713

// to select the proper instruction.

4714

SDValue NewLD =

4715

DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, LdResVTs, Ops,

4716

MVT::i8, MemSD->getMemOperand());

4717

4718

Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,

4719

NewLD.getValue(0)));

4720

Results.push_back(NewLD.getValue(1));

4721

}

4722

}

4723

}

4724

}

4725

4726

void NVPTXTargetLowering::ReplaceNodeResults(

4727

SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {

4728

switch (N->getOpcode()) {

4729

default:

4730

report_fatal_error("Unhandled custom legalization");

4731

case ISD::LOAD:

4732

ReplaceLoadVector(N, DAG, Results);

4733

return;

4734

case ISD::INTRINSIC_W_CHAIN:

4735

ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);

4736

return;

4737

}

4738

}

4739

4740

// Pin NVPTXSection's and NVPTXTargetObjectFile's vtables to this file.

4741

void NVPTXSection::anchor() {}

4742

4743

NVPTXTargetObjectFile::~NVPTXTargetObjectFile() {

4744

delete static_cast<NVPTXSection *>(TextSection);

4745

delete static_cast<NVPTXSection *>(DataSection);

4746

delete static_cast<NVPTXSection *>(BSSSection);

4747

delete static_cast<NVPTXSection *>(ReadOnlySection);

4748

4749

delete static_cast<NVPTXSection *>(StaticCtorSection);

4750

delete static_cast<NVPTXSection *>(StaticDtorSection);

4751

delete static_cast<NVPTXSection *>(LSDASection);

4752

delete static_cast<NVPTXSection *>(EHFrameSection);

4753

delete static_cast<NVPTXSection *>(DwarfAbbrevSection);

4754

delete static_cast<NVPTXSection *>(DwarfInfoSection);

4755

delete static_cast<NVPTXSection *>(DwarfLineSection);

4756

delete static_cast<NVPTXSection *>(DwarfFrameSection);

4757

delete static_cast<NVPTXSection *>(DwarfPubTypesSection);

4758

delete static_cast<const NVPTXSection *>(DwarfDebugInlineSection);

4759

delete static_cast<NVPTXSection *>(DwarfStrSection);

4760

delete static_cast<NVPTXSection *>(DwarfLocSection);

4761

delete static_cast<NVPTXSection *>(DwarfARangesSection);

4762

delete static_cast<NVPTXSection *>(DwarfRangesSection);

4763

delete static_cast<NVPTXSection *>(DwarfMacinfoSection);

4764

}

4765

4766

MCSection *NVPTXTargetObjectFile::SelectSectionForGlobal(

4767

const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const {

4768

return getDataSection();

4769

}