/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

Bug Summary

File:	lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
Warning:	line 1506, column 9 Called C++ object pointer is null

Annotated Source Code

//===-- SelectionDAGISel.cpp - Implement the SelectionDAGISel class -------===//

// The LLVM Compiler Infrastructure

// This file is distributed under the University of Illinois Open Source

// License. See LICENSE.TXT for details.

//===----------------------------------------------------------------------===//

// This implements the SelectionDAGISel class.

//===----------------------------------------------------------------------===//

#include "ScheduleDAGSDNodes.h"

#include "SelectionDAGBuilder.h"

#include "llvm/ADT/APInt.h"

#include "llvm/ADT/DenseMap.h"

#include "llvm/ADT/None.h"

#include "llvm/ADT/PostOrderIterator.h"

#include "llvm/ADT/SmallPtrSet.h"

#include "llvm/ADT/SmallSet.h"

#include "llvm/ADT/SmallVector.h"

#include "llvm/ADT/Statistic.h"

#include "llvm/ADT/STLExtras.h"

#include "llvm/ADT/StringRef.h"

#include "llvm/Analysis/AliasAnalysis.h"

#include "llvm/Analysis/BranchProbabilityInfo.h"

#include "llvm/Analysis/CFG.h"

#include "llvm/Analysis/OptimizationDiagnosticInfo.h"

#include "llvm/Analysis/TargetLibraryInfo.h"

#include "llvm/CodeGen/FastISel.h"

#include "llvm/CodeGen/FunctionLoweringInfo.h"

#include "llvm/CodeGen/GCMetadata.h"

#include "llvm/CodeGen/MachineBasicBlock.h"

#include "llvm/CodeGen/MachineFrameInfo.h"

#include "llvm/CodeGen/MachineFunction.h"

#include "llvm/CodeGen/MachineFunctionPass.h"

#include "llvm/CodeGen/MachineInstr.h"

#include "llvm/CodeGen/MachineInstrBuilder.h"

#include "llvm/CodeGen/MachineMemOperand.h"

#include "llvm/CodeGen/MachineOperand.h"

#include "llvm/CodeGen/MachinePassRegistry.h"

#include "llvm/CodeGen/MachineRegisterInfo.h"

#include "llvm/CodeGen/MachineValueType.h"

#include "llvm/CodeGen/SchedulerRegistry.h"

#include "llvm/CodeGen/SelectionDAG.h"

#include "llvm/CodeGen/SelectionDAGISel.h"

#include "llvm/CodeGen/SelectionDAGNodes.h"

#include "llvm/CodeGen/StackProtector.h"

#include "llvm/CodeGen/ValueTypes.h"

#include "llvm/IR/BasicBlock.h"

#include "llvm/IR/Constants.h"

#include "llvm/IR/DebugInfoMetadata.h"

#include "llvm/IR/DebugLoc.h"

#include "llvm/IR/DiagnosticInfo.h"

#include "llvm/IR/Function.h"

#include "llvm/IR/InlineAsm.h"

#include "llvm/IR/InstrTypes.h"

#include "llvm/IR/Instruction.h"

#include "llvm/IR/Instructions.h"

#include "llvm/IR/IntrinsicInst.h"

#include "llvm/IR/Intrinsics.h"

#include "llvm/IR/Metadata.h"

#include "llvm/IR/Type.h"

#include "llvm/IR/User.h"

#include "llvm/MC/MCInstrDesc.h"

#include "llvm/MC/MCRegisterInfo.h"

#include "llvm/Pass.h"

#include "llvm/Support/BranchProbability.h"

#include "llvm/Support/Casting.h"

#include "llvm/Support/CodeGen.h"

#include "llvm/Support/CommandLine.h"

#include "llvm/Support/Compiler.h"

#include "llvm/Support/Debug.h"

#include "llvm/Support/ErrorHandling.h"

#include "llvm/Support/KnownBits.h"

#include "llvm/Support/Timer.h"

#include "llvm/Support/raw_ostream.h"

#include "llvm/Target/TargetInstrInfo.h"

#include "llvm/Target/TargetIntrinsicInfo.h"

#include "llvm/Target/TargetLowering.h"

#include "llvm/Target/TargetMachine.h"

#include "llvm/Target/TargetOptions.h"

#include "llvm/Target/TargetRegisterInfo.h"

#include "llvm/Target/TargetSubtargetInfo.h"

#include "llvm/Transforms/Utils/BasicBlockUtils.h"

#include <algorithm>

#include <cassert>

#include <cstdint>

#include <iterator>

#include <memory>

#include <string>

#include <utility>

#include <vector>

using namespace llvm;

#define DEBUG_TYPE"isel" "isel"

100

STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on")static llvm::Statistic NumFastIselFailures = {"isel", "NumFastIselFailures"
, "Number of instructions fast isel failed on", {0}, false};

101

STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected")static llvm::Statistic NumFastIselSuccess = {"isel", "NumFastIselSuccess"
, "Number of instructions fast isel selected", {0}, false};

102

STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel")static llvm::Statistic NumFastIselBlocks = {"isel", "NumFastIselBlocks"
, "Number of blocks selected entirely by fast isel", {0}, false
};

103

STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG")static llvm::Statistic NumDAGBlocks = {"isel", "NumDAGBlocks"
, "Number of blocks selected using DAG", {0}, false};

104

STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path")static llvm::Statistic NumDAGIselRetries = {"isel", "NumDAGIselRetries"
, "Number of times dag isel has to try another path", {0}, false
};

105

STATISTIC(NumEntryBlocks, "Number of entry blocks encountered")static llvm::Statistic NumEntryBlocks = {"isel", "NumEntryBlocks"
, "Number of entry blocks encountered", {0}, false};

106

STATISTIC(NumFastIselFailLowerArguments,static llvm::Statistic NumFastIselFailLowerArguments = {"isel"
, "NumFastIselFailLowerArguments", "Number of entry blocks where fast isel failed to lower arguments"
, {0}, false}

107

"Number of entry blocks where fast isel failed to lower arguments")static llvm::Statistic NumFastIselFailLowerArguments = {"isel"
, "NumFastIselFailLowerArguments", "Number of entry blocks where fast isel failed to lower arguments"
, {0}, false};

108

109

static cl::opt<int> EnableFastISelAbort(

110

"fast-isel-abort", cl::Hidden,

111

cl::desc("Enable abort calls when \"fast\" instruction selection "

112

"fails to lower an instruction: 0 disable the abort, 1 will "

113

"abort but for args, calls and terminators, 2 will also "

114

"abort for argument lowering, and 3 will never fallback "

115

"to SelectionDAG."));

116

117

static cl::opt<bool> EnableFastISelFallbackReport(

118

"fast-isel-report-on-fallback", cl::Hidden,

119

cl::desc("Emit a diagnostic when \"fast\" instruction selection "

120

"falls back to SelectionDAG."));

121

122

static cl::opt<bool>

123

UseMBPI("use-mbpi",

124

cl::desc("use Machine Branch Probability Info"),

125

cl::init(true), cl::Hidden);

126

127

#ifndef NDEBUG

128

static cl::opt<std::string>

129

FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,

130

cl::desc("Only display the basic block whose name "

131

"matches this for all view-*-dags options"));

132

static cl::opt<bool>

133

ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,

134

cl::desc("Pop up a window to show dags before the first "

135

"dag combine pass"));

136

static cl::opt<bool>

137

ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,

138

cl::desc("Pop up a window to show dags before legalize types"));

139

static cl::opt<bool>

140

ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,

141

cl::desc("Pop up a window to show dags before legalize"));

142

static cl::opt<bool>

143

ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,

144

cl::desc("Pop up a window to show dags before the second "

145

"dag combine pass"));

146

static cl::opt<bool>

147

ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,

148

cl::desc("Pop up a window to show dags before the post legalize types"

149

" dag combine pass"));

150

static cl::opt<bool>

151

ViewISelDAGs("view-isel-dags", cl::Hidden,

152

cl::desc("Pop up a window to show isel dags as they are selected"));

153

static cl::opt<bool>

154

ViewSchedDAGs("view-sched-dags", cl::Hidden,

155

cl::desc("Pop up a window to show sched dags as they are processed"));

156

static cl::opt<bool>

157

ViewSUnitDAGs("view-sunit-dags", cl::Hidden,

158

cl::desc("Pop up a window to show SUnit dags after they are processed"));

159

#else

160

static const bool ViewDAGCombine1 = false,

161

ViewLegalizeTypesDAGs = false, ViewLegalizeDAGs = false,

162

ViewDAGCombine2 = false,

163

ViewDAGCombineLT = false,

164

ViewISelDAGs = false, ViewSchedDAGs = false,

165

ViewSUnitDAGs = false;

166

#endif

167

168

//===---------------------------------------------------------------------===//

169

///

170

/// RegisterScheduler class - Track the registration of instruction schedulers.

171

///

172

//===---------------------------------------------------------------------===//

173

MachinePassRegistry RegisterScheduler::Registry;

174

175

//===---------------------------------------------------------------------===//

176

///

177

/// ISHeuristic command line option for instruction schedulers.

178

///

179

//===---------------------------------------------------------------------===//

180

static cl::opt<RegisterScheduler::FunctionPassCtor, false,

181

RegisterPassParser<RegisterScheduler>>

182

ISHeuristic("pre-RA-sched",

183

cl::init(&createDefaultScheduler), cl::Hidden,

184

cl::desc("Instruction schedulers available (before register"

185

" allocation):"));

186

187

static RegisterScheduler

188

defaultListDAGScheduler("default", "Best scheduler for the target",

189

createDefaultScheduler);

190

191

namespace llvm {

192

193

//===--------------------------------------------------------------------===//

194

/// \brief This class is used by SelectionDAGISel to temporarily override

195

/// the optimization level on a per-function basis.

196

class OptLevelChanger {

197

SelectionDAGISel &IS;

198

CodeGenOpt::Level SavedOptLevel;

199

bool SavedFastISel;

200

201

public:

202

OptLevelChanger(SelectionDAGISel &ISel,

203

CodeGenOpt::Level NewOptLevel) : IS(ISel) {

204

SavedOptLevel = IS.OptLevel;

205

if (NewOptLevel == SavedOptLevel)

206

return;

207

IS.OptLevel = NewOptLevel;

208

IS.TM.setOptLevel(NewOptLevel);

209

DEBUG(dbgs() << "\nChanging optimization level for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nChanging optimization level for Function "
<< IS.MF->getFunction()->getName() << "\n"
; } } while (false)

210

<< IS.MF->getFunction()->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nChanging optimization level for Function "
<< IS.MF->getFunction()->getName() << "\n"
; } } while (false);

211

DEBUG(dbgs() << "\tBefore: -O" << SavedOptLeveldo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << SavedOptLevel
<< " ; After: -O" << NewOptLevel << "\n"; }
} while (false)

212

<< " ; After: -O" << NewOptLevel << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << SavedOptLevel
<< " ; After: -O" << NewOptLevel << "\n"; }
} while (false);

213

SavedFastISel = IS.TM.Options.EnableFastISel;

214

if (NewOptLevel == CodeGenOpt::None) {

215

IS.TM.setFastISel(IS.TM.getO0WantsFastISel());

216

DEBUG(dbgs() << "\tFastISel is "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false)

217

<< (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false)

218

<< "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tFastISel is " << (IS.TM.
Options.EnableFastISel ? "enabled" : "disabled") << "\n"
; } } while (false);

219

}

220

}

221

222

~OptLevelChanger() {

223

if (IS.OptLevel == SavedOptLevel)

224

return;

225

DEBUG(dbgs() << "\nRestoring optimization level for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nRestoring optimization level for Function "
<< IS.MF->getFunction()->getName() << "\n"
; } } while (false)

226

<< IS.MF->getFunction()->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\nRestoring optimization level for Function "
<< IS.MF->getFunction()->getName() << "\n"
; } } while (false);

227

DEBUG(dbgs() << "\tBefore: -O" << IS.OptLeveldo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << IS.OptLevel
<< " ; After: -O" << SavedOptLevel << "\n"
; } } while (false)

228

<< " ; After: -O" << SavedOptLevel << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\tBefore: -O" << IS.OptLevel
<< " ; After: -O" << SavedOptLevel << "\n"
; } } while (false);

229

IS.OptLevel = SavedOptLevel;

230

IS.TM.setOptLevel(SavedOptLevel);

231

IS.TM.setFastISel(SavedFastISel);

232

}

233

};

234

235

//===--------------------------------------------------------------------===//

236

/// createDefaultScheduler - This creates an instruction scheduler appropriate

237

/// for the target.

238

ScheduleDAGSDNodes* createDefaultScheduler(SelectionDAGISel *IS,

239

CodeGenOpt::Level OptLevel) {

240

const TargetLowering *TLI = IS->TLI;

241

const TargetSubtargetInfo &ST = IS->MF->getSubtarget();

242

243

// Try first to see if the Target has its own way of selecting a scheduler

244

if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {

245

return SchedulerCtor(IS, OptLevel);

246

}

247

248

if (OptLevel == CodeGenOpt::None ||

249

(ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||

250

TLI->getSchedulingPreference() == Sched::Source)

251

return createSourceListDAGScheduler(IS, OptLevel);

252

if (TLI->getSchedulingPreference() == Sched::RegPressure)

253

return createBURRListDAGScheduler(IS, OptLevel);

254

if (TLI->getSchedulingPreference() == Sched::Hybrid)

255

return createHybridListDAGScheduler(IS, OptLevel);

256

if (TLI->getSchedulingPreference() == Sched::VLIW)

257

return createVLIWDAGScheduler(IS, OptLevel);

258

assert(TLI->getSchedulingPreference() == Sched::ILP &&((TLI->getSchedulingPreference() == Sched::ILP && "Unknown sched type!"
) ? static_cast<void> (0) : __assert_fail ("TLI->getSchedulingPreference() == Sched::ILP && \"Unknown sched type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 259, __PRETTY_FUNCTION__))

259

"Unknown sched type!")((TLI->getSchedulingPreference() == Sched::ILP && "Unknown sched type!"
) ? static_cast<void> (0) : __assert_fail ("TLI->getSchedulingPreference() == Sched::ILP && \"Unknown sched type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 259, __PRETTY_FUNCTION__));

260

return createILPListDAGScheduler(IS, OptLevel);

261

}

262

263

} // end namespace llvm

264

265

// EmitInstrWithCustomInserter - This method should be implemented by targets

266

// that mark instructions with the 'usesCustomInserter' flag. These

267

// instructions are special in various ways, which require special support to

268

// insert. The specified MachineInstr is created but not inserted into any

269

// basic blocks, and this method is called to expand it into a sequence of

270

// instructions, potentially also creating new basic blocks and control flow.

271

// When new basic blocks are inserted and the edges from MBB to its successors

272

// are modified, the method should insert pairs of <OldSucc, NewSucc> into the

273

// DenseMap.

274

MachineBasicBlock *

275

TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,

276

MachineBasicBlock *MBB) const {

277

#ifndef NDEBUG

278

dbgs() << "If a target marks an instruction with "

279

"'usesCustomInserter', it must implement "

280

"TargetLowering::EmitInstrWithCustomInserter!";

281

#endif

282

llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 282);

283

}

284

285

void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,

286

SDNode *Node) const {

287

assert(!MI.hasPostISelHook() &&((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', "
"it must implement TargetLowering::AdjustInstrPostInstrSelection!"
) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 289, __PRETTY_FUNCTION__))

288

"If a target marks an instruction with 'hasPostISelHook', "((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', "
"it must implement TargetLowering::AdjustInstrPostInstrSelection!"
) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 289, __PRETTY_FUNCTION__))

289

"it must implement TargetLowering::AdjustInstrPostInstrSelection!")((!MI.hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', "
"it must implement TargetLowering::AdjustInstrPostInstrSelection!"
) ? static_cast<void> (0) : __assert_fail ("!MI.hasPostISelHook() && \"If a target marks an instruction with 'hasPostISelHook', \" \"it must implement TargetLowering::AdjustInstrPostInstrSelection!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 289, __PRETTY_FUNCTION__));

290

}

291

292

//===----------------------------------------------------------------------===//

293

// SelectionDAGISel code

294

//===----------------------------------------------------------------------===//

295

296

SelectionDAGISel::SelectionDAGISel(TargetMachine &tm,

297

CodeGenOpt::Level OL) :

298

MachineFunctionPass(ID), TM(tm),

299

FuncInfo(new FunctionLoweringInfo()),

300

CurDAG(new SelectionDAG(tm, OL)),

301

SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)),

302

GFI(),

303

OptLevel(OL),

304

DAGSize(0) {

305

initializeGCModuleInfoPass(*PassRegistry::getPassRegistry());

306

initializeBranchProbabilityInfoWrapperPassPass(

307

*PassRegistry::getPassRegistry());

308

initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());

309

initializeTargetLibraryInfoWrapperPassPass(

310

*PassRegistry::getPassRegistry());

311

}

312

313

SelectionDAGISel::~SelectionDAGISel() {

314

delete SDB;

315

delete CurDAG;

316

delete FuncInfo;

317

}

318

319

void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {

320

AU.addRequired<AAResultsWrapperPass>();

321

AU.addRequired<GCModuleInfo>();

322

AU.addRequired<StackProtector>();

323

AU.addPreserved<StackProtector>();

324

AU.addPreserved<GCModuleInfo>();

325

AU.addRequired<TargetLibraryInfoWrapperPass>();

326

if (UseMBPI && OptLevel != CodeGenOpt::None)

327

AU.addRequired<BranchProbabilityInfoWrapperPass>();

328

MachineFunctionPass::getAnalysisUsage(AU);

329

}

330

331

/// SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that

332

/// may trap on it. In this case we have to split the edge so that the path

333

/// through the predecessor block that doesn't go to the phi block doesn't

334

/// execute the possibly trapping instruction.

335

///

336

/// This is required for correctness, so it must be done at -O0.

337

///

338

static void SplitCriticalSideEffectEdges(Function &Fn) {

339

// Loop for blocks with phi nodes.

340

for (BasicBlock &BB : Fn) {

341

PHINode *PN = dyn_cast<PHINode>(BB.begin());

342

if (!PN) continue;

343

344

ReprocessBlock:

345

// For each block with a PHI node, check to see if any of the input values

346

// are potentially trapping constant expressions. Constant expressions are

347

// the only potentially trapping value that can occur as the argument to a

348

// PHI.

349

for (BasicBlock::iterator I = BB.begin(); (PN = dyn_cast<PHINode>(I)); ++I)

350

for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {

351

ConstantExpr *CE = dyn_cast<ConstantExpr>(PN->getIncomingValue(i));

352

if (!CE || !CE->canTrap()) continue;

353

354

// The only case we have to worry about is when the edge is critical.

355

// Since this block has a PHI Node, we assume it has multiple input

356

// edges: check to see if the pred has multiple successors.

357

BasicBlock *Pred = PN->getIncomingBlock(i);

358

if (Pred->getTerminator()->getNumSuccessors() == 1)

359

continue;

360

361

// Okay, we have to split this edge.

362

SplitCriticalEdge(

363

Pred->getTerminator(), GetSuccessorNumber(Pred, &BB),

364

CriticalEdgeSplittingOptions().setMergeIdenticalEdges());

365

goto ReprocessBlock;

366

}

367

}

368

}

369

370

bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {

371

// If we already selected that function, we do not need to run SDISel.

372

if (mf.getProperties().hasProperty(

373

MachineFunctionProperties::Property::Selected))

374

return false;

375

// Do some sanity-checking on the command-line options.

376

assert((!EnableFastISelAbort || TM.Options.EnableFastISel) &&(((!EnableFastISelAbort || TM.Options.EnableFastISel) &&
"-fast-isel-abort > 0 requires -fast-isel") ? static_cast
<void> (0) : __assert_fail ("(!EnableFastISelAbort || TM.Options.EnableFastISel) && \"-fast-isel-abort > 0 requires -fast-isel\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 377, __PRETTY_FUNCTION__))

377

"-fast-isel-abort > 0 requires -fast-isel")(((!EnableFastISelAbort || TM.Options.EnableFastISel) &&
"-fast-isel-abort > 0 requires -fast-isel") ? static_cast
<void> (0) : __assert_fail ("(!EnableFastISelAbort || TM.Options.EnableFastISel) && \"-fast-isel-abort > 0 requires -fast-isel\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 377, __PRETTY_FUNCTION__));

378

379

const Function &Fn = *mf.getFunction();

380

MF = &mf;

381

382

// Reset the target options before resetting the optimization

383

// level below.

384

// FIXME: This is a horrible hack and should be processed via

385

// codegen looking at the optimization level explicitly when

386

// it wants to look at it.

387

TM.resetTargetOptions(Fn);

388

// Reset OptLevel to None for optnone functions.

389

CodeGenOpt::Level NewOptLevel = OptLevel;

390

if (OptLevel != CodeGenOpt::None && skipFunction(Fn))

391

NewOptLevel = CodeGenOpt::None;

392

OptLevelChanger OLC(*this, NewOptLevel);

393

394

TII = MF->getSubtarget().getInstrInfo();

395

TLI = MF->getSubtarget().getTargetLowering();

396

RegInfo = &MF->getRegInfo();

397

AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();

398

LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();

399

GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : nullptr;

400

ORE = make_unique<OptimizationRemarkEmitter>(&Fn);

401

402

DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "\n\n\n=== " << Fn.getName(
) << "\n"; } } while (false);

403

404

SplitCriticalSideEffectEdges(const_cast<Function &>(Fn));

405

406

CurDAG->init(*MF, *ORE);

407

FuncInfo->set(Fn, *MF, CurDAG);

408

409

if (UseMBPI && OptLevel != CodeGenOpt::None)

410

FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();

411

else

412

FuncInfo->BPI = nullptr;

413

414

SDB->init(GFI, *AA, LibInfo);

415

416

MF->setHasInlineAsm(false);

417

418

FuncInfo->SplitCSR = false;

419

420

// We split CSR if the target supports it for the given function

421

// and the function has only return exits.

422

if (OptLevel != CodeGenOpt::None && TLI->supportSplitCSR(MF)) {

423

FuncInfo->SplitCSR = true;

424

425

// Collect all the return blocks.

426

for (const BasicBlock &BB : Fn) {

427

if (!succ_empty(&BB))

428

continue;

429

430

const TerminatorInst *Term = BB.getTerminator();

431

if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))

432

continue;

433

434

// Bail out if the exit block is not Return nor Unreachable.

435

FuncInfo->SplitCSR = false;

436

break;

437

}

438

}

439

440

MachineBasicBlock *EntryMBB = &MF->front();

441

if (FuncInfo->SplitCSR)

442

// This performs initialization so lowering for SplitCSR will be correct.

443

TLI->initializeSplitCSR(EntryMBB);

444

445

SelectAllBasicBlocks(Fn);

446

if (FastISelFailed && EnableFastISelFallbackReport) {

447

DiagnosticInfoISelFallback DiagFallback(Fn);

448

Fn.getContext().diagnose(DiagFallback);

449

}

450

451

// If the first basic block in the function has live ins that need to be

452

// copied into vregs, emit the copies into the top of the block before

453

// emitting the code for the block.

454

const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();

455

RegInfo->EmitLiveInCopies(EntryMBB, TRI, *TII);

456

457

// Insert copies in the entry block and the return blocks.

458

if (FuncInfo->SplitCSR) {

459

SmallVector<MachineBasicBlock*, 4> Returns;

460

// Collect all the return blocks.

461

for (MachineBasicBlock &MBB : mf) {

462

if (!MBB.succ_empty())

463

continue;

464

465

MachineBasicBlock::iterator Term = MBB.getFirstTerminator();

466

if (Term != MBB.end() && Term->isReturn()) {

467

Returns.push_back(&MBB);

468

continue;

469

}

470

}

471

TLI->insertCopiesSplitCSR(EntryMBB, Returns);

472

}

473

474

DenseMap<unsigned, unsigned> LiveInMap;

475

if (!FuncInfo->ArgDbgValues.empty())

476

for (MachineRegisterInfo::livein_iterator LI = RegInfo->livein_begin(),

477

E = RegInfo->livein_end(); LI != E; ++LI)

478

if (LI->second)

479

LiveInMap.insert(std::make_pair(LI->first, LI->second));

480

481

// Insert DBG_VALUE instructions for function arguments to the entry block.

482

for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {

483

MachineInstr *MI = FuncInfo->ArgDbgValues[e-i-1];

484

bool hasFI = MI->getOperand(0).isFI();

485

unsigned Reg =

486

hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg();

487

if (TargetRegisterInfo::isPhysicalRegister(Reg))

488

EntryMBB->insert(EntryMBB->begin(), MI);

489

else {

490

MachineInstr *Def = RegInfo->getVRegDef(Reg);

491

if (Def) {

492

MachineBasicBlock::iterator InsertPos = Def;

493

// FIXME: VR def may not be in entry block.

494

Def->getParent()->insert(std::next(InsertPos), MI);

495

} else

496

DEBUG(dbgs() << "Dropping debug info for dead vreg"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for dead vreg"
<< TargetRegisterInfo::virtReg2Index(Reg) << "\n"
; } } while (false)

497

<< TargetRegisterInfo::virtReg2Index(Reg) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for dead vreg"
<< TargetRegisterInfo::virtReg2Index(Reg) << "\n"
; } } while (false);

498

}

499

500

// If Reg is live-in then update debug info to track its copy in a vreg.

501

DenseMap<unsigned, unsigned>::iterator LDI = LiveInMap.find(Reg);

502

if (LDI != LiveInMap.end()) {

503

assert(!hasFI && "There's no handling of frame pointer updating here yet "((!hasFI && "There's no handling of frame pointer updating here yet "
"- add if needed") ? static_cast<void> (0) : __assert_fail
("!hasFI && \"There's no handling of frame pointer updating here yet \" \"- add if needed\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 504, __PRETTY_FUNCTION__))

504

"- add if needed")((!hasFI && "There's no handling of frame pointer updating here yet "
"- add if needed") ? static_cast<void> (0) : __assert_fail
("!hasFI && \"There's no handling of frame pointer updating here yet \" \"- add if needed\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 504, __PRETTY_FUNCTION__));

505

MachineInstr *Def = RegInfo->getVRegDef(LDI->second);

506

MachineBasicBlock::iterator InsertPos = Def;

507

const MDNode *Variable = MI->getDebugVariable();

508

const MDNode *Expr = MI->getDebugExpression();

509

DebugLoc DL = MI->getDebugLoc();

510

bool IsIndirect = MI->isIndirectDebugValue();

511

unsigned Offset = IsIndirect ? MI->getOperand(1).getImm() : 0;

512

assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&((cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? static_cast
<void> (0) : __assert_fail ("cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 513, __PRETTY_FUNCTION__))

513

"Expected inlined-at fields to agree")((cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? static_cast
<void> (0) : __assert_fail ("cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 513, __PRETTY_FUNCTION__));

514

// Def is never a terminator here, so it is ok to increment InsertPos.

515

BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),

516

IsIndirect, LDI->second, Offset, Variable, Expr);

517

518

// If this vreg is directly copied into an exported register then

519

// that COPY instructions also need DBG_VALUE, if it is the only

520

// user of LDI->second.

521

MachineInstr *CopyUseMI = nullptr;

522

for (MachineRegisterInfo::use_instr_iterator

523

UI = RegInfo->use_instr_begin(LDI->second),

524

E = RegInfo->use_instr_end(); UI != E; ) {

525

MachineInstr *UseMI = &*(UI++);

526

if (UseMI->isDebugValue()) continue;

527

if (UseMI->isCopy() && !CopyUseMI && UseMI->getParent() == EntryMBB) {

528

CopyUseMI = UseMI; continue;

529

}

530

// Otherwise this is another use or second copy use.

531

CopyUseMI = nullptr; break;

532

}

533

if (CopyUseMI) {

534

// Use MI's debug location, which describes where Variable was

535

// declared, rather than whatever is attached to CopyUseMI.

536

MachineInstr *NewMI =

537

BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,

538

CopyUseMI->getOperand(0).getReg(), Offset, Variable, Expr);

539

MachineBasicBlock::iterator Pos = CopyUseMI;

540

EntryMBB->insertAfter(Pos, NewMI);

541

}

542

}

543

}

544

545

// Determine if there are any calls in this machine function.

546

MachineFrameInfo &MFI = MF->getFrameInfo();

547

for (const auto &MBB : *MF) {

548

if (MFI.hasCalls() && MF->hasInlineAsm())

549

break;

550

551

for (const auto &MI : MBB) {

552

const MCInstrDesc &MCID = TII->get(MI.getOpcode());

553

if ((MCID.isCall() && !MCID.isReturn()) ||

554

MI.isStackAligningInlineAsm()) {

555

MFI.setHasCalls(true);

556

}

557

if (MI.isInlineAsm()) {

558

MF->setHasInlineAsm(true);

559

}

560

}

561

}

562

563

// Determine if there is a call to setjmp in the machine function.

564

MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice());

565

566

// Replace forward-declared registers with the registers containing

567

// the desired value.

568

MachineRegisterInfo &MRI = MF->getRegInfo();

569

for (DenseMap<unsigned, unsigned>::iterator

570

I = FuncInfo->RegFixups.begin(), E = FuncInfo->RegFixups.end();

571

I != E; ++I) {

572

unsigned From = I->first;

573

unsigned To = I->second;

574

// If To is also scheduled to be replaced, find what its ultimate

575

// replacement is.

576

while (true) {

577

DenseMap<unsigned, unsigned>::iterator J = FuncInfo->RegFixups.find(To);

578

if (J == E) break;

579

To = J->second;

580

}

581

// Make sure the new register has a sufficiently constrained register class.

582

if (TargetRegisterInfo::isVirtualRegister(From) &&

583

TargetRegisterInfo::isVirtualRegister(To))

584

MRI.constrainRegClass(To, MRI.getRegClass(From));

585

// Replace it.

586

587

588

// Replacing one register with another won't touch the kill flags.

589

// We need to conservatively clear the kill flags as a kill on the old

590

// register might dominate existing uses of the new register.

591

if (!MRI.use_empty(To))

592

MRI.clearKillFlags(From);

593

MRI.replaceRegWith(From, To);

594

}

595

596

TLI->finalizeLowering(*MF);

597

598

// Release function-specific state. SDB and CurDAG are already cleared

599

// at this point.

600

FuncInfo->clear();

601

602

DEBUG(dbgs() << "*** MachineFunction at end of ISel ***\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "*** MachineFunction at end of ISel ***\n"
; } } while (false);

603

DEBUG(MF->print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { MF->print(dbgs()); } } while (false);

604

605

return true;

606

}

607

608

static void reportFastISelFailure(MachineFunction &MF,

609

OptimizationRemarkEmitter &ORE,

610

OptimizationRemarkMissed &R,

611

bool ShouldAbort) {

612

// Print the function name explicitly if we don't have a debug location (which

613

// makes the diagnostic less useful) or if we're going to emit a raw error.

614

if (!R.getLocation().isValid() || ShouldAbort)

615

R << (" (in function: " + MF.getName() + ")").str();

616

617

if (ShouldAbort)

618

report_fatal_error(R.getMsg());

619

620

ORE.emit(R);

621

}

622

623

void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,

624

BasicBlock::const_iterator End,

625

bool &HadTailCall) {

626

// Lower the instructions. If a call is emitted as a tail call, cease emitting

627

// nodes for this block.

628

for (BasicBlock::const_iterator I = Begin; I != End && !SDB->HasTailCall; ++I) {

629

if (!ElidedArgCopyInstrs.count(&*I))

630

SDB->visit(*I);

631

}

632

633

// Make sure the root of the DAG is up-to-date.

634

CurDAG->setRoot(SDB->getControlRoot());

635

HadTailCall = SDB->HasTailCall;

636

SDB->clear();

637

638

// Final step, emit the lowered DAG as machine code.

639

CodeGenAndEmitDAG();

640

}

641

642

void SelectionDAGISel::ComputeLiveOutVRegInfo() {

643

SmallPtrSet<SDNode*, 16> VisitedNodes;

644

SmallVector<SDNode*, 128> Worklist;

645

646

Worklist.push_back(CurDAG->getRoot().getNode());

647

648

KnownBits Known;

649

650

do {

651

SDNode *N = Worklist.pop_back_val();

652

653

// If we've already seen this node, ignore it.

654

if (!VisitedNodes.insert(N).second)

655

continue;

656

657

// Otherwise, add all chain operands to the worklist.

658

for (const SDValue &Op : N->op_values())

659

if (Op.getValueType() == MVT::Other)

660

Worklist.push_back(Op.getNode());

661

662

// If this is a CopyToReg with a vreg dest, process it.

663

if (N->getOpcode() != ISD::CopyToReg)

664

continue;

665

666

unsigned DestReg = cast<RegisterSDNode>(N->getOperand(1))->getReg();

667

if (!TargetRegisterInfo::isVirtualRegister(DestReg))

668

continue;

669

670

// Ignore non-scalar or non-integer values.

671

SDValue Src = N->getOperand(2);

672

EVT SrcVT = Src.getValueType();

673

if (!SrcVT.isInteger() || SrcVT.isVector())

674

continue;

675

676

unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);

677

CurDAG->computeKnownBits(Src, Known);

678

FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);

679

} while (!Worklist.empty());

680

}

681

682

void SelectionDAGISel::CodeGenAndEmitDAG() {

683

StringRef GroupName = "sdag";

684

StringRef GroupDescription = "Instruction Selection and Scheduling";

685

std::string BlockName;

686

int BlockNumber = -1;

687

(void)BlockNumber;

688

bool MatchFilterBB = false; (void)MatchFilterBB;

689

690

// Pre-type legalization allow creation of any node types.

691

CurDAG->NewNodesMustHaveLegalTypes = false;

692

693

#ifndef NDEBUG

694

MatchFilterBB = (FilterDAGBasicBlockName.empty() ||

695

FilterDAGBasicBlockName ==

696

FuncInfo->MBB->getBasicBlock()->getName().str());

697

#endif

698

#ifdef NDEBUG

699

if (ViewDAGCombine1 || ViewLegalizeTypesDAGs || ViewLegalizeDAGs ||

700

ViewDAGCombine2 || ViewDAGCombineLT || ViewISelDAGs || ViewSchedDAGs ||

701

ViewSUnitDAGs)

702

#endif

703

{

704

BlockNumber = FuncInfo->MBB->getNumber();

705

BlockName =

706

(MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();

707

}

708

DEBUG(dbgs() << "Initial selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Initial selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false)

709

<< " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Initial selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false);

710

711

if (ViewDAGCombine1 && MatchFilterBB)

712

CurDAG->viewGraph("dag-combine1 input for " + BlockName);

713

714

// Run the DAG combiner in pre-legalize mode.

715

{

716

NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,

717

GroupDescription, TimePassesIsEnabled);

718

CurDAG->Combine(BeforeLegalizeTypes, *AA, OptLevel);

719

}

720

721

DEBUG(dbgs() << "Optimized lowered selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized lowered selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)

722

<< " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized lowered selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false);

723

724

// Second step, hack on the DAG until it only uses operations and types that

725

// the target supports.

726

if (ViewLegalizeTypesDAGs && MatchFilterBB)

727

CurDAG->viewGraph("legalize-types input for " + BlockName);

728

729

bool Changed;

730

{

731

NamedRegionTimer T("legalize_types", "Type Legalization", GroupName,

732

GroupDescription, TimePassesIsEnabled);

733

Changed = CurDAG->LegalizeTypes();

734

}

735

736

DEBUG(dbgs() << "Type-legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)

737

<< " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false);

738

739

// Only allow creation of legal node types.

740

CurDAG->NewNodesMustHaveLegalTypes = true;

741

742

if (Changed) {

743

if (ViewDAGCombineLT && MatchFilterBB)

744

CurDAG->viewGraph("dag-combine-lt input for " + BlockName);

745

746

// Run the DAG combiner in post-type-legalize mode.

747

{

748

NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",

749

GroupName, GroupDescription, TimePassesIsEnabled);

750

CurDAG->Combine(AfterLegalizeTypes, *AA, OptLevel);

751

}

752

753

DEBUG(dbgs() << "Optimized type-legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)

754

<< " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false);

755

756

}

757

758

{

759

NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName,

760

GroupDescription, TimePassesIsEnabled);

761

Changed = CurDAG->LegalizeVectors();

762

}

763

764

if (Changed) {

765

DEBUG(dbgs() << "Vector-legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)

766

<< " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false);

767

768

{

769

NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName,

770

GroupDescription, TimePassesIsEnabled);

771

CurDAG->LegalizeTypes();

772

}

773

774

DEBUG(dbgs() << "Vector/type-legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector/type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)

775

<< " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Vector/type-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false);

776

777

if (ViewDAGCombineLT && MatchFilterBB)

778

CurDAG->viewGraph("dag-combine-lv input for " + BlockName);

779

780

// Run the DAG combiner in post-type-legalize mode.

781

{

782

NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",

783

GroupName, GroupDescription, TimePassesIsEnabled);

784

CurDAG->Combine(AfterLegalizeVectorOps, *AA, OptLevel);

785

}

786

787

DEBUG(dbgs() << "Optimized vector-legalized selection DAG: BB#"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized vector-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)

788

<< BlockNumber << " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized vector-legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false);

789

}

790

791

if (ViewLegalizeDAGs && MatchFilterBB)

792

CurDAG->viewGraph("legalize input for " + BlockName);

793

794

{

795

NamedRegionTimer T("legalize", "DAG Legalization", GroupName,

796

GroupDescription, TimePassesIsEnabled);

797

CurDAG->Legalize();

798

}

799

800

DEBUG(dbgs() << "Legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Legalized selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false)

801

<< " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Legalized selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false);

802

803

if (ViewDAGCombine2 && MatchFilterBB)

804

CurDAG->viewGraph("dag-combine2 input for " + BlockName);

805

806

// Run the DAG combiner in post-legalize mode.

807

{

808

NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,

809

GroupDescription, TimePassesIsEnabled);

810

CurDAG->Combine(AfterLegalizeDAG, *AA, OptLevel);

811

}

812

813

DEBUG(dbgs() << "Optimized legalized selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false)

814

<< " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Optimized legalized selection DAG: BB#"
<< BlockNumber << " '" << BlockName <<
"'\n"; CurDAG->dump(); } } while (false);

815

816

if (OptLevel != CodeGenOpt::None)

817

ComputeLiveOutVRegInfo();

818

819

if (ViewISelDAGs && MatchFilterBB)

820

CurDAG->viewGraph("isel input for " + BlockName);

821

822

// Third, instruction select all of the operations to machine code, adding the

823

// code to the MachineBasicBlock.

824

{

825

NamedRegionTimer T("isel", "Instruction Selection", GroupName,

826

GroupDescription, TimePassesIsEnabled);

827

DoInstructionSelection();

828

}

829

830

DEBUG(dbgs() << "Selected selection DAG: BB#" << BlockNumberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Selected selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false)

831

<< " '" << BlockName << "'\n"; CurDAG->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Selected selection DAG: BB#" <<
BlockNumber << " '" << BlockName << "'\n";
CurDAG->dump(); } } while (false);

832

833

if (ViewSchedDAGs && MatchFilterBB)

834

CurDAG->viewGraph("scheduler input for " + BlockName);

835

836

// Schedule machine code.

837

ScheduleDAGSDNodes *Scheduler = CreateScheduler();

838

{

839

NamedRegionTimer T("sched", "Instruction Scheduling", GroupName,

840

GroupDescription, TimePassesIsEnabled);

841

Scheduler->Run(CurDAG, FuncInfo->MBB);

842

}

843

844

if (ViewSUnitDAGs && MatchFilterBB)

845

Scheduler->viewGraph();

846

847

// Emit machine code to BB. This can change 'BB' to the last block being

848

// inserted into.

849

MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;

850

{

851

NamedRegionTimer T("emit", "Instruction Creation", GroupName,

852

GroupDescription, TimePassesIsEnabled);

853

854

// FuncInfo->InsertPt is passed by reference and set to the end of the

855

// scheduled instructions.

856

LastMBB = FuncInfo->MBB = Scheduler->EmitSchedule(FuncInfo->InsertPt);

857

}

858

859

// If the block was split, make sure we update any references that are used to

860

// update PHI nodes later on.

861

if (FirstMBB != LastMBB)

862

SDB->UpdateSplitBlock(FirstMBB, LastMBB);

863

864

// Free the scheduler state.

865

{

866

NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName,

867

GroupDescription, TimePassesIsEnabled);

868

delete Scheduler;

869

}

870

871

// Free the SelectionDAG state, now that we're finished with it.

872

CurDAG->clear();

873

}

874

875

namespace {

876

877

/// ISelUpdater - helper class to handle updates of the instruction selection

878

/// graph.

879

class ISelUpdater : public SelectionDAG::DAGUpdateListener {

880

SelectionDAG::allnodes_iterator &ISelPosition;

881

882

public:

883

ISelUpdater(SelectionDAG &DAG, SelectionDAG::allnodes_iterator &isp)

884

: SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}

885

886

/// NodeDeleted - Handle nodes deleted from the graph. If the node being

887

/// deleted is the current ISelPosition node, update ISelPosition.

888

///

889

void NodeDeleted(SDNode *N, SDNode *E) override {

890

if (ISelPosition == SelectionDAG::allnodes_iterator(N))

891

++ISelPosition;

892

}

893

};

894

895

} // end anonymous namespace

896

897

static bool isStrictFPOp(SDNode *Node, unsigned &NewOpc) {

898

unsigned OrigOpc = Node->getOpcode();

899

switch (OrigOpc) {

900

case ISD::STRICT_FADD: NewOpc = ISD::FADD; return true;

901

case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; return true;

902

case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; return true;

903

case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; return true;

904

case ISD::STRICT_FREM: NewOpc = ISD::FREM; return true;

905

default: return false;

906

}

907

}

908

909

SDNode* SelectionDAGISel::MutateStrictFPToFP(SDNode *Node, unsigned NewOpc) {

910

assert(((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) ||((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))

911

(Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) ||((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))

912

(Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) ||((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))

913

(Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) ||((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))

914

(Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__))

915

"Unexpected StrictFP opcode!")((((Node->getOpcode() == ISD::STRICT_FADD && NewOpc
== ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB &&
NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL
&& NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD
::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode
() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) &&
"Unexpected StrictFP opcode!") ? static_cast<void> (0)
: __assert_fail ("((Node->getOpcode() == ISD::STRICT_FADD && NewOpc == ISD::FADD) || (Node->getOpcode() == ISD::STRICT_FSUB && NewOpc == ISD::FSUB) || (Node->getOpcode() == ISD::STRICT_FMUL && NewOpc == ISD::FMUL) || (Node->getOpcode() == ISD::STRICT_FDIV && NewOpc == ISD::FDIV) || (Node->getOpcode() == ISD::STRICT_FREM && NewOpc == ISD::FREM)) && \"Unexpected StrictFP opcode!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 915, __PRETTY_FUNCTION__));

916

917

// We're taking this node out of the chain, so we need to re-link things.

918

SDValue InputChain = Node->getOperand(0);

919

SDValue OutputChain = SDValue(Node, 1);

920

CurDAG->ReplaceAllUsesOfValueWith(OutputChain, InputChain);

921

922

SDVTList VTs = CurDAG->getVTList(Node->getOperand(1).getValueType());

923

SDValue Ops[2] = { Node->getOperand(1), Node->getOperand(2) };

924

SDNode *Res = CurDAG->MorphNodeTo(Node, NewOpc, VTs, Ops);

925

926

// MorphNodeTo can operate in two ways: if an existing node with the

927

// specified operands exists, it can just return it. Otherwise, it

928

// updates the node in place to have the requested operands.

929

if (Res == Node) {

930

// If we updated the node in place, reset the node ID. To the isel,

931

// this should be just like a newly allocated machine node.

932

Res->setNodeId(-1);

933

} else {

934

CurDAG->ReplaceAllUsesWith(Node, Res);

935

CurDAG->RemoveDeadNode(Node);

936

}

937

938

return Res;

939

}

940

941

void SelectionDAGISel::DoInstructionSelection() {

942

DEBUG(dbgs() << "===== Instruction selection begins: BB#"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection begins: BB#"
<< FuncInfo->MBB->getNumber() << " '" <<
FuncInfo->MBB->getName() << "'\n"; } } while (false
)

943

<< FuncInfo->MBB->getNumber()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection begins: BB#"
<< FuncInfo->MBB->getNumber() << " '" <<
FuncInfo->MBB->getName() << "'\n"; } } while (false
)

944

<< " '" << FuncInfo->MBB->getName() << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection begins: BB#"
<< FuncInfo->MBB->getNumber() << " '" <<
FuncInfo->MBB->getName() << "'\n"; } } while (false
);

945

946

PreprocessISelDAG();

947

948

// Select target instructions for the DAG.

949

{

950

// Number all nodes with a topological order and set DAGSize.

951

DAGSize = CurDAG->AssignTopologicalOrder();

952

953

// Create a dummy node (which is not added to allnodes), that adds

954

// a reference to the root node, preventing it from being deleted,

955

// and tracking any changes of the root.

956

HandleSDNode Dummy(CurDAG->getRoot());

957

SelectionDAG::allnodes_iterator ISelPosition (CurDAG->getRoot().getNode());

958

++ISelPosition;

959

960

// Make sure that ISelPosition gets properly updated when nodes are deleted

961

// in calls made from this function.

962

ISelUpdater ISU(*CurDAG, ISelPosition);

963

964

// The AllNodes list is now topological-sorted. Visit the

965

// nodes by starting at the end of the list (the root of the

966

// graph) and preceding back toward the beginning (the entry

967

// node).

968

while (ISelPosition != CurDAG->allnodes_begin()) {

969

SDNode *Node = &*--ISelPosition;

970

// Skip dead nodes. DAGCombiner is expected to eliminate all dead nodes,

971

// but there are currently some corner cases that it misses. Also, this

972

// makes it theoretically possible to disable the DAGCombiner.

973

if (Node->use_empty())

974

continue;

975

976

// When we are using non-default rounding modes or FP exception behavior

977

// FP operations are represented by StrictFP pseudo-operations. They

978

// need to be simplified here so that the target-specific instruction

979

// selectors know how to handle them.

980

981

// If the current node is a strict FP pseudo-op, the isStrictFPOp()

982

// function will provide the corresponding normal FP opcode to which the

983

// node should be mutated.

984

unsigned NormalFPOpc = ISD::UNDEF;

985

bool IsStrictFPOp = isStrictFPOp(Node, NormalFPOpc);

986

if (IsStrictFPOp)

987

Node = MutateStrictFPToFP(Node, NormalFPOpc);

988

989

Select(Node);

990

991

// FIXME: Add code here to attach an implicit def and use of

992

// target-specific FP environment registers.

993

}

994

995

CurDAG->setRoot(Dummy.getValue());

996

}

997

998

DEBUG(dbgs() << "===== Instruction selection ends:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "===== Instruction selection ends:\n"
; } } while (false);

999

1000

PostprocessISelDAG();

1001

}

1002

1003

static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI) {

1004

for (const User *U : CPI->users()) {

1005

if (const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {

1006

Intrinsic::ID IID = EHPtrCall->getIntrinsicID();

1007

if (IID == Intrinsic::eh_exceptionpointer ||

1008

IID == Intrinsic::eh_exceptioncode)

1009

return true;

1010

}

1011

}

1012

return false;

1013

}

1014

1015

/// PrepareEHLandingPad - Emit an EH_LABEL, set up live-in registers, and

1016

/// do other setup for EH landing-pad blocks.

1017

bool SelectionDAGISel::PrepareEHLandingPad() {

1018

MachineBasicBlock *MBB = FuncInfo->MBB;

1019

const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();

1020

const BasicBlock *LLVMBB = MBB->getBasicBlock();

1021

const TargetRegisterClass *PtrRC =

1022

TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout()));

1023

1024

// Catchpads have one live-in register, which typically holds the exception

1025

// pointer or code.

1026

if (const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->getFirstNonPHI())) {

1027

if (hasExceptionPointerOrCodeUser(CPI)) {

1028

// Get or create the virtual register to hold the pointer or code. Mark

1029

// the live in physreg and copy into the vreg.

1030

MCPhysReg EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);

1031

assert(EHPhysReg && "target lacks exception pointer register")((EHPhysReg && "target lacks exception pointer register"
) ? static_cast<void> (0) : __assert_fail ("EHPhysReg && \"target lacks exception pointer register\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1031, __PRETTY_FUNCTION__));

1032

MBB->addLiveIn(EHPhysReg);

1033

unsigned VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);

1034

BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(),

1035

TII->get(TargetOpcode::COPY), VReg)

1036

.addReg(EHPhysReg, RegState::Kill);

1037

}

1038

return true;

1039

}

1040

1041

if (!LLVMBB->isLandingPad())

1042

return true;

1043

1044

// Add a label to mark the beginning of the landing pad. Deletion of the

1045

// landing pad can thus be detected via the MachineModuleInfo.

1046

MCSymbol *Label = MF->addLandingPad(MBB);

1047

1048

// Assign the call site to the landing pad's begin label.

1049

MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);

1050

1051

const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL);

1052

BuildMI(*MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II)

1053

.addSym(Label);

1054

1055

// Mark exception register as live in.

1056

if (unsigned Reg = TLI->getExceptionPointerRegister(PersonalityFn))

1057

FuncInfo->ExceptionPointerVirtReg = MBB->addLiveIn(Reg, PtrRC);

1058

1059

// Mark exception selector register as live in.

1060

if (unsigned Reg = TLI->getExceptionSelectorRegister(PersonalityFn))

1061

FuncInfo->ExceptionSelectorVirtReg = MBB->addLiveIn(Reg, PtrRC);

1062

1063

return true;

1064

}

1065

1066

/// isFoldedOrDeadInstruction - Return true if the specified instruction is

1067

/// side-effect free and is either dead or folded into a generated instruction.

1068

/// Return false if it needs to be emitted.

1069

static bool isFoldedOrDeadInstruction(const Instruction *I,

1070

FunctionLoweringInfo *FuncInfo) {

1071

return !I->mayWriteToMemory() && // Side-effecting instructions aren't folded.

1072

!isa<TerminatorInst>(I) && // Terminators aren't folded.

1073

!isa<DbgInfoIntrinsic>(I) && // Debug instructions aren't folded.

1074

!I->isEHPad() && // EH pad instructions aren't folded.

1075

!FuncInfo->isExportedInst(I); // Exported instrs must be computed.

1076

}

1077

1078

/// Set up SwiftErrorVals by going through the function. If the function has

1079

/// swifterror argument, it will be the first entry.

1080

static void setupSwiftErrorVals(const Function &Fn, const TargetLowering *TLI,

1081

FunctionLoweringInfo *FuncInfo) {

1082

if (!TLI->supportSwiftError())

1083

return;

1084

1085

FuncInfo->SwiftErrorVals.clear();

1086

FuncInfo->SwiftErrorVRegDefMap.clear();

1087

FuncInfo->SwiftErrorVRegUpwardsUse.clear();

1088

FuncInfo->SwiftErrorArg = nullptr;

1089

1090

// Check if function has a swifterror argument.

1091

bool HaveSeenSwiftErrorArg = false;

1092

for (Function::const_arg_iterator AI = Fn.arg_begin(), AE = Fn.arg_end();

1093

AI != AE; ++AI)

1094

if (AI->hasSwiftErrorAttr()) {

1095

assert(!HaveSeenSwiftErrorArg &&((!HaveSeenSwiftErrorArg && "Must have only one swifterror parameter"
) ? static_cast<void> (0) : __assert_fail ("!HaveSeenSwiftErrorArg && \"Must have only one swifterror parameter\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1096, __PRETTY_FUNCTION__))

1096

"Must have only one swifterror parameter")((!HaveSeenSwiftErrorArg && "Must have only one swifterror parameter"
) ? static_cast<void> (0) : __assert_fail ("!HaveSeenSwiftErrorArg && \"Must have only one swifterror parameter\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1096, __PRETTY_FUNCTION__));

1097

(void)HaveSeenSwiftErrorArg; // silence warning.

1098

HaveSeenSwiftErrorArg = true;

1099

FuncInfo->SwiftErrorArg = &*AI;

1100

FuncInfo->SwiftErrorVals.push_back(&*AI);

1101

}

1102

1103

for (const auto &LLVMBB : Fn)

1104

for (const auto &Inst : LLVMBB) {

1105

if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(&Inst))

1106

if (Alloca->isSwiftError())

1107

FuncInfo->SwiftErrorVals.push_back(Alloca);

1108

}

1109

}

1110

1111

static void createSwiftErrorEntriesInEntryBlock(FunctionLoweringInfo *FuncInfo,

1112

FastISel *FastIS,

1113

const TargetLowering *TLI,

1114

const TargetInstrInfo *TII,

1115

SelectionDAGBuilder *SDB) {

1116

if (!TLI->supportSwiftError())

1117

return;

1118

1119

// We only need to do this when we have swifterror parameter or swifterror

1120

// alloc.

1121

if (FuncInfo->SwiftErrorVals.empty())

1122

return;

1123

1124

assert(FuncInfo->MBB == &*FuncInfo->MF->begin() &&((FuncInfo->MBB == &*FuncInfo->MF->begin() &&
"expected to insert into entry block") ? static_cast<void
> (0) : __assert_fail ("FuncInfo->MBB == &*FuncInfo->MF->begin() && \"expected to insert into entry block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1125, __PRETTY_FUNCTION__))

1125

"expected to insert into entry block")((FuncInfo->MBB == &*FuncInfo->MF->begin() &&
"expected to insert into entry block") ? static_cast<void
> (0) : __assert_fail ("FuncInfo->MBB == &*FuncInfo->MF->begin() && \"expected to insert into entry block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1125, __PRETTY_FUNCTION__));

1126

auto &DL = FuncInfo->MF->getDataLayout();

1127

auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL));

1128

for (const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) {

1129

// We will always generate a copy from the argument. It is always used at

1130

// least by the 'return' of the swifterror.

1131

if (FuncInfo->SwiftErrorArg && FuncInfo->SwiftErrorArg == SwiftErrorVal)

1132

continue;

1133

unsigned VReg = FuncInfo->MF->getRegInfo().createVirtualRegister(RC);

1134

// Assign Undef to Vreg. We construct MI directly to make sure it works

1135

// with FastISel.

1136

BuildMI(*FuncInfo->MBB, FuncInfo->MBB->getFirstNonPHI(),

1137

SDB->getCurDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF),

1138

VReg);

1139

1140

// Keep FastIS informed about the value we just inserted.

1141

if (FastIS)

1142

FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));

1143

1144

FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorVal, VReg);

1145

}

1146

}

1147

1148

/// Propagate swifterror values through the machine function CFG.

1149

static void propagateSwiftErrorVRegs(FunctionLoweringInfo *FuncInfo) {

1150

auto *TLI = FuncInfo->TLI;

1151

if (!TLI->supportSwiftError())

1152

return;

1153

1154

// We only need to do this when we have swifterror parameter or swifterror

1155

// alloc.

1156

if (FuncInfo->SwiftErrorVals.empty())

1157

return;

1158

1159

// For each machine basic block in reverse post order.

1160

ReversePostOrderTraversal<MachineFunction *> RPOT(FuncInfo->MF);

1161

for (ReversePostOrderTraversal<MachineFunction *>::rpo_iterator

1162

It = RPOT.begin(),

1163

E = RPOT.end();

1164

It != E; ++It) {

1165

MachineBasicBlock *MBB = *It;

1166

1167

// For each swifterror value in the function.

1168

for(const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) {

1169

auto Key = std::make_pair(MBB, SwiftErrorVal);

1170

auto UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key);

1171

auto VRegDefIt = FuncInfo->SwiftErrorVRegDefMap.find(Key);

1172

bool UpwardsUse = UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end();

1173

unsigned UUseVReg = UpwardsUse ? UUseIt->second : 0;

1174

bool DownwardDef = VRegDefIt != FuncInfo->SwiftErrorVRegDefMap.end();

1175

assert(!(UpwardsUse && !DownwardDef) &&((!(UpwardsUse && !DownwardDef) && "We can't have an upwards use but no downwards def"
) ? static_cast<void> (0) : __assert_fail ("!(UpwardsUse && !DownwardDef) && \"We can't have an upwards use but no downwards def\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1176, __PRETTY_FUNCTION__))

1176

"We can't have an upwards use but no downwards def")((!(UpwardsUse && !DownwardDef) && "We can't have an upwards use but no downwards def"
) ? static_cast<void> (0) : __assert_fail ("!(UpwardsUse && !DownwardDef) && \"We can't have an upwards use but no downwards def\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1176, __PRETTY_FUNCTION__));

1177

1178

// If there is no upwards exposed use and an entry for the swifterror in

1179

// the def map for this value we don't need to do anything: We already

1180

// have a downward def for this basic block.

1181

if (!UpwardsUse && DownwardDef)

1182

continue;

1183

1184

// Otherwise we either have an upwards exposed use vreg that we need to

1185

// materialize or need to forward the downward def from predecessors.

1186

1187

// Check whether we have a single vreg def from all predecessors.

1188

// Otherwise we need a phi.

1189

SmallVector<std::pair<MachineBasicBlock *, unsigned>, 4> VRegs;

1190

SmallSet<const MachineBasicBlock*, 8> Visited;

1191

for (auto *Pred : MBB->predecessors()) {

1192

if (!Visited.insert(Pred).second)

1193

continue;

1194

VRegs.push_back(std::make_pair(

1195

Pred, FuncInfo->getOrCreateSwiftErrorVReg(Pred, SwiftErrorVal)));

1196

if (Pred != MBB)

1197

continue;

1198

// We have a self-edge.

1199

// If there was no upwards use in this basic block there is now one: the

1200

// phi needs to use it self.

1201

if (!UpwardsUse) {

1202

UpwardsUse = true;

1203

UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key);

1204

assert(UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end())((UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end()) ? static_cast
<void> (0) : __assert_fail ("UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end()"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1204, __PRETTY_FUNCTION__));

1205

UUseVReg = UUseIt->second;

1206

}

1207

}

1208

1209

// We need a phi node if we have more than one predecessor with different

1210

// downward defs.

1211

bool needPHI =

1212

VRegs.size() >= 1 &&

1213

std::find_if(

1214

VRegs.begin(), VRegs.end(),

1215

[&](const std::pair<const MachineBasicBlock *, unsigned> &V)

1216

-> bool { return V.second != VRegs[0].second; }) !=

1217

VRegs.end();

1218

1219

// If there is no upwards exposed used and we don't need a phi just

1220

// forward the swifterror vreg from the predecessor(s).

1221

if (!UpwardsUse && !needPHI) {

1222

assert(!VRegs.empty() &&((!VRegs.empty() && "No predecessors? The entry block should bail out earlier"
) ? static_cast<void> (0) : __assert_fail ("!VRegs.empty() && \"No predecessors? The entry block should bail out earlier\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1223, __PRETTY_FUNCTION__))

1223

"No predecessors? The entry block should bail out earlier")((!VRegs.empty() && "No predecessors? The entry block should bail out earlier"
) ? static_cast<void> (0) : __assert_fail ("!VRegs.empty() && \"No predecessors? The entry block should bail out earlier\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1223, __PRETTY_FUNCTION__));

1224

// Just forward the swifterror vreg from the predecessor(s).

1225

FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, VRegs[0].second);

1226

continue;

1227

}

1228

1229

auto DLoc = isa<Instruction>(SwiftErrorVal)

1230

? dyn_cast<Instruction>(SwiftErrorVal)->getDebugLoc()

1231

: DebugLoc();

1232

const auto *TII = FuncInfo->MF->getSubtarget().getInstrInfo();

1233

1234

// If we don't need a phi create a copy to the upward exposed vreg.

1235

if (!needPHI) {

1236

assert(UpwardsUse)((UpwardsUse) ? static_cast<void> (0) : __assert_fail (
"UpwardsUse", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1236, __PRETTY_FUNCTION__));

1237

unsigned DestReg = UUseVReg;

1238

BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, TII->get(TargetOpcode::COPY),

1239

DestReg)

1240

.addReg(VRegs[0].second);

1241

continue;

1242

}

1243

1244

// We need a phi: if there is an upwards exposed use we already have a

1245

// destination virtual register number otherwise we generate a new one.

1246

auto &DL = FuncInfo->MF->getDataLayout();

1247

auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL));

1248

unsigned PHIVReg =

1249

UpwardsUse ? UUseVReg

1250

: FuncInfo->MF->getRegInfo().createVirtualRegister(RC);

1251

MachineInstrBuilder SwiftErrorPHI =

1252

BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc,

1253

TII->get(TargetOpcode::PHI), PHIVReg);

1254

for (auto BBRegPair : VRegs) {

1255

SwiftErrorPHI.addReg(BBRegPair.second).addMBB(BBRegPair.first);

1256

}

1257

1258

// We did not have a definition in this block before: store the phi's vreg

1259

// as this block downward exposed def.

1260

if (!UpwardsUse)

1261

FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, PHIVReg);

1262

}

1263

}

1264

}

1265

1266

void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {

1267

FastISelFailed = false;

1268

// Initialize the Fast-ISel state, if needed.

1269

FastISel *FastIS = nullptr;

'FastIS' initialized to a null pointer value

→

1270

if (TM.Options.EnableFastISel)

←

Assuming the condition is false

→

←

Taking false branch

→

1271

FastIS = TLI->createFastISel(*FuncInfo, LibInfo);

1272

1273

setupSwiftErrorVals(Fn, TLI, FuncInfo);

1274

1275

ReversePostOrderTraversal<const Function*> RPOT(&Fn);

1276

1277

// Lower arguments up front. An RPO iteration always visits the entry block

1278

// first.

1279

assert(*RPOT.begin() == &Fn.getEntryBlock())((*RPOT.begin() == &Fn.getEntryBlock()) ? static_cast<
void> (0) : __assert_fail ("*RPOT.begin() == &Fn.getEntryBlock()"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1279, __PRETTY_FUNCTION__));

1280

++NumEntryBlocks;

1281

1282

// Set up FuncInfo for ISel. Entry blocks never have PHIs.

1283

FuncInfo->MBB = FuncInfo->MBBMap[&Fn.getEntryBlock()];

1284

FuncInfo->InsertPt = FuncInfo->MBB->begin();

1285

1286

if (!FastIS) {

←

Taking true branch

→

1287

LowerArguments(Fn);

1288

} else {

1289

// See if fast isel can lower the arguments.

1290

FastIS->startNewBlock();

1291

if (!FastIS->lowerArguments()) {

1292

FastISelFailed = true;

1293

// Fast isel failed to lower these arguments

1294

++NumFastIselFailLowerArguments;

1295

1296

OptimizationRemarkMissed R("sdagisel", "FastISelFailure",

1297

Fn.getSubprogram(),

1298

&Fn.getEntryBlock());

1299

R << "FastISel didn't lower all arguments: "

1300

<< ore::NV("Prototype", Fn.getType());

1301

reportFastISelFailure(*MF, *ORE, R, EnableFastISelAbort > 1);

1302

1303

// Use SelectionDAG argument lowering

1304

LowerArguments(Fn);

1305

CurDAG->setRoot(SDB->getControlRoot());

1306

SDB->clear();

1307

CodeGenAndEmitDAG();

1308

}

1309

1310

// If we inserted any instructions at the beginning, make a note of

1311

// where they are, so we can be sure to emit subsequent instructions

1312

// after them.

1313

if (FuncInfo->InsertPt != FuncInfo->MBB->begin())

1314

FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt));

1315

else

1316

FastIS->setLastLocalValue(nullptr);

1317

}

1318

createSwiftErrorEntriesInEntryBlock(FuncInfo, FastIS, TLI, TII, SDB);

1319

1320

// Iterate over all basic blocks in the function.

1321

for (const BasicBlock *LLVMBB : RPOT) {

1322

if (OptLevel != CodeGenOpt::None) {

←

Assuming the condition is false

Taking false branch

Assuming the condition is false

Taking false branch

Assuming the condition is false

Taking false branch

Assuming the condition is false

→

←

Taking false branch

→

1323

bool AllPredsVisited = true;

1324

for (const_pred_iterator PI = pred_begin(LLVMBB), PE = pred_end(LLVMBB);

1325

PI != PE; ++PI) {

1326

if (!FuncInfo->VisitedBBs.count(*PI)) {

1327

AllPredsVisited = false;

1328

break;

1329

}

1330

}

1331

1332

if (AllPredsVisited) {

1333

for (BasicBlock::const_iterator I = LLVMBB->begin();

1334

const PHINode *PN = dyn_cast<PHINode>(I); ++I)

1335

FuncInfo->ComputePHILiveOutRegInfo(PN);

1336

} else {

1337

for (BasicBlock::const_iterator I = LLVMBB->begin();

1338

const PHINode *PN = dyn_cast<PHINode>(I); ++I)

1339

FuncInfo->InvalidatePHILiveOutRegInfo(PN);

1340

}

1341

1342

FuncInfo->VisitedBBs.insert(LLVMBB);

1343

}

1344

1345

BasicBlock::const_iterator const Begin =

1346

LLVMBB->getFirstNonPHI()->getIterator();

1347

BasicBlock::const_iterator const End = LLVMBB->end();

1348

BasicBlock::const_iterator BI = End;

1349

1350

FuncInfo->MBB = FuncInfo->MBBMap[LLVMBB];

1351

if (!FuncInfo->MBB)

←

Assuming the condition is false

Taking false branch

Assuming the condition is false

Taking false branch

Assuming the condition is false

Taking false branch

Assuming the condition is false

→

←

Taking false branch

→

1352

continue; // Some blocks like catchpads have no code or MBB.

1353

1354

// Insert new instructions after any phi or argument setup code.

1355

FuncInfo->InsertPt = FuncInfo->MBB->end();

1356

1357

// Setup an EH landing-pad block.

1358

FuncInfo->ExceptionPointerVirtReg = 0;

1359

FuncInfo->ExceptionSelectorVirtReg = 0;

1360

if (LLVMBB->isEHPad())

←

Assuming the condition is false

Taking false branch

Assuming the condition is false

Taking false branch

Assuming the condition is false

Taking false branch

Assuming the condition is false

→

←

Taking false branch

→

1361

if (!PrepareEHLandingPad())

1362

continue;

1363

1364

// Before doing SelectionDAG ISel, see if FastISel has been requested.

1365

if (FastIS) {

Taking false branch

Taking false branch

Taking false branch

Taking false branch

1366

if (LLVMBB != &Fn.getEntryBlock())

1367

FastIS->startNewBlock();

1368

1369

unsigned NumFastIselRemaining = std::distance(Begin, End);

1370

// Do FastISel on as many instructions as possible.

1371

for (; BI != Begin; --BI) {

1372

const Instruction *Inst = &*std::prev(BI);

1373

1374

// If we no longer require this instruction, skip it.

1375

if (isFoldedOrDeadInstruction(Inst, FuncInfo) ||

1376

ElidedArgCopyInstrs.count(Inst)) {

1377

--NumFastIselRemaining;

1378

continue;

1379

}

1380

1381

// Bottom-up: reset the insert pos at the top, after any local-value

1382

// instructions.

1383

FastIS->recomputeInsertPt();

1384

1385

// Try to select the instruction with FastISel.

1386

if (FastIS->selectInstruction(Inst)) {

1387

FastISelFailed = true;

1388

--NumFastIselRemaining;

1389

++NumFastIselSuccess;

1390

// If fast isel succeeded, skip over all the folded instructions, and

1391

// then see if there is a load right before the selected instructions.

1392

// Try to fold the load if so.

1393

const Instruction *BeforeInst = Inst;

1394

while (BeforeInst != &*Begin) {

1395

BeforeInst = &*std::prev(BasicBlock::const_iterator(BeforeInst));

1396

if (!isFoldedOrDeadInstruction(BeforeInst, FuncInfo))

1397

break;

1398

}

1399

if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&

1400

BeforeInst->hasOneUse() &&

1401

FastIS->tryToFoldLoad(cast<LoadInst>(BeforeInst), Inst)) {

1402

// If we succeeded, don't re-select the load.

1403

BI = std::next(BasicBlock::const_iterator(BeforeInst));

1404

--NumFastIselRemaining;

1405

++NumFastIselSuccess;

1406

}

1407

continue;

1408

}

1409

1410

// Then handle certain instructions as single-LLVM-Instruction blocks.

1411

if (isa<CallInst>(Inst)) {

1412

OptimizationRemarkMissed R("sdagisel", "FastISelFailure",

1413

Inst->getDebugLoc(), LLVMBB);

1414

1415

R << "FastISel missed call";

1416

1417

if (R.isEnabled() || EnableFastISelAbort) {

1418

std::string InstStrStorage;

1419

raw_string_ostream InstStr(InstStrStorage);

1420

InstStr << *Inst;

1421

1422

R << ": " << InstStr.str();

1423

}

1424

1425

reportFastISelFailure(*MF, *ORE, R, EnableFastISelAbort > 2);

1426

1427

if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&

1428

!Inst->use_empty()) {

1429

unsigned &R = FuncInfo->ValueMap[Inst];

1430

if (!R)

1431

R = FuncInfo->CreateRegs(Inst->getType());

1432

}

1433

1434

bool HadTailCall = false;

1435

MachineBasicBlock::iterator SavedInsertPt = FuncInfo->InsertPt;

1436

SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);

1437

1438

// If the call was emitted as a tail call, we're done with the block.

1439

// We also need to delete any previously emitted instructions.

1440

if (HadTailCall) {

1441

FastIS->removeDeadCode(SavedInsertPt, FuncInfo->MBB->end());

1442

--BI;

1443

break;

1444

}

1445

1446

// Recompute NumFastIselRemaining as Selection DAG instruction

1447

// selection may have handled the call, input args, etc.

1448

unsigned RemainingNow = std::distance(Begin, BI);

1449

NumFastIselFailures += NumFastIselRemaining - RemainingNow;

1450

NumFastIselRemaining = RemainingNow;

1451

continue;

1452

}

1453

1454

OptimizationRemarkMissed R("sdagisel", "FastISelFailure",

1455

Inst->getDebugLoc(), LLVMBB);

1456

1457

bool ShouldAbort = EnableFastISelAbort;

1458

if (isa<TerminatorInst>(Inst)) {

1459

// Use a different message for terminator misses.

1460

R << "FastISel missed terminator";

1461

// Don't abort for terminator unless the level is really high

1462

ShouldAbort = (EnableFastISelAbort > 2);

1463

} else {

1464

R << "FastISel missed";

1465

}

1466

1467

if (R.isEnabled() || EnableFastISelAbort) {

1468

std::string InstStrStorage;

1469

raw_string_ostream InstStr(InstStrStorage);

1470

InstStr << *Inst;

1471

R << ": " << InstStr.str();

1472

}

1473

1474

reportFastISelFailure(*MF, *ORE, R, ShouldAbort);

1475

1476

NumFastIselFailures += NumFastIselRemaining;

1477

break;

1478

}

1479

1480

FastIS->recomputeInsertPt();

1481

}

1482

1483

if (getAnalysis<StackProtector>().shouldEmitSDCheck(*LLVMBB)) {

←

Assuming the condition is false

Taking false branch

Assuming the condition is false

Taking false branch

Assuming the condition is false

Taking false branch

Assuming the condition is false

→

←

Taking false branch

→

1484

bool FunctionBasedInstrumentation =

1485

TLI->getSSPStackGuardCheck(*Fn.getParent());

1486

SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->MBBMap[LLVMBB],

1487

FunctionBasedInstrumentation);

1488

}

1489

1490

if (Begin != BI)

Taking false branch

Taking false branch

Taking false branch

Taking true branch

1491

++NumDAGBlocks;

1492

else

1493

++NumFastIselBlocks;

1494

1495

if (Begin != BI) {

Taking false branch

Taking false branch

Taking false branch

Taking true branch

1496

// Run SelectionDAG instruction selection on the remainder of the block

1497

// not handled by FastISel. If FastISel is not run, this is the entire

1498

// block.

1499

bool HadTailCall;

1500

SelectBasicBlock(Begin, BI, HadTailCall);

1501

1502

// But if FastISel was run, we already selected some of the block.

1503

// If we emitted a tail-call, we need to delete any previously emitted

1504

// instruction that follows it.

1505

if (HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())

←

Assuming 'HadTailCall' is not equal to 0

→

←

Taking true branch

→

1506

FastIS->removeDeadCode(FuncInfo->InsertPt, FuncInfo->MBB->end());

←

Called C++ object pointer is null

1507

}

1508

1509

FinishBasicBlock();

1510

FuncInfo->PHINodesToUpdate.clear();

1511

ElidedArgCopyInstrs.clear();

1512

}

1513

1514

propagateSwiftErrorVRegs(FuncInfo);

1515

1516

delete FastIS;

1517

SDB->clearDanglingDebugInfo();

1518

SDB->SPDescriptor.resetPerFunctionState();

1519

}

1520

1521

/// Given that the input MI is before a partial terminator sequence TSeq, return

1522

/// true if M + TSeq also a partial terminator sequence.

1523

///

1524

/// A Terminator sequence is a sequence of MachineInstrs which at this point in

1525

/// lowering copy vregs into physical registers, which are then passed into

1526

/// terminator instructors so we can satisfy ABI constraints. A partial

1527

/// terminator sequence is an improper subset of a terminator sequence (i.e. it

1528

/// may be the whole terminator sequence).

1529

static bool MIIsInTerminatorSequence(const MachineInstr &MI) {

1530

// If we do not have a copy or an implicit def, we return true if and only if

1531

// MI is a debug value.

1532

if (!MI.isCopy() && !MI.isImplicitDef())

1533

// Sometimes DBG_VALUE MI sneak in between the copies from the vregs to the

1534

// physical registers if there is debug info associated with the terminator

1535

// of our mbb. We want to include said debug info in our terminator

1536

// sequence, so we return true in that case.

1537

return MI.isDebugValue();

1538

1539

// We have left the terminator sequence if we are not doing one of the

1540

// following:

1541

1542

// 1. Copying a vreg into a physical register.

1543

// 2. Copying a vreg into a vreg.

1544

// 3. Defining a register via an implicit def.

1545

1546

// OPI should always be a register definition...

1547

MachineInstr::const_mop_iterator OPI = MI.operands_begin();

1548

if (!OPI->isReg() || !OPI->isDef())

1549

return false;

1550

1551

// Defining any register via an implicit def is always ok.

1552

if (MI.isImplicitDef())

1553

return true;

1554

1555

// Grab the copy source...

1556

MachineInstr::const_mop_iterator OPI2 = OPI;

1557

++OPI2;

1558

assert(OPI2 != MI.operands_end()((OPI2 != MI.operands_end() && "Should have a copy implying we should have 2 arguments."
) ? static_cast<void> (0) : __assert_fail ("OPI2 != MI.operands_end() && \"Should have a copy implying we should have 2 arguments.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1559, __PRETTY_FUNCTION__))

1559

&& "Should have a copy implying we should have 2 arguments.")((OPI2 != MI.operands_end() && "Should have a copy implying we should have 2 arguments."
) ? static_cast<void> (0) : __assert_fail ("OPI2 != MI.operands_end() && \"Should have a copy implying we should have 2 arguments.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1559, __PRETTY_FUNCTION__));

1560

1561

// Make sure that the copy dest is not a vreg when the copy source is a

1562

// physical register.

1563

if (!OPI2->isReg() ||

1564

(!TargetRegisterInfo::isPhysicalRegister(OPI->getReg()) &&

1565

TargetRegisterInfo::isPhysicalRegister(OPI2->getReg())))

1566

return false;

1567

1568

return true;

1569

}

1570

1571

/// Find the split point at which to splice the end of BB into its success stack

1572

/// protector check machine basic block.

1573

///

1574

/// On many platforms, due to ABI constraints, terminators, even before register

1575

/// allocation, use physical registers. This creates an issue for us since

1576

/// physical registers at this point can not travel across basic

1577

/// blocks. Luckily, selectiondag always moves physical registers into vregs

1578

/// when they enter functions and moves them through a sequence of copies back

1579

/// into the physical registers right before the terminator creating a

1580

/// ``Terminator Sequence''. This function is searching for the beginning of the

1581

/// terminator sequence so that we can ensure that we splice off not just the

1582

/// terminator, but additionally the copies that move the vregs into the

1583

/// physical registers.

1584

static MachineBasicBlock::iterator

1585

FindSplitPointForStackProtector(MachineBasicBlock *BB) {

1586

MachineBasicBlock::iterator SplitPoint = BB->getFirstTerminator();

1587

1588

if (SplitPoint == BB->begin())

1589

return SplitPoint;

1590

1591

MachineBasicBlock::iterator Start = BB->begin();

1592

MachineBasicBlock::iterator Previous = SplitPoint;

1593

--Previous;

1594

1595

while (MIIsInTerminatorSequence(*Previous)) {

1596

SplitPoint = Previous;

1597

if (Previous == Start)

1598

break;

1599

--Previous;

1600

}

1601

1602

return SplitPoint;

1603

}

1604

1605

void

1606

SelectionDAGISel::FinishBasicBlock() {

1607

DEBUG(dbgs() << "Total amount of phi nodes to update: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " <PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)

1608

<< FuncInfo->PHINodesToUpdate.size() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " <PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)

1609

for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " <PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)

1610

dbgs() << "Node " <PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " <PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)

1611

<< FuncInfo->PHINodesToUpdate[i].firstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " <PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false)

1612

<< ", " << FuncInfo->PHINodesToUpdate[i].second << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Total amount of phi nodes to update: "
<< FuncInfo->PHINodesToUpdate.size() << "\n";
for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(
); i != e; ++i) dbgs() << "Node " <PHINodesToUpdate[i].first << ", "
<< FuncInfo->PHINodesToUpdate[i].second << ")\n"
; } } while (false);

1613

1614

// Next, now that we know what the last MBB the LLVM BB expanded is, update

1615

// PHI nodes in successors.

1616

for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {

1617

MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);

1618

assert(PHI->isPHI() &&((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1619, __PRETTY_FUNCTION__))

1619

"This is not a machine PHI node that we are updating!")((PHI->isPHI() && "This is not a machine PHI node that we are updating!"
) ? static_cast<void> (0) : __assert_fail ("PHI->isPHI() && \"This is not a machine PHI node that we are updating!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1619, __PRETTY_FUNCTION__));

1620

if (!FuncInfo->MBB->isSuccessor(PHI->getParent()))

1621

continue;

1622

PHI.addReg(FuncInfo->PHINodesToUpdate[i].second).addMBB(FuncInfo->MBB);

1623

}

1624

1625

// Handle stack protector.

1626

if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {

1627

// The target provides a guard check function. There is no need to

1628

// generate error handling code or to split current basic block.

1629

MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();

1630

1631

// Add load and check to the basicblock.

1632

FuncInfo->MBB = ParentMBB;

1633

FuncInfo->InsertPt =

1634

FindSplitPointForStackProtector(ParentMBB);

1635

SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);

1636

CurDAG->setRoot(SDB->getRoot());

1637

SDB->clear();

1638

CodeGenAndEmitDAG();

1639

1640

// Clear the Per-BB State.

1641

SDB->SPDescriptor.resetPerBBState();

1642

} else if (SDB->SPDescriptor.shouldEmitStackProtector()) {

1643

MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();

1644

MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();

1645

1646

// Find the split point to split the parent mbb. At the same time copy all

1647

// physical registers used in the tail of parent mbb into virtual registers

1648

// before the split point and back into physical registers after the split

1649

// point. This prevents us needing to deal with Live-ins and many other

1650

// register allocation issues caused by us splitting the parent mbb. The

1651

// register allocator will clean up said virtual copies later on.

1652

MachineBasicBlock::iterator SplitPoint =

1653

FindSplitPointForStackProtector(ParentMBB);

1654

1655

// Splice the terminator of ParentMBB into SuccessMBB.

1656

SuccessMBB->splice(SuccessMBB->end(), ParentMBB,

1657

SplitPoint,

1658

ParentMBB->end());

1659

1660

// Add compare/jump on neq/jump to the parent BB.

1661

FuncInfo->MBB = ParentMBB;

1662

FuncInfo->InsertPt = ParentMBB->end();

1663

SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);

1664

CurDAG->setRoot(SDB->getRoot());

1665

SDB->clear();

1666

CodeGenAndEmitDAG();

1667

1668

// CodeGen Failure MBB if we have not codegened it yet.

1669

MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();

1670

if (FailureMBB->empty()) {

1671

FuncInfo->MBB = FailureMBB;

1672

FuncInfo->InsertPt = FailureMBB->end();

1673

SDB->visitSPDescriptorFailure(SDB->SPDescriptor);

1674

CurDAG->setRoot(SDB->getRoot());

1675

SDB->clear();

1676

CodeGenAndEmitDAG();

1677

}

1678

1679

// Clear the Per-BB State.

1680

SDB->SPDescriptor.resetPerBBState();

1681

}

1682

1683

// Lower each BitTestBlock.

1684

for (auto &BTB : SDB->BitTestCases) {

1685

// Lower header first, if it wasn't already lowered

1686

if (!BTB.Emitted) {

1687

// Set the current basic block to the mbb we wish to insert the code into

1688

FuncInfo->MBB = BTB.Parent;

1689

FuncInfo->InsertPt = FuncInfo->MBB->end();

1690

// Emit the code

1691

SDB->visitBitTestHeader(BTB, FuncInfo->MBB);

1692

CurDAG->setRoot(SDB->getRoot());

1693

SDB->clear();

1694

CodeGenAndEmitDAG();

1695

}

1696

1697

BranchProbability UnhandledProb = BTB.Prob;

1698

for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {

1699

UnhandledProb -= BTB.Cases[j].ExtraProb;

1700

// Set the current basic block to the mbb we wish to insert the code into

1701

FuncInfo->MBB = BTB.Cases[j].ThisBB;

1702

FuncInfo->InsertPt = FuncInfo->MBB->end();

1703

// Emit the code

1704

1705

// If all cases cover a contiguous range, it is not necessary to jump to

1706

// the default block after the last bit test fails. This is because the

1707

// range check during bit test header creation has guaranteed that every

1708

// case here doesn't go outside the range. In this case, there is no need

1709

// to perform the last bit test, as it will always be true. Instead, make

1710

// the second-to-last bit-test fall through to the target of the last bit

1711

// test, and delete the last bit test.

1712

1713

MachineBasicBlock *NextMBB;

1714

if (BTB.ContiguousRange && j + 2 == ej) {

1715

// Second-to-last bit-test with contiguous range: fall through to the

1716

// target of the final bit test.

1717

NextMBB = BTB.Cases[j + 1].TargetBB;

1718

} else if (j + 1 == ej) {

1719

// For the last bit test, fall through to Default.

1720

NextMBB = BTB.Default;

1721

} else {

1722

// Otherwise, fall through to the next bit test.

1723

NextMBB = BTB.Cases[j + 1].ThisBB;

1724

}

1725

1726

SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],

1727

FuncInfo->MBB);

1728

1729

CurDAG->setRoot(SDB->getRoot());

1730

SDB->clear();

1731

CodeGenAndEmitDAG();

1732

1733

if (BTB.ContiguousRange && j + 2 == ej) {

1734

// Since we're not going to use the final bit test, remove it.

1735

BTB.Cases.pop_back();

1736

break;

1737

}

1738

}

1739

1740

// Update PHI Nodes

1741

for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();

1742

pi != pe; ++pi) {

1743

MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);

1744

MachineBasicBlock *PHIBB = PHI->getParent();

1745

1746

1747

// This is "default" BB. We have two jumps to it. From "header" BB and

1748

// from last "case" BB, unless the latter was skipped.

1749

if (PHIBB == BTB.Default) {

1750

PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(BTB.Parent);

1751

if (!BTB.ContiguousRange) {

1752

PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)

1753

.addMBB(BTB.Cases.back().ThisBB);

1754

}

1755

}

1756

// One of "cases" BB.

1757

for (unsigned j = 0, ej = BTB.Cases.size();

1758

j != ej; ++j) {

1759

MachineBasicBlock* cBB = BTB.Cases[j].ThisBB;

1760

if (cBB->isSuccessor(PHIBB))

1761

PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(cBB);

1762

}

1763

}

1764

}

1765

SDB->BitTestCases.clear();

1766

1767

// If the JumpTable record is filled in, then we need to emit a jump table.

1768

// Updating the PHI nodes is tricky in this case, since we need to determine

1769

// whether the PHI is a successor of the range check MBB or the jump table MBB

1770

for (unsigned i = 0, e = SDB->JTCases.size(); i != e; ++i) {

1771

// Lower header first, if it wasn't already lowered

1772

if (!SDB->JTCases[i].first.Emitted) {

1773

// Set the current basic block to the mbb we wish to insert the code into

1774

FuncInfo->MBB = SDB->JTCases[i].first.HeaderBB;

1775

FuncInfo->InsertPt = FuncInfo->MBB->end();

1776

// Emit the code

1777

SDB->visitJumpTableHeader(SDB->JTCases[i].second, SDB->JTCases[i].first,

1778

FuncInfo->MBB);

1779

CurDAG->setRoot(SDB->getRoot());

1780

SDB->clear();

1781

CodeGenAndEmitDAG();

1782

}

1783

1784

// Set the current basic block to the mbb we wish to insert the code into

1785

FuncInfo->MBB = SDB->JTCases[i].second.MBB;

1786

FuncInfo->InsertPt = FuncInfo->MBB->end();

1787

// Emit the code

1788

SDB->visitJumpTable(SDB->JTCases[i].second);

1789

CurDAG->setRoot(SDB->getRoot());

1790

SDB->clear();

1791

CodeGenAndEmitDAG();

1792

1793

// Update PHI Nodes

1794

for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();

1795

pi != pe; ++pi) {

1796

MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);

1797

MachineBasicBlock *PHIBB = PHI->getParent();

1798

1799

1800

// "default" BB. We can go there only from header BB.

1801

if (PHIBB == SDB->JTCases[i].second.Default)

1802

PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)

1803

.addMBB(SDB->JTCases[i].first.HeaderBB);

1804

// JT BB. Just iterate over successors here

1805

if (FuncInfo->MBB->isSuccessor(PHIBB))

1806

PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);

1807

}

1808

}

1809

SDB->JTCases.clear();

1810

1811

// If we generated any switch lowering information, build and codegen any

1812

// additional DAGs necessary.

1813

for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) {

1814

// Set the current basic block to the mbb we wish to insert the code into

1815

FuncInfo->MBB = SDB->SwitchCases[i].ThisBB;

1816

FuncInfo->InsertPt = FuncInfo->MBB->end();

1817

1818

// Determine the unique successors.

1819

SmallVector<MachineBasicBlock *, 2> Succs;

1820

Succs.push_back(SDB->SwitchCases[i].TrueBB);

1821

if (SDB->SwitchCases[i].TrueBB != SDB->SwitchCases[i].FalseBB)

1822

Succs.push_back(SDB->SwitchCases[i].FalseBB);

1823

1824

// Emit the code. Note that this could result in FuncInfo->MBB being split.

1825

SDB->visitSwitchCase(SDB->SwitchCases[i], FuncInfo->MBB);

1826

CurDAG->setRoot(SDB->getRoot());

1827

SDB->clear();

1828

CodeGenAndEmitDAG();

1829

1830

// Remember the last block, now that any splitting is done, for use in

1831

// populating PHI nodes in successors.

1832

MachineBasicBlock *ThisBB = FuncInfo->MBB;

1833

1834

// Handle any PHI nodes in successors of this chunk, as if we were coming

1835

// from the original BB before switch expansion. Note that PHI nodes can

1836

// occur multiple times in PHINodesToUpdate. We have to be very careful to

1837

// handle them the right number of times.

1838

for (unsigned i = 0, e = Succs.size(); i != e; ++i) {

1839

FuncInfo->MBB = Succs[i];

1840

FuncInfo->InsertPt = FuncInfo->MBB->end();

1841

// FuncInfo->MBB may have been removed from the CFG if a branch was

1842

// constant folded.

1843

if (ThisBB->isSuccessor(FuncInfo->MBB)) {

1844

for (MachineBasicBlock::iterator

1845

MBBI = FuncInfo->MBB->begin(), MBBE = FuncInfo->MBB->end();

1846

MBBI != MBBE && MBBI->isPHI(); ++MBBI) {

1847

MachineInstrBuilder PHI(*MF, MBBI);

1848

// This value for this PHI node is recorded in PHINodesToUpdate.

1849

for (unsigned pn = 0; ; ++pn) {

1850

assert(pn != FuncInfo->PHINodesToUpdate.size() &&((pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!"
) ? static_cast<void> (0) : __assert_fail ("pn != FuncInfo->PHINodesToUpdate.size() && \"Didn't find PHI entry!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1851, __PRETTY_FUNCTION__))

1851

"Didn't find PHI entry!")((pn != FuncInfo->PHINodesToUpdate.size() && "Didn't find PHI entry!"
) ? static_cast<void> (0) : __assert_fail ("pn != FuncInfo->PHINodesToUpdate.size() && \"Didn't find PHI entry!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1851, __PRETTY_FUNCTION__));

1852

if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {

1853

PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);

1854

break;

1855

}

1856

}

1857

}

1858

}

1859

}

1860

}

1861

SDB->SwitchCases.clear();

1862

}

1863

1864

/// Create the scheduler. If a specific scheduler was specified

1865

/// via the SchedulerRegistry, use it, otherwise select the

1866

/// one preferred by the target.

1867

///

1868

ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() {

1869

return ISHeuristic(this, OptLevel);

1870

}

1871

1872

//===----------------------------------------------------------------------===//

1873

// Helper functions used by the generated instruction selector.

1874

//===----------------------------------------------------------------------===//

1875

// Calls to these methods are generated by tblgen.

1876

1877

/// CheckAndMask - The isel is trying to match something like (and X, 255). If

1878

/// the dag combiner simplified the 255, we still want to match. RHS is the

1879

/// actual value in the DAG on the RHS of an AND, and DesiredMaskS is the value

1880

/// specified in the .td file (e.g. 255).

1881

bool SelectionDAGISel::CheckAndMask(SDValue LHS, ConstantSDNode *RHS,

1882

int64_t DesiredMaskS) const {

1883

const APInt &ActualMask = RHS->getAPIntValue();

1884

const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);

1885

1886

// If the actual mask exactly matches, success!

1887

if (ActualMask == DesiredMask)

1888

return true;

1889

1890

// If the actual AND mask is allowing unallowed bits, this doesn't match.

1891

if (ActualMask.intersects(~DesiredMask))

1892

return false;

1893

1894

// Otherwise, the DAG Combiner may have proven that the value coming in is

1895

// either already zero or is not demanded. Check for known zero input bits.

1896

APInt NeededMask = DesiredMask & ~ActualMask;

1897

if (CurDAG->MaskedValueIsZero(LHS, NeededMask))

1898

return true;

1899

1900

// TODO: check to see if missing bits are just not demanded.

1901

1902

// Otherwise, this pattern doesn't match.

1903

return false;

1904

}

1905

1906

/// CheckOrMask - The isel is trying to match something like (or X, 255). If

1907

/// the dag combiner simplified the 255, we still want to match. RHS is the

1908

/// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value

1909

/// specified in the .td file (e.g. 255).

1910

bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS,

1911

int64_t DesiredMaskS) const {

1912

const APInt &ActualMask = RHS->getAPIntValue();

1913

const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS);

1914

1915

// If the actual mask exactly matches, success!

1916

if (ActualMask == DesiredMask)

1917

return true;

1918

1919

// If the actual AND mask is allowing unallowed bits, this doesn't match.

1920

if (ActualMask.intersects(~DesiredMask))

1921

return false;

1922

1923

// Otherwise, the DAG Combiner may have proven that the value coming in is

1924

// either already zero or is not demanded. Check for known zero input bits.

1925

APInt NeededMask = DesiredMask & ~ActualMask;

1926

1927

KnownBits Known;

1928

CurDAG->computeKnownBits(LHS, Known);

1929

1930

// If all the missing bits in the or are already known to be set, match!

1931

if (NeededMask.isSubsetOf(Known.One))

1932

return true;

1933

1934

// TODO: check to see if missing bits are just not demanded.

1935

1936

// Otherwise, this pattern doesn't match.

1937

return false;

1938

}

1939

1940

/// SelectInlineAsmMemoryOperands - Calls to this are automatically generated

1941

/// by tblgen. Others should not call it.

1942

void SelectionDAGISel::SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops,

1943

const SDLoc &DL) {

1944

std::vector<SDValue> InOps;

1945

std::swap(InOps, Ops);

1946

1947

Ops.push_back(InOps[InlineAsm::Op_InputChain]); // 0

1948

Ops.push_back(InOps[InlineAsm::Op_AsmString]); // 1

1949

Ops.push_back(InOps[InlineAsm::Op_MDNode]); // 2, !srcloc

1950

Ops.push_back(InOps[InlineAsm::Op_ExtraInfo]); // 3 (SideEffect, AlignStack)

1951

1952

unsigned i = InlineAsm::Op_FirstOperand, e = InOps.size();

1953

if (InOps[e-1].getValueType() == MVT::Glue)

1954

--e; // Don't process a glue operand if it is here.

1955

1956

while (i != e) {

1957

unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();

1958

if (!InlineAsm::isMemKind(Flags)) {

1959

// Just skip over this operand, copying the operands verbatim.

1960

Ops.insert(Ops.end(), InOps.begin()+i,

1961

InOps.begin()+i+InlineAsm::getNumOperandRegisters(Flags) + 1);

1962

i += InlineAsm::getNumOperandRegisters(Flags) + 1;

1963

} else {

1964

assert(InlineAsm::getNumOperandRegisters(Flags) == 1 &&((InlineAsm::getNumOperandRegisters(Flags) == 1 && "Memory operand with multiple values?"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(Flags) == 1 && \"Memory operand with multiple values?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1965, __PRETTY_FUNCTION__))

1965

"Memory operand with multiple values?")((InlineAsm::getNumOperandRegisters(Flags) == 1 && "Memory operand with multiple values?"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(Flags) == 1 && \"Memory operand with multiple values?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 1965, __PRETTY_FUNCTION__));

1966

1967

unsigned TiedToOperand;

1968

if (InlineAsm::isUseOperandTiedToDef(Flags, TiedToOperand)) {

1969

// We need the constraint ID from the operand this is tied to.

1970

unsigned CurOp = InlineAsm::Op_FirstOperand;

1971

Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();

1972

for (; TiedToOperand; --TiedToOperand) {

1973

CurOp += InlineAsm::getNumOperandRegisters(Flags)+1;

1974

Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();

1975

}

1976

}

1977

1978

// Otherwise, this is a memory operand. Ask the target to select it.

1979

std::vector<SDValue> SelOps;

1980

unsigned ConstraintID = InlineAsm::getMemoryConstraintID(Flags);

1981

if (SelectInlineAsmMemoryOperand(InOps[i+1], ConstraintID, SelOps))

1982

report_fatal_error("Could not match memory address. Inline asm"

1983

" failure!");

1984

1985

// Add this to the output node.

1986

unsigned NewFlags =

1987

InlineAsm::getFlagWord(InlineAsm::Kind_Mem, SelOps.size());

1988

NewFlags = InlineAsm::getFlagWordForMem(NewFlags, ConstraintID);

1989

Ops.push_back(CurDAG->getTargetConstant(NewFlags, DL, MVT::i32));

1990

Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());

1991

i += 2;

1992

}

1993

}

1994

1995

// Add the glue input back if present.

1996

if (e != InOps.size())

1997

Ops.push_back(InOps.back());

1998

}

1999

2000

/// findGlueUse - Return use of MVT::Glue value produced by the specified

2001

/// SDNode.

2002

///

2003

static SDNode *findGlueUse(SDNode *N) {

2004

unsigned FlagResNo = N->getNumValues()-1;

2005

for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {

2006

SDUse &Use = I.getUse();

2007

if (Use.getResNo() == FlagResNo)

2008

return Use.getUser();

2009

}

2010

return nullptr;

2011

}

2012

2013

/// findNonImmUse - Return true if "Use" is a non-immediate use of "Def".

2014

/// This function recursively traverses up the operand chain, ignoring

2015

/// certain nodes.

2016

static bool findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse,

2017

SDNode *Root, SmallPtrSetImpl<SDNode*> &Visited,

2018

bool IgnoreChains) {

2019

// The NodeID's are given uniques ID's where a node ID is guaranteed to be

2020

// greater than all of its (recursive) operands. If we scan to a point where

2021

// 'use' is smaller than the node we're scanning for, then we know we will

2022

// never find it.

2023

2024

// The Use may be -1 (unassigned) if it is a newly allocated node. This can

2025

// happen because we scan down to newly selected nodes in the case of glue

2026

// uses.

2027

if ((Use->getNodeId() < Def->getNodeId() && Use->getNodeId() != -1))

2028

return false;

2029

2030

// Don't revisit nodes if we already scanned it and didn't fail, we know we

2031

// won't fail if we scan it again.

2032

if (!Visited.insert(Use).second)

2033

return false;

2034

2035

for (const SDValue &Op : Use->op_values()) {

2036

// Ignore chain uses, they are validated by HandleMergeInputChains.

2037

if (Op.getValueType() == MVT::Other && IgnoreChains)

2038

continue;

2039

2040

SDNode *N = Op.getNode();

2041

if (N == Def) {

2042

if (Use == ImmedUse || Use == Root)

2043

continue; // We are not looking for immediate use.

2044

assert(N != Root)((N != Root) ? static_cast<void> (0) : __assert_fail ("N != Root"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2044, __PRETTY_FUNCTION__));

2045

return true;

2046

}

2047

2048

// Traverse up the operand chain.

2049

if (findNonImmUse(N, Def, ImmedUse, Root, Visited, IgnoreChains))

2050

return true;

2051

}

2052

return false;

2053

}

2054

2055

/// IsProfitableToFold - Returns true if it's profitable to fold the specific

2056

/// operand node N of U during instruction selection that starts at Root.

2057

bool SelectionDAGISel::IsProfitableToFold(SDValue N, SDNode *U,

2058

SDNode *Root) const {

2059

if (OptLevel == CodeGenOpt::None) return false;

2060

return N.hasOneUse();

2061

}

2062

2063

/// IsLegalToFold - Returns true if the specific operand node N of

2064

/// U can be folded during instruction selection that starts at Root.

2065

bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,

2066

CodeGenOpt::Level OptLevel,

2067

bool IgnoreChains) {

2068

if (OptLevel == CodeGenOpt::None) return false;

2069

2070

// If Root use can somehow reach N through a path that that doesn't contain

2071

// U then folding N would create a cycle. e.g. In the following

2072

// diagram, Root can reach N through X. If N is folded into into Root, then

2073

// X is both a predecessor and a successor of U.

2074

2075

// [N*] //

2076

// ^ ^ //

2077

// / \ //

2078

// [U*] [X]? //

2079

// ^ ^ //

2080

// \ / //

2081

// \ / //

2082

// [Root*] //

2083

2084

// * indicates nodes to be folded together.

2085

2086

// If Root produces glue, then it gets (even more) interesting. Since it

2087

// will be "glued" together with its glue use in the scheduler, we need to

2088

// check if it might reach N.

2089

2090

// [N*] //

2091

// ^ ^ //

2092

// / \ //

2093

// [U*] [X]? //

2094

// ^ ^ //

2095

// \ \ //

2096

// \ | //

2097

// [Root*] | //

2098

// ^ | //

2099

// f | //

2100

// | / //

2101

// [Y] / //

2102

// ^ / //

2103

// f / //

2104

// | / //

2105

// [GU] //

2106

2107

// If GU (glue use) indirectly reaches N (the load), and Root folds N

2108

// (call it Fold), then X is a predecessor of GU and a successor of

2109

// Fold. But since Fold and GU are glued together, this will create

2110

// a cycle in the scheduling graph.

2111

2112

// If the node has glue, walk down the graph to the "lowest" node in the

2113

// glueged set.

2114

EVT VT = Root->getValueType(Root->getNumValues()-1);

2115

while (VT == MVT::Glue) {

2116

SDNode *GU = findGlueUse(Root);

2117

if (!GU)

2118

break;

2119

Root = GU;

2120

VT = Root->getValueType(Root->getNumValues()-1);

2121

2122

// If our query node has a glue result with a use, we've walked up it. If

2123

// the user (which has already been selected) has a chain or indirectly uses

2124

// the chain, our WalkChainUsers predicate will not consider it. Because of

2125

// this, we cannot ignore chains in this predicate.

2126

IgnoreChains = false;

2127

}

2128

2129

SmallPtrSet<SDNode*, 16> Visited;

2130

return !findNonImmUse(Root, N.getNode(), U, Root, Visited, IgnoreChains);

2131

}

2132

2133

void SelectionDAGISel::Select_INLINEASM(SDNode *N) {

2134

SDLoc DL(N);

2135

2136

std::vector<SDValue> Ops(N->op_begin(), N->op_end());

2137

SelectInlineAsmMemoryOperands(Ops, DL);

2138

2139

const EVT VTs[] = {MVT::Other, MVT::Glue};

2140

SDValue New = CurDAG->getNode(ISD::INLINEASM, DL, VTs, Ops);

2141

New->setNodeId(-1);

2142

ReplaceUses(N, New.getNode());

2143

CurDAG->RemoveDeadNode(N);

2144

}

2145

2146

void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {

2147

SDLoc dl(Op);

2148

MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));

2149

const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));

2150

unsigned Reg =

2151

TLI->getRegisterByName(RegStr->getString().data(), Op->getValueType(0),

2152

*CurDAG);

2153

SDValue New = CurDAG->getCopyFromReg(

2154

Op->getOperand(0), dl, Reg, Op->getValueType(0));

2155

New->setNodeId(-1);

2156

ReplaceUses(Op, New.getNode());

2157

CurDAG->RemoveDeadNode(Op);

2158

}

2159

2160

void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {

2161

SDLoc dl(Op);

2162

MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));

2163

const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));

2164

unsigned Reg = TLI->getRegisterByName(RegStr->getString().data(),

2165

Op->getOperand(2).getValueType(),

2166

*CurDAG);

2167

SDValue New = CurDAG->getCopyToReg(

2168

Op->getOperand(0), dl, Reg, Op->getOperand(2));

2169

New->setNodeId(-1);

2170

ReplaceUses(Op, New.getNode());

2171

CurDAG->RemoveDeadNode(Op);

2172

}

2173

2174

void SelectionDAGISel::Select_UNDEF(SDNode *N) {

2175

CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));

2176

}

2177

2178

/// GetVBR - decode a vbr encoding whose top bit is set.

2179

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline uint64_t

2180

GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {

2181

assert(Val >= 128 && "Not a VBR")((Val >= 128 && "Not a VBR") ? static_cast<void
> (0) : __assert_fail ("Val >= 128 && \"Not a VBR\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2181, __PRETTY_FUNCTION__));

2182

Val &= 127; // Remove first vbr bit.

2183

2184

unsigned Shift = 7;

2185

uint64_t NextBits;

2186

do {

2187

NextBits = MatcherTable[Idx++];

2188

Val |= (NextBits&127) << Shift;

2189

Shift += 7;

2190

} while (NextBits & 128);

2191

2192

return Val;

2193

}

2194

2195

/// When a match is complete, this method updates uses of interior chain results

2196

/// to use the new results.

2197

void SelectionDAGISel::UpdateChains(

2198

SDNode *NodeToMatch, SDValue InputChain,

2199

SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) {

2200

SmallVector<SDNode*, 4> NowDeadNodes;

2201

2202

// Now that all the normal results are replaced, we replace the chain and

2203

// glue results if present.

2204

if (!ChainNodesMatched.empty()) {

2205

assert(InputChain.getNode() &&((InputChain.getNode() && "Matched input chains but didn't produce a chain"
) ? static_cast<void> (0) : __assert_fail ("InputChain.getNode() && \"Matched input chains but didn't produce a chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2206, __PRETTY_FUNCTION__))

2206

"Matched input chains but didn't produce a chain")((InputChain.getNode() && "Matched input chains but didn't produce a chain"
) ? static_cast<void> (0) : __assert_fail ("InputChain.getNode() && \"Matched input chains but didn't produce a chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2206, __PRETTY_FUNCTION__));

2207

// Loop over all of the nodes we matched that produced a chain result.

2208

// Replace all the chain results with the final chain we ended up with.

2209

for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {

2210

SDNode *ChainNode = ChainNodesMatched[i];

2211

// If ChainNode is null, it's because we replaced it on a previous

2212

// iteration and we cleared it out of the map. Just skip it.

2213

if (!ChainNode)

2214

continue;

2215

2216

assert(ChainNode->getOpcode() != ISD::DELETED_NODE &&((ChainNode->getOpcode() != ISD::DELETED_NODE && "Deleted node left in chain"
) ? static_cast<void> (0) : __assert_fail ("ChainNode->getOpcode() != ISD::DELETED_NODE && \"Deleted node left in chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2217, __PRETTY_FUNCTION__))

2217

"Deleted node left in chain")((ChainNode->getOpcode() != ISD::DELETED_NODE && "Deleted node left in chain"
) ? static_cast<void> (0) : __assert_fail ("ChainNode->getOpcode() != ISD::DELETED_NODE && \"Deleted node left in chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2217, __PRETTY_FUNCTION__));

2218

2219

// Don't replace the results of the root node if we're doing a

2220

// MorphNodeTo.

2221

if (ChainNode == NodeToMatch && isMorphNodeTo)

2222

continue;

2223

2224

SDValue ChainVal = SDValue(ChainNode, ChainNode->getNumValues()-1);

2225

if (ChainVal.getValueType() == MVT::Glue)

2226

ChainVal = ChainVal.getValue(ChainVal->getNumValues()-2);

2227

assert(ChainVal.getValueType() == MVT::Other && "Not a chain?")((ChainVal.getValueType() == MVT::Other && "Not a chain?"
) ? static_cast<void> (0) : __assert_fail ("ChainVal.getValueType() == MVT::Other && \"Not a chain?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2227, __PRETTY_FUNCTION__));

2228

SelectionDAG::DAGNodeDeletedListener NDL(

2229

*CurDAG, [&](SDNode *N, SDNode *E) {

2230

std::replace(ChainNodesMatched.begin(), ChainNodesMatched.end(), N,

2231

static_cast<SDNode *>(nullptr));

2232

});

2233

CurDAG->ReplaceAllUsesOfValueWith(ChainVal, InputChain);

2234

2235

// If the node became dead and we haven't already seen it, delete it.

2236

if (ChainNode != NodeToMatch && ChainNode->use_empty() &&

2237

!std::count(NowDeadNodes.begin(), NowDeadNodes.end(), ChainNode))

2238

NowDeadNodes.push_back(ChainNode);

2239

}

2240

}

2241

2242

if (!NowDeadNodes.empty())

2243

CurDAG->RemoveDeadNodes(NowDeadNodes);

2244

2245

DEBUG(dbgs() << "ISEL: Match complete!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Match complete!\n"; } } while
(false);

2246

}

2247

2248

enum ChainResult {

2249

CR_Simple,

2250

CR_InducesCycle,

2251

CR_LeadsToInteriorNode

2252

};

2253

2254

/// WalkChainUsers - Walk down the users of the specified chained node that is

2255

/// part of the pattern we're matching, looking at all of the users we find.

2256

/// This determines whether something is an interior node, whether we have a

2257

/// non-pattern node in between two pattern nodes (which prevent folding because

2258

/// it would induce a cycle) and whether we have a TokenFactor node sandwiched

2259

/// between pattern nodes (in which case the TF becomes part of the pattern).

2260

///

2261

/// The walk we do here is guaranteed to be small because we quickly get down to

2262

/// already selected nodes "below" us.

2263

static ChainResult

2264

WalkChainUsers(const SDNode *ChainedNode,

2265

SmallVectorImpl<SDNode *> &ChainedNodesInPattern,

2266

DenseMap<const SDNode *, ChainResult> &TokenFactorResult,

2267

SmallVectorImpl<SDNode *> &InteriorChainedNodes) {

2268

ChainResult Result = CR_Simple;

2269

2270

for (SDNode::use_iterator UI = ChainedNode->use_begin(),

2271

E = ChainedNode->use_end(); UI != E; ++UI) {

2272

// Make sure the use is of the chain, not some other value we produce.

2273

if (UI.getUse().getValueType() != MVT::Other) continue;

2274

2275

SDNode *User = *UI;

2276

2277

if (User->getOpcode() == ISD::HANDLENODE) // Root of the graph.

2278

continue;

2279

2280

// If we see an already-selected machine node, then we've gone beyond the

2281

// pattern that we're selecting down into the already selected chunk of the

2282

// DAG.

2283

unsigned UserOpcode = User->getOpcode();

2284

if (User->isMachineOpcode() ||

2285

UserOpcode == ISD::CopyToReg ||

2286

UserOpcode == ISD::CopyFromReg ||

2287

UserOpcode == ISD::INLINEASM ||

2288

UserOpcode == ISD::EH_LABEL ||

2289

UserOpcode == ISD::LIFETIME_START ||

2290

UserOpcode == ISD::LIFETIME_END) {

2291

// If their node ID got reset to -1 then they've already been selected.

2292

// Treat them like a MachineOpcode.

2293

if (User->getNodeId() == -1)

2294

continue;

2295

}

2296

2297

// If we have a TokenFactor, we handle it specially.

2298

if (User->getOpcode() != ISD::TokenFactor) {

2299

// If the node isn't a token factor and isn't part of our pattern, then it

2300

// must be a random chained node in between two nodes we're selecting.

2301

// This happens when we have something like:

2302

// x = load ptr

2303

// call

2304

// y = x+4

2305

// store y -> ptr

2306

// Because we structurally match the load/store as a read/modify/write,

2307

// but the call is chained between them. We cannot fold in this case

2308

// because it would induce a cycle in the graph.

2309

if (!std::count(ChainedNodesInPattern.begin(),

2310

ChainedNodesInPattern.end(), User))

2311

return CR_InducesCycle;

2312

2313

// Otherwise we found a node that is part of our pattern. For example in:

2314

// x = load ptr

2315

// y = x+4

2316

// store y -> ptr

2317

// This would happen when we're scanning down from the load and see the

2318

// store as a user. Record that there is a use of ChainedNode that is

2319

// part of the pattern and keep scanning uses.

2320

Result = CR_LeadsToInteriorNode;

2321

InteriorChainedNodes.push_back(User);

2322

continue;

2323

}

2324

2325

// If we found a TokenFactor, there are two cases to consider: first if the

2326

// TokenFactor is just hanging "below" the pattern we're matching (i.e. no

2327

// uses of the TF are in our pattern) we just want to ignore it. Second,

2328

// the TokenFactor can be sandwiched in between two chained nodes, like so:

2329

// [Load chain]

2330

// ^

2331

// |

2332

// [Load]

2333

// ^ ^

2334

// | \ DAG's like cheese

2335

// / \ do you?

2336

// / |

2337

// [TokenFactor] [Op]

2338

// ^ ^

2339

// | |

2340

// \ /

2341

// \ /

2342

// [Store]

2343

2344

// In this case, the TokenFactor becomes part of our match and we rewrite it

2345

// as a new TokenFactor.

2346

2347

// To distinguish these two cases, do a recursive walk down the uses.

2348

auto MemoizeResult = TokenFactorResult.find(User);

2349

bool Visited = MemoizeResult != TokenFactorResult.end();

2350

// Recursively walk chain users only if the result is not memoized.

2351

if (!Visited) {

2352

auto Res = WalkChainUsers(User, ChainedNodesInPattern, TokenFactorResult,

2353

InteriorChainedNodes);

2354

MemoizeResult = TokenFactorResult.insert(std::make_pair(User, Res)).first;

2355

}

2356

switch (MemoizeResult->second) {

2357

case CR_Simple:

2358

// If the uses of the TokenFactor are just already-selected nodes, ignore

2359

// it, it is "below" our pattern.

2360

continue;

2361

case CR_InducesCycle:

2362

// If the uses of the TokenFactor lead to nodes that are not part of our

2363

// pattern that are not selected, folding would turn this into a cycle,

2364

// bail out now.

2365

return CR_InducesCycle;

2366

case CR_LeadsToInteriorNode:

2367

break; // Otherwise, keep processing.

2368

}

2369

2370

// Okay, we know we're in the interesting interior case. The TokenFactor

2371

// is now going to be considered part of the pattern so that we rewrite its

2372

// uses (it may have uses that are not part of the pattern) with the

2373

// ultimate chain result of the generated code. We will also add its chain

2374

// inputs as inputs to the ultimate TokenFactor we create.

2375

Result = CR_LeadsToInteriorNode;

2376

if (!Visited) {

2377

ChainedNodesInPattern.push_back(User);

2378

InteriorChainedNodes.push_back(User);

2379

}

2380

}

2381

2382

return Result;

2383

}

2384

2385

/// HandleMergeInputChains - This implements the OPC_EmitMergeInputChains

2386

/// operation for when the pattern matched at least one node with a chains. The

2387

/// input vector contains a list of all of the chained nodes that we match. We

2388

/// must determine if this is a valid thing to cover (i.e. matching it won't

2389

/// induce cycles in the DAG) and if so, creating a TokenFactor node. that will

2390

/// be used as the input node chain for the generated nodes.

2391

static SDValue

2392

HandleMergeInputChains(SmallVectorImpl<SDNode*> &ChainNodesMatched,

2393

SelectionDAG *CurDAG) {

2394

// Used for memoization. Without it WalkChainUsers could take exponential

2395

// time to run.

2396

DenseMap<const SDNode *, ChainResult> TokenFactorResult;

2397

// Walk all of the chained nodes we've matched, recursively scanning down the

2398

// users of the chain result. This adds any TokenFactor nodes that are caught

2399

// in between chained nodes to the chained and interior nodes list.

2400

SmallVector<SDNode*, 3> InteriorChainedNodes;

2401

for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {

2402

if (WalkChainUsers(ChainNodesMatched[i], ChainNodesMatched,

2403

TokenFactorResult,

2404

InteriorChainedNodes) == CR_InducesCycle)

2405

return SDValue(); // Would induce a cycle.

2406

}

2407

2408

// Okay, we have walked all the matched nodes and collected TokenFactor nodes

2409

// that we are interested in. Form our input TokenFactor node.

2410

SmallVector<SDValue, 3> InputChains;

2411

for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {

2412

// Add the input chain of this node to the InputChains list (which will be

2413

// the operands of the generated TokenFactor) if it's not an interior node.

2414

SDNode *N = ChainNodesMatched[i];

2415

if (N->getOpcode() != ISD::TokenFactor) {

2416

if (std::count(InteriorChainedNodes.begin(),InteriorChainedNodes.end(),N))

2417

continue;

2418

2419

// Otherwise, add the input chain.

2420

SDValue InChain = ChainNodesMatched[i]->getOperand(0);

2421

assert(InChain.getValueType() == MVT::Other && "Not a chain")((InChain.getValueType() == MVT::Other && "Not a chain"
) ? static_cast<void> (0) : __assert_fail ("InChain.getValueType() == MVT::Other && \"Not a chain\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2421, __PRETTY_FUNCTION__));

2422

InputChains.push_back(InChain);

2423

continue;

2424

}

2425

2426

// If we have a token factor, we want to add all inputs of the token factor

2427

// that are not part of the pattern we're matching.

2428

for (const SDValue &Op : N->op_values()) {

2429

if (!std::count(ChainNodesMatched.begin(), ChainNodesMatched.end(),

2430

Op.getNode()))

2431

InputChains.push_back(Op);

2432

}

2433

}

2434

2435

if (InputChains.size() == 1)

2436

return InputChains[0];

2437

return CurDAG->getNode(ISD::TokenFactor, SDLoc(ChainNodesMatched[0]),

2438

MVT::Other, InputChains);

2439

}

2440

2441

/// MorphNode - Handle morphing a node in place for the selector.

2442

SDNode *SelectionDAGISel::

2443

MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,

2444

ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) {

2445

// It is possible we're using MorphNodeTo to replace a node with no

2446

// normal results with one that has a normal result (or we could be

2447

// adding a chain) and the input could have glue and chains as well.

2448

// In this case we need to shift the operands down.

2449

// FIXME: This is a horrible hack and broken in obscure cases, no worse

2450

// than the old isel though.

2451

int OldGlueResultNo = -1, OldChainResultNo = -1;

2452

2453

unsigned NTMNumResults = Node->getNumValues();

2454

if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {

2455

OldGlueResultNo = NTMNumResults-1;

2456

if (NTMNumResults != 1 &&

2457

Node->getValueType(NTMNumResults-2) == MVT::Other)

2458

OldChainResultNo = NTMNumResults-2;

2459

} else if (Node->getValueType(NTMNumResults-1) == MVT::Other)

2460

OldChainResultNo = NTMNumResults-1;

2461

2462

// Call the underlying SelectionDAG routine to do the transmogrification. Note

2463

// that this deletes operands of the old node that become dead.

2464

SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);

2465

2466

// MorphNodeTo can operate in two ways: if an existing node with the

2467

// specified operands exists, it can just return it. Otherwise, it

2468

// updates the node in place to have the requested operands.

2469

if (Res == Node) {

2470

// If we updated the node in place, reset the node ID. To the isel,

2471

// this should be just like a newly allocated machine node.

2472

Res->setNodeId(-1);

2473

}

2474

2475

unsigned ResNumResults = Res->getNumValues();

2476

// Move the glue if needed.

2477

if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&

2478

(unsigned)OldGlueResultNo != ResNumResults-1)

2479

CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldGlueResultNo),

2480

SDValue(Res, ResNumResults-1));

2481

2482

if ((EmitNodeInfo & OPFL_GlueOutput) != 0)

2483

--ResNumResults;

2484

2485

// Move the chain reference if needed.

2486

if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&

2487

(unsigned)OldChainResultNo != ResNumResults-1)

2488

CurDAG->ReplaceAllUsesOfValueWith(SDValue(Node, OldChainResultNo),

2489

SDValue(Res, ResNumResults-1));

2490

2491

// Otherwise, no replacement happened because the node already exists. Replace

2492

// Uses of the old node with the new one.

2493

if (Res != Node) {

2494

CurDAG->ReplaceAllUsesWith(Node, Res);

2495

CurDAG->RemoveDeadNode(Node);

2496

}

2497

2498

return Res;

2499

}

2500

2501

/// CheckSame - Implements OP_CheckSame.

2502

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2503

CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2504

SDValue N,

2505

const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {

2506

// Accept if it is exactly the same as a previously recorded node.

2507

unsigned RecNo = MatcherTable[MatcherIndex++];

2508

assert(RecNo < RecordedNodes.size() && "Invalid CheckSame")((RecNo < RecordedNodes.size() && "Invalid CheckSame"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid CheckSame\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2508, __PRETTY_FUNCTION__));

2509

return N == RecordedNodes[RecNo].first;

2510

}

2511

2512

/// CheckChildSame - Implements OP_CheckChildXSame.

2513

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2514

CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2515

SDValue N,

2516

const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes,

2517

unsigned ChildNo) {

2518

if (ChildNo >= N.getNumOperands())

2519

return false; // Match fails if out of range child #.

2520

return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),

2521

RecordedNodes);

2522

}

2523

2524

/// CheckPatternPredicate - Implements OP_CheckPatternPredicate.

2525

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2526

CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2527

const SelectionDAGISel &SDISel) {

2528

return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);

2529

}

2530

2531

/// CheckNodePredicate - Implements OP_CheckNodePredicate.

2532

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2533

CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2534

const SelectionDAGISel &SDISel, SDNode *N) {

2535

return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);

2536

}

2537

2538

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2539

CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2540

SDNode *N) {

2541

uint16_t Opc = MatcherTable[MatcherIndex++];

2542

Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;

2543

return N->getOpcode() == Opc;

2544

}

2545

2546

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2547

CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,

2548

const TargetLowering *TLI, const DataLayout &DL) {

2549

MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];

2550

if (N.getValueType() == VT) return true;

2551

2552

// Handle the case when VT is iPTR.

2553

return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);

2554

}

2555

2556

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2557

CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2558

SDValue N, const TargetLowering *TLI, const DataLayout &DL,

2559

unsigned ChildNo) {

2560

if (ChildNo >= N.getNumOperands())

2561

return false; // Match fails if out of range child #.

2562

return ::CheckType(MatcherTable, MatcherIndex, N.getOperand(ChildNo), TLI,

2563

DL);

2564

}

2565

2566

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2567

CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2568

SDValue N) {

2569

return cast<CondCodeSDNode>(N)->get() ==

2570

(ISD::CondCode)MatcherTable[MatcherIndex++];

2571

}

2572

2573

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2574

CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2575

SDValue N, const TargetLowering *TLI, const DataLayout &DL) {

2576

MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];

2577

if (cast<VTSDNode>(N)->getVT() == VT)

2578

return true;

2579

2580

// Handle the case when VT is iPTR.

2581

return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);

2582

}

2583

2584

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2585

CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2586

SDValue N) {

2587

int64_t Val = MatcherTable[MatcherIndex++];

2588

if (Val & 128)

2589

Val = GetVBR(Val, MatcherTable, MatcherIndex);

2590

2591

ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);

2592

return C && C->getSExtValue() == Val;

2593

}

2594

2595

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2596

CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2597

SDValue N, unsigned ChildNo) {

2598

if (ChildNo >= N.getNumOperands())

2599

return false; // Match fails if out of range child #.

2600

return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));

2601

}

2602

2603

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2604

CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2605

SDValue N, const SelectionDAGISel &SDISel) {

2606

int64_t Val = MatcherTable[MatcherIndex++];

2607

if (Val & 128)

2608

Val = GetVBR(Val, MatcherTable, MatcherIndex);

2609

2610

if (N->getOpcode() != ISD::AND) return false;

2611

2612

ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));

2613

return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);

2614

}

2615

2616

LLVM_ATTRIBUTE_ALWAYS_INLINE__attribute__((always_inline)) static inline bool

2617

CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,

2618

SDValue N, const SelectionDAGISel &SDISel) {

2619

int64_t Val = MatcherTable[MatcherIndex++];

2620

if (Val & 128)

2621

Val = GetVBR(Val, MatcherTable, MatcherIndex);

2622

2623

if (N->getOpcode() != ISD::OR) return false;

2624

2625

ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));

2626

return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);

2627

}

2628

2629

/// IsPredicateKnownToFail - If we know how and can do so without pushing a

2630

/// scope, evaluate the current node. If the current predicate is known to

2631

/// fail, set Result=true and return anything. If the current predicate is

2632

/// known to pass, set Result=false and return the MatcherIndex to continue

2633

/// with. If the current predicate is unknown, set Result=false and return the

2634

/// MatcherIndex to continue with.

2635

static unsigned IsPredicateKnownToFail(const unsigned char *Table,

2636

unsigned Index, SDValue N,

2637

bool &Result,

2638

const SelectionDAGISel &SDISel,

2639

SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {

2640

switch (Table[Index++]) {

2641

default:

2642

Result = false;

2643

return Index-1; // Could not evaluate this predicate.

2644

case SelectionDAGISel::OPC_CheckSame:

2645

Result = !::CheckSame(Table, Index, N, RecordedNodes);

2646

return Index;

2647

case SelectionDAGISel::OPC_CheckChild0Same:

2648

case SelectionDAGISel::OPC_CheckChild1Same:

2649

case SelectionDAGISel::OPC_CheckChild2Same:

2650

case SelectionDAGISel::OPC_CheckChild3Same:

2651

Result = !::CheckChildSame(Table, Index, N, RecordedNodes,

2652

Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Same);

2653

return Index;

2654

case SelectionDAGISel::OPC_CheckPatternPredicate:

2655

Result = !::CheckPatternPredicate(Table, Index, SDISel);

2656

return Index;

2657

case SelectionDAGISel::OPC_CheckPredicate:

2658

Result = !::CheckNodePredicate(Table, Index, SDISel, N.getNode());

2659

return Index;

2660

case SelectionDAGISel::OPC_CheckOpcode:

2661

Result = !::CheckOpcode(Table, Index, N.getNode());

2662

return Index;

2663

case SelectionDAGISel::OPC_CheckType:

2664

Result = !::CheckType(Table, Index, N, SDISel.TLI,

2665

SDISel.CurDAG->getDataLayout());

2666

return Index;

2667

case SelectionDAGISel::OPC_CheckChild0Type:

2668

case SelectionDAGISel::OPC_CheckChild1Type:

2669

case SelectionDAGISel::OPC_CheckChild2Type:

2670

case SelectionDAGISel::OPC_CheckChild3Type:

2671

case SelectionDAGISel::OPC_CheckChild4Type:

2672

case SelectionDAGISel::OPC_CheckChild5Type:

2673

case SelectionDAGISel::OPC_CheckChild6Type:

2674

case SelectionDAGISel::OPC_CheckChild7Type:

2675

Result = !::CheckChildType(

2676

Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout(),

2677

Table[Index - 1] - SelectionDAGISel::OPC_CheckChild0Type);

2678

return Index;

2679

case SelectionDAGISel::OPC_CheckCondCode:

2680

Result = !::CheckCondCode(Table, Index, N);

2681

return Index;

2682

case SelectionDAGISel::OPC_CheckValueType:

2683

Result = !::CheckValueType(Table, Index, N, SDISel.TLI,

2684

SDISel.CurDAG->getDataLayout());

2685

return Index;

2686

case SelectionDAGISel::OPC_CheckInteger:

2687

Result = !::CheckInteger(Table, Index, N);

2688

return Index;

2689

case SelectionDAGISel::OPC_CheckChild0Integer:

2690

case SelectionDAGISel::OPC_CheckChild1Integer:

2691

case SelectionDAGISel::OPC_CheckChild2Integer:

2692

case SelectionDAGISel::OPC_CheckChild3Integer:

2693

case SelectionDAGISel::OPC_CheckChild4Integer:

2694

Result = !::CheckChildInteger(Table, Index, N,

2695

Table[Index-1] - SelectionDAGISel::OPC_CheckChild0Integer);

2696

return Index;

2697

case SelectionDAGISel::OPC_CheckAndImm:

2698

Result = !::CheckAndImm(Table, Index, N, SDISel);

2699

return Index;

2700

case SelectionDAGISel::OPC_CheckOrImm:

2701

Result = !::CheckOrImm(Table, Index, N, SDISel);

2702

return Index;

2703

}

2704

}

2705

2706

namespace {

2707

2708

struct MatchScope {

2709

/// FailIndex - If this match fails, this is the index to continue with.

2710

unsigned FailIndex;

2711

2712

/// NodeStack - The node stack when the scope was formed.

2713

SmallVector<SDValue, 4> NodeStack;

2714

2715

/// NumRecordedNodes - The number of recorded nodes when the scope was formed.

2716

unsigned NumRecordedNodes;

2717

2718

/// NumMatchedMemRefs - The number of matched memref entries.

2719

unsigned NumMatchedMemRefs;

2720

2721

/// InputChain/InputGlue - The current chain/glue

2722

SDValue InputChain, InputGlue;

2723

2724

/// HasChainNodesMatched - True if the ChainNodesMatched list is non-empty.

2725

bool HasChainNodesMatched;

2726

};

2727

2728

/// \\brief A DAG update listener to keep the matching state

2729

/// (i.e. RecordedNodes and MatchScope) uptodate if the target is allowed to

2730

/// change the DAG while matching. X86 addressing mode matcher is an example

2731

/// for this.

2732

class MatchStateUpdater : public SelectionDAG::DAGUpdateListener

2733

{

2734

SDNode **NodeToMatch;

2735

SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes;

2736

SmallVectorImpl<MatchScope> &MatchScopes;

2737

2738

public:

2739

MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch,

2740

SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,

2741

SmallVectorImpl<MatchScope> &MS)

2742

: SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),

2743

RecordedNodes(RN), MatchScopes(MS) {}

2744

2745

void NodeDeleted(SDNode *N, SDNode *E) override {

2746

// Some early-returns here to avoid the search if we deleted the node or

2747

// if the update comes from MorphNodeTo (MorphNodeTo is the last thing we

2748

// do, so it's unnecessary to update matching state at that point).

2749

// Neither of these can occur currently because we only install this

2750

// update listener during matching a complex patterns.

2751

if (!E || E->isMachineOpcode())

2752

return;

2753

// Check if NodeToMatch was updated.

2754

if (N == *NodeToMatch)

2755

*NodeToMatch = E;

2756

// Performing linear search here does not matter because we almost never

2757

// run this code. You'd have to have a CSE during complex pattern

2758

// matching.

2759

for (auto &I : RecordedNodes)

2760

if (I.first.getNode() == N)

2761

I.first.setNode(E);

2762

2763

for (auto &I : MatchScopes)

2764

for (auto &J : I.NodeStack)

2765

if (J.getNode() == N)

2766

J.setNode(E);

2767

}

2768

};

2769

2770

} // end anonymous namespace

2771

2772

void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,

2773

const unsigned char *MatcherTable,

2774

unsigned TableSize) {

2775

// FIXME: Should these even be selected? Handle these cases in the caller?

2776

switch (NodeToMatch->getOpcode()) {

2777

default:

2778

break;

2779

case ISD::EntryToken: // These nodes remain the same.

2780

case ISD::BasicBlock:

2781

case ISD::Register:

2782

case ISD::RegisterMask:

2783

case ISD::HANDLENODE:

2784

case ISD::MDNODE_SDNODE:

2785

case ISD::TargetConstant:

2786

case ISD::TargetConstantFP:

2787

case ISD::TargetConstantPool:

2788

case ISD::TargetFrameIndex:

2789

case ISD::TargetExternalSymbol:

2790

case ISD::MCSymbol:

2791

case ISD::TargetBlockAddress:

2792

case ISD::TargetJumpTable:

2793

case ISD::TargetGlobalTLSAddress:

2794

case ISD::TargetGlobalAddress:

2795

case ISD::TokenFactor:

2796

case ISD::CopyFromReg:

2797

case ISD::CopyToReg:

2798

case ISD::EH_LABEL:

2799

case ISD::LIFETIME_START:

2800

case ISD::LIFETIME_END:

2801

NodeToMatch->setNodeId(-1); // Mark selected.

2802

return;

2803

case ISD::AssertSext:

2804

case ISD::AssertZext:

2805

CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, 0),

2806

NodeToMatch->getOperand(0));

2807

CurDAG->RemoveDeadNode(NodeToMatch);

2808

return;

2809

case ISD::INLINEASM:

2810

Select_INLINEASM(NodeToMatch);

2811

return;

2812

case ISD::READ_REGISTER:

2813

Select_READ_REGISTER(NodeToMatch);

2814

return;

2815

case ISD::WRITE_REGISTER:

2816

Select_WRITE_REGISTER(NodeToMatch);

2817

return;

2818

case ISD::UNDEF:

2819

Select_UNDEF(NodeToMatch);

2820

return;

2821

}

2822

2823

assert(!NodeToMatch->isMachineOpcode() && "Node already selected!")((!NodeToMatch->isMachineOpcode() && "Node already selected!"
) ? static_cast<void> (0) : __assert_fail ("!NodeToMatch->isMachineOpcode() && \"Node already selected!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2823, __PRETTY_FUNCTION__));

2824

2825

// Set up the node stack with NodeToMatch as the only node on the stack.

2826

SmallVector<SDValue, 8> NodeStack;

2827

SDValue N = SDValue(NodeToMatch, 0);

2828

NodeStack.push_back(N);

2829

2830

// MatchScopes - Scopes used when matching, if a match failure happens, this

2831

// indicates where to continue checking.

2832

SmallVector<MatchScope, 8> MatchScopes;

2833

2834

// RecordedNodes - This is the set of nodes that have been recorded by the

2835

// state machine. The second value is the parent of the node, or null if the

2836

// root is recorded.

2837

SmallVector<std::pair<SDValue, SDNode*>, 8> RecordedNodes;

2838

2839

// MatchedMemRefs - This is the set of MemRef's we've seen in the input

2840

// pattern.

2841

SmallVector<MachineMemOperand*, 2> MatchedMemRefs;

2842

2843

// These are the current input chain and glue for use when generating nodes.

2844

// Various Emit operations change these. For example, emitting a copytoreg

2845

// uses and updates these.

2846

SDValue InputChain, InputGlue;

2847

2848

// ChainNodesMatched - If a pattern matches nodes that have input/output

2849

// chains, the OPC_EmitMergeInputChains operation is emitted which indicates

2850

// which ones they are. The result is captured into this list so that we can

2851

// update the chain results when the pattern is complete.

2852

SmallVector<SDNode*, 3> ChainNodesMatched;

2853

2854

DEBUG(dbgs() << "ISEL: Starting pattern match on root node: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Starting pattern match on root node: "
; NodeToMatch->dump(CurDAG); dbgs() << '\n'; } } while
(false)

2855

NodeToMatch->dump(CurDAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Starting pattern match on root node: "
; NodeToMatch->dump(CurDAG); dbgs() << '\n'; } } while
(false)

2856

dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "ISEL: Starting pattern match on root node: "
; NodeToMatch->dump(CurDAG); dbgs() << '\n'; } } while
(false);

2857

2858

// Determine where to start the interpreter. Normally we start at opcode #0,

2859

// but if the state machine starts with an OPC_SwitchOpcode, then we

2860

// accelerate the first lookup (which is guaranteed to be hot) with the

2861

// OpcodeOffset table.

2862

unsigned MatcherIndex = 0;

2863

2864

if (!OpcodeOffset.empty()) {

2865

// Already computed the OpcodeOffset table, just index into it.

2866

if (N.getOpcode() < OpcodeOffset.size())

2867

MatcherIndex = OpcodeOffset[N.getOpcode()];

2868

DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Initial Opcode index to " <<
MatcherIndex << "\n"; } } while (false);

2869

2870

} else if (MatcherTable[0] == OPC_SwitchOpcode) {

2871

// Otherwise, the table isn't computed, but the state machine does start

2872

// with an OPC_SwitchOpcode instruction. Populate the table now, since this

2873

// is the first time we're selecting an instruction.

2874

unsigned Idx = 1;

2875

while (true) {

2876

// Get the size of this case.

2877

unsigned CaseSize = MatcherTable[Idx++];

2878

if (CaseSize & 128)

2879

CaseSize = GetVBR(CaseSize, MatcherTable, Idx);

2880

if (CaseSize == 0) break;

2881

2882

// Get the opcode, add the index to the table.

2883

uint16_t Opc = MatcherTable[Idx++];

2884

Opc |= (unsigned short)MatcherTable[Idx++] << 8;

2885

if (Opc >= OpcodeOffset.size())

2886

OpcodeOffset.resize((Opc+1)*2);

2887

OpcodeOffset[Opc] = Idx;

2888

Idx += CaseSize;

2889

}

2890

2891

// Okay, do the lookup for the first opcode.

2892

if (N.getOpcode() < OpcodeOffset.size())

2893

MatcherIndex = OpcodeOffset[N.getOpcode()];

2894

}

2895

2896

while (true) {

2897

assert(MatcherIndex < TableSize && "Invalid index")((MatcherIndex < TableSize && "Invalid index") ? static_cast
<void> (0) : __assert_fail ("MatcherIndex < TableSize && \"Invalid index\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 2897, __PRETTY_FUNCTION__));

2898

#ifndef NDEBUG

2899

unsigned CurrentOpcodeIndex = MatcherIndex;

2900

#endif

2901

BuiltinOpcodes Opcode = (BuiltinOpcodes)MatcherTable[MatcherIndex++];

2902

switch (Opcode) {

2903

case OPC_Scope: {

2904

// Okay, the semantics of this operation are that we should push a scope

2905

// then evaluate the first child. However, pushing a scope only to have

2906

// the first check fail (which then pops it) is inefficient. If we can

2907

// determine immediately that the first check (or first several) will

2908

// immediately fail, don't even bother pushing a scope for them.

2909

unsigned FailIndex;

2910

2911

while (true) {

2912

unsigned NumToSkip = MatcherTable[MatcherIndex++];

2913

if (NumToSkip & 128)

2914

NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);

2915

// Found the end of the scope with no match.

2916

if (NumToSkip == 0) {

2917

FailIndex = 0;

2918

break;

2919

}

2920

2921

FailIndex = MatcherIndex+NumToSkip;

2922

2923

unsigned MatcherIndexOfPredicate = MatcherIndex;

2924

(void)MatcherIndexOfPredicate; // silence warning.

2925

2926

// If we can't evaluate this predicate without pushing a scope (e.g. if

2927

// it is a 'MoveParent') or if the predicate succeeds on this node, we

2928

// push the scope and evaluate the full predicate chain.

2929

bool Result;

2930

MatcherIndex = IsPredicateKnownToFail(MatcherTable, MatcherIndex, N,

2931

Result, *this, RecordedNodes);

2932

if (!Result)

2933

break;

2934

2935

DEBUG(dbgs() << " Skipped scope entry (due to false predicate) at "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false)

2936

<< "index " << MatcherIndexOfPredicatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false)

2937

<< ", continuing at " << FailIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Skipped scope entry (due to false predicate) at "
<< "index " << MatcherIndexOfPredicate << ", continuing at "
<< FailIndex << "\n"; } } while (false);

2938

++NumDAGIselRetries;

2939

2940

// Otherwise, we know that this case of the Scope is guaranteed to fail,

2941

// move to the next case.

2942

MatcherIndex = FailIndex;

2943

}

2944

2945

// If the whole scope failed to match, bail.

2946

if (FailIndex == 0) break;

2947

2948

// Push a MatchScope which indicates where to go if the first child fails

2949

// to match.

2950

MatchScope NewEntry;

2951

NewEntry.FailIndex = FailIndex;

2952

NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());

2953

NewEntry.NumRecordedNodes = RecordedNodes.size();

2954

NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();

2955

NewEntry.InputChain = InputChain;

2956

NewEntry.InputGlue = InputGlue;

2957

NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();

2958

MatchScopes.push_back(NewEntry);

2959

continue;

2960

}

2961

case OPC_RecordNode: {

2962

// Remember this node, it may end up being an operand in the pattern.

2963

SDNode *Parent = nullptr;

2964

if (NodeStack.size() > 1)

2965

Parent = NodeStack[NodeStack.size()-2].getNode();

2966

RecordedNodes.push_back(std::make_pair(N, Parent));

2967

continue;

2968

}

2969

2970

case OPC_RecordChild0: case OPC_RecordChild1:

2971

case OPC_RecordChild2: case OPC_RecordChild3:

2972

case OPC_RecordChild4: case OPC_RecordChild5:

2973

case OPC_RecordChild6: case OPC_RecordChild7: {

2974

unsigned ChildNo = Opcode-OPC_RecordChild0;

2975

if (ChildNo >= N.getNumOperands())

2976

break; // Match fails if out of range child #.

2977

2978

RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),

2979

N.getNode()));

2980

continue;

2981

}

2982

case OPC_RecordMemRef:

2983

MatchedMemRefs.push_back(cast<MemSDNode>(N)->getMemOperand());

2984

continue;

2985

2986

case OPC_CaptureGlueInput:

2987

// If the current node has an input glue, capture it in InputGlue.

2988

if (N->getNumOperands() != 0 &&

2989

N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)

2990

InputGlue = N->getOperand(N->getNumOperands()-1);

2991

continue;

2992

2993

case OPC_MoveChild: {

2994

unsigned ChildNo = MatcherTable[MatcherIndex++];

2995

if (ChildNo >= N.getNumOperands())

2996

break; // Match fails if out of range child #.

2997

N = N.getOperand(ChildNo);

2998

NodeStack.push_back(N);

2999

continue;

3000

}

3001

3002

case OPC_MoveChild0: case OPC_MoveChild1:

3003

case OPC_MoveChild2: case OPC_MoveChild3:

3004

case OPC_MoveChild4: case OPC_MoveChild5:

3005

case OPC_MoveChild6: case OPC_MoveChild7: {

3006

unsigned ChildNo = Opcode-OPC_MoveChild0;

3007

if (ChildNo >= N.getNumOperands())

3008

break; // Match fails if out of range child #.

3009

N = N.getOperand(ChildNo);

3010

NodeStack.push_back(N);

3011

continue;

3012

}

3013

3014

case OPC_MoveParent:

3015

// Pop the current node off the NodeStack.

3016

NodeStack.pop_back();

3017

assert(!NodeStack.empty() && "Node stack imbalance!")((!NodeStack.empty() && "Node stack imbalance!") ? static_cast
<void> (0) : __assert_fail ("!NodeStack.empty() && \"Node stack imbalance!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3017, __PRETTY_FUNCTION__));

3018

N = NodeStack.back();

3019

continue;

3020

3021

case OPC_CheckSame:

3022

if (!::CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes)) break;

3023

continue;

3024

3025

case OPC_CheckChild0Same: case OPC_CheckChild1Same:

3026

case OPC_CheckChild2Same: case OPC_CheckChild3Same:

3027

if (!::CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,

3028

Opcode-OPC_CheckChild0Same))

3029

break;

3030

continue;

3031

3032

case OPC_CheckPatternPredicate:

3033

if (!::CheckPatternPredicate(MatcherTable, MatcherIndex, *this)) break;

3034

continue;

3035

case OPC_CheckPredicate:

3036

if (!::CheckNodePredicate(MatcherTable, MatcherIndex, *this,

3037

N.getNode()))

3038

break;

3039

continue;

3040

case OPC_CheckComplexPat: {

3041

unsigned CPNum = MatcherTable[MatcherIndex++];

3042

unsigned RecNo = MatcherTable[MatcherIndex++];

3043

assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat")((RecNo < RecordedNodes.size() && "Invalid CheckComplexPat"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid CheckComplexPat\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3043, __PRETTY_FUNCTION__));

3044

3045

// If target can modify DAG during matching, keep the matching state

3046

// consistent.

3047

std::unique_ptr<MatchStateUpdater> MSU;

3048

if (ComplexPatternFuncMutatesDAG())

3049

MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes,

3050

MatchScopes));

3051

3052

if (!CheckComplexPattern(NodeToMatch, RecordedNodes[RecNo].second,

3053

RecordedNodes[RecNo].first, CPNum,

3054

RecordedNodes))

3055

break;

3056

continue;

3057

}

3058

case OPC_CheckOpcode:

3059

if (!::CheckOpcode(MatcherTable, MatcherIndex, N.getNode())) break;

3060

continue;

3061

3062

case OPC_CheckType:

3063

if (!::CheckType(MatcherTable, MatcherIndex, N, TLI,

3064

CurDAG->getDataLayout()))

3065

break;

3066

continue;

3067

3068

case OPC_SwitchOpcode: {

3069

unsigned CurNodeOpcode = N.getOpcode();

3070

unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;

3071

unsigned CaseSize;

3072

while (true) {

3073

// Get the size of this case.

3074

CaseSize = MatcherTable[MatcherIndex++];

3075

if (CaseSize & 128)

3076

CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);

3077

if (CaseSize == 0) break;

3078

3079

uint16_t Opc = MatcherTable[MatcherIndex++];

3080

Opc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;

3081

3082

// If the opcode matches, then we will execute this case.

3083

if (CurNodeOpcode == Opc)

3084

break;

3085

3086

// Otherwise, skip over this case.

3087

MatcherIndex += CaseSize;

3088

}

3089

3090

// If no cases matched, bail out.

3091

if (CaseSize == 0) break;

3092

3093

// Otherwise, execute the case we found.

3094

DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStartdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " OpcodeSwitch from " << SwitchStart
<< " to " << MatcherIndex << "\n"; } } while
(false)

3095

<< " to " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " OpcodeSwitch from " << SwitchStart
<< " to " << MatcherIndex << "\n"; } } while
(false);

3096

continue;

3097

}

3098

3099

case OPC_SwitchType: {

3100

MVT CurNodeVT = N.getSimpleValueType();

3101

unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;

3102

unsigned CaseSize;

3103

while (true) {

3104

// Get the size of this case.

3105

CaseSize = MatcherTable[MatcherIndex++];

3106

if (CaseSize & 128)

3107

CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);

3108

if (CaseSize == 0) break;

3109

3110

MVT CaseVT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];

3111

if (CaseVT == MVT::iPTR)

3112

CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());

3113

3114

// If the VT matches, then we will execute this case.

3115

if (CurNodeVT == CaseVT)

3116

break;

3117

3118

// Otherwise, skip over this case.

3119

MatcherIndex += CaseSize;

3120

}

3121

3122

// If no cases matched, bail out.

3123

if (CaseSize == 0) break;

3124

3125

// Otherwise, execute the case we found.

3126

DEBUG(dbgs() << " TypeSwitch[" << EVT(CurNodeVT).getEVTString()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT
).getEVTString() << "] from " << SwitchStart <<
" to " << MatcherIndex<<'\n'; } } while (false)

3127

<< "] from " << SwitchStart << " to " << MatcherIndex<<'\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " TypeSwitch[" << EVT(CurNodeVT
).getEVTString() << "] from " << SwitchStart <<
" to " << MatcherIndex<<'\n'; } } while (false);

3128

continue;

3129

}

3130

case OPC_CheckChild0Type: case OPC_CheckChild1Type:

3131

case OPC_CheckChild2Type: case OPC_CheckChild3Type:

3132

case OPC_CheckChild4Type: case OPC_CheckChild5Type:

3133

case OPC_CheckChild6Type: case OPC_CheckChild7Type:

3134

if (!::CheckChildType(MatcherTable, MatcherIndex, N, TLI,

3135

CurDAG->getDataLayout(),

3136

Opcode - OPC_CheckChild0Type))

3137

break;

3138

continue;

3139

case OPC_CheckCondCode:

3140

if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break;

3141

continue;

3142

case OPC_CheckValueType:

3143

if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI,

3144

CurDAG->getDataLayout()))

3145

break;

3146

continue;

3147

case OPC_CheckInteger:

3148

if (!::CheckInteger(MatcherTable, MatcherIndex, N)) break;

3149

continue;

3150

case OPC_CheckChild0Integer: case OPC_CheckChild1Integer:

3151

case OPC_CheckChild2Integer: case OPC_CheckChild3Integer:

3152

case OPC_CheckChild4Integer:

3153

if (!::CheckChildInteger(MatcherTable, MatcherIndex, N,

3154

Opcode-OPC_CheckChild0Integer)) break;

3155

continue;

3156

case OPC_CheckAndImm:

3157

if (!::CheckAndImm(MatcherTable, MatcherIndex, N, *this)) break;

3158

continue;

3159

case OPC_CheckOrImm:

3160

if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break;

3161

continue;

3162

3163

case OPC_CheckFoldableChainNode: {

3164

assert(NodeStack.size() != 1 && "No parent node")((NodeStack.size() != 1 && "No parent node") ? static_cast
<void> (0) : __assert_fail ("NodeStack.size() != 1 && \"No parent node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3164, __PRETTY_FUNCTION__));

3165

// Verify that all intermediate nodes between the root and this one have

3166

// a single use.

3167

bool HasMultipleUses = false;

3168

for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i)

3169

if (!NodeStack[i].getNode()->hasOneUse()) {

3170

HasMultipleUses = true;

3171

break;

3172

}

3173

if (HasMultipleUses) break;

3174

3175

// Check to see that the target thinks this is profitable to fold and that

3176

// we can fold it without inducing cycles in the graph.

3177

if (!IsProfitableToFold(N, NodeStack[NodeStack.size()-2].getNode(),

3178

NodeToMatch) ||

3179

!IsLegalToFold(N, NodeStack[NodeStack.size()-2].getNode(),

3180

NodeToMatch, OptLevel,

3181

true/*We validate our own chains*/))

3182

break;

3183

3184

continue;

3185

}

3186

case OPC_EmitInteger: {

3187

MVT::SimpleValueType VT =

3188

(MVT::SimpleValueType)MatcherTable[MatcherIndex++];

3189

int64_t Val = MatcherTable[MatcherIndex++];

3190

if (Val & 128)

3191

Val = GetVBR(Val, MatcherTable, MatcherIndex);

3192

RecordedNodes.push_back(std::pair<SDValue, SDNode*>(

3193

CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch),

3194

VT), nullptr));

3195

continue;

3196

}

3197

case OPC_EmitRegister: {

3198

MVT::SimpleValueType VT =

3199

(MVT::SimpleValueType)MatcherTable[MatcherIndex++];

3200

unsigned RegNo = MatcherTable[MatcherIndex++];

3201

RecordedNodes.push_back(std::pair<SDValue, SDNode*>(

3202

CurDAG->getRegister(RegNo, VT), nullptr));

3203

continue;

3204

}

3205

case OPC_EmitRegister2: {

3206

// For targets w/ more than 256 register names, the register enum

3207

// values are stored in two bytes in the matcher table (just like

3208

// opcodes).

3209

MVT::SimpleValueType VT =

3210

(MVT::SimpleValueType)MatcherTable[MatcherIndex++];

3211

unsigned RegNo = MatcherTable[MatcherIndex++];

3212

RegNo |= MatcherTable[MatcherIndex++] << 8;

3213

RecordedNodes.push_back(std::pair<SDValue, SDNode*>(

3214

CurDAG->getRegister(RegNo, VT), nullptr));

3215

continue;

3216

}

3217

3218

case OPC_EmitConvertToTarget: {

3219

// Convert from IMM/FPIMM to target version.

3220

unsigned RecNo = MatcherTable[MatcherIndex++];

3221

assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget")((RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitConvertToTarget\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3221, __PRETTY_FUNCTION__));

3222

SDValue Imm = RecordedNodes[RecNo].first;

3223

3224

if (Imm->getOpcode() == ISD::Constant) {

3225

const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();

3226

Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch),

3227

Imm.getValueType());

3228

} else if (Imm->getOpcode() == ISD::ConstantFP) {

3229

const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();

3230

Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch),

3231

Imm.getValueType());

3232

}

3233

3234

RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));

3235

continue;

3236

}

3237

3238

case OPC_EmitMergeInputChains1_0: // OPC_EmitMergeInputChains, 1, 0

3239

case OPC_EmitMergeInputChains1_1: // OPC_EmitMergeInputChains, 1, 1

3240

case OPC_EmitMergeInputChains1_2: { // OPC_EmitMergeInputChains, 1, 2

3241

// These are space-optimized forms of OPC_EmitMergeInputChains.

3242

assert(!InputChain.getNode() &&((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3243, __PRETTY_FUNCTION__))

3243

"EmitMergeInputChains should be the first chain producing node")((!InputChain.getNode() && "EmitMergeInputChains should be the first chain producing node"
) ? static_cast<void> (0) : __assert_fail ("!InputChain.getNode() && \"EmitMergeInputChains should be the first chain producing node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3243, __PRETTY_FUNCTION__));

3244

assert(ChainNodesMatched.empty() &&((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3245, __PRETTY_FUNCTION__))

3245

"Should only have one EmitMergeInputChains per match")((ChainNodesMatched.empty() && "Should only have one EmitMergeInputChains per match"
) ? static_cast<void> (0) : __assert_fail ("ChainNodesMatched.empty() && \"Should only have one EmitMergeInputChains per match\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3245, __PRETTY_FUNCTION__));

3246

3247

// Read all of the chained nodes.

3248

unsigned RecNo = Opcode - OPC_EmitMergeInputChains1_0;

3249

assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains")((RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitMergeInputChains\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3249, __PRETTY_FUNCTION__));

3250

ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());

3251

3252

// FIXME: What if other value results of the node have uses not matched

3253

// by this pattern?

3254

if (ChainNodesMatched.back() != NodeToMatch &&

3255

!RecordedNodes[RecNo].first.hasOneUse()) {

3256

ChainNodesMatched.clear();

3257

break;

3258

}

3259

3260

// Merge the input chains if they are not intra-pattern references.

3261

InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);

3262

3263

if (!InputChain.getNode())

3264

break; // Failed to merge.

3265

continue;

3266

}

3267

3268

case OPC_EmitMergeInputChains: {

3269

3270

3271

// This node gets a list of nodes we matched in the input that have

3272

// chains. We want to token factor all of the input chains to these nodes

3273

// together. However, if any of the input chains is actually one of the

3274

// nodes matched in this pattern, then we have an intra-match reference.

3275

// Ignore these because the newly token factored chain should not refer to

3276

// the old nodes.

3277

unsigned NumChains = MatcherTable[MatcherIndex++];

3278

assert(NumChains != 0 && "Can't TF zero chains")((NumChains != 0 && "Can't TF zero chains") ? static_cast
<void> (0) : __assert_fail ("NumChains != 0 && \"Can't TF zero chains\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3278, __PRETTY_FUNCTION__));

3279

3280

3281

3282

3283

// Read all of the chained nodes.

3284

for (unsigned i = 0; i != NumChains; ++i) {

3285

unsigned RecNo = MatcherTable[MatcherIndex++];

3286

3287

ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());

3288

3289

// FIXME: What if other value results of the node have uses not matched

3290

// by this pattern?

3291

if (ChainNodesMatched.back() != NodeToMatch &&

3292

!RecordedNodes[RecNo].first.hasOneUse()) {

3293

ChainNodesMatched.clear();

3294

break;

3295

}

3296

}

3297

3298

// If the inner loop broke out, the match fails.

3299

if (ChainNodesMatched.empty())

3300

break;

3301

3302

// Merge the input chains if they are not intra-pattern references.

3303

InputChain = HandleMergeInputChains(ChainNodesMatched, CurDAG);

3304

3305

if (!InputChain.getNode())

3306

break; // Failed to merge.

3307

3308

continue;

3309

}

3310

3311

case OPC_EmitCopyToReg: {

3312

unsigned RecNo = MatcherTable[MatcherIndex++];

3313

assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg")((RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitCopyToReg\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3313, __PRETTY_FUNCTION__));

3314

unsigned DestPhysReg = MatcherTable[MatcherIndex++];

3315

3316

if (!InputChain.getNode())

3317

InputChain = CurDAG->getEntryNode();

3318

3319

InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),

3320

DestPhysReg, RecordedNodes[RecNo].first,

3321

InputGlue);

3322

3323

InputGlue = InputChain.getValue(1);

3324

continue;

3325

}

3326

3327

case OPC_EmitNodeXForm: {

3328

unsigned XFormNo = MatcherTable[MatcherIndex++];

3329

unsigned RecNo = MatcherTable[MatcherIndex++];

3330

assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm")((RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitNodeXForm\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3330, __PRETTY_FUNCTION__));

3331

SDValue Res = RunSDNodeXForm(RecordedNodes[RecNo].first, XFormNo);

3332

RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));

3333

continue;

3334

}

3335

case OPC_Coverage: {

3336

// This is emitted right before MorphNode/EmitNode.

3337

// So it should be safe to assume that this node has been selected

3338

unsigned index = MatcherTable[MatcherIndex++];

3339

index |= (MatcherTable[MatcherIndex++] << 8);

3340

dbgs() << "COVERED: " << getPatternForIndex(index) << "\n";

3341

dbgs() << "INCLUDED: " << getIncludePathForIndex(index) << "\n";

3342

continue;

3343

}

3344

3345

case OPC_EmitNode: case OPC_MorphNodeTo:

3346

case OPC_EmitNode0: case OPC_EmitNode1: case OPC_EmitNode2:

3347

case OPC_MorphNodeTo0: case OPC_MorphNodeTo1: case OPC_MorphNodeTo2: {

3348

uint16_t TargetOpc = MatcherTable[MatcherIndex++];

3349

TargetOpc |= (unsigned short)MatcherTable[MatcherIndex++] << 8;

3350

unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];

3351

// Get the result VT list.

3352

unsigned NumVTs;

3353

// If this is one of the compressed forms, get the number of VTs based

3354

// on the Opcode. Otherwise read the next byte from the table.

3355

if (Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2)

3356

NumVTs = Opcode - OPC_MorphNodeTo0;

3357

else if (Opcode >= OPC_EmitNode0 && Opcode <= OPC_EmitNode2)

3358

NumVTs = Opcode - OPC_EmitNode0;

3359

else

3360

NumVTs = MatcherTable[MatcherIndex++];

3361

SmallVector<EVT, 4> VTs;

3362

for (unsigned i = 0; i != NumVTs; ++i) {

3363

MVT::SimpleValueType VT =

3364

(MVT::SimpleValueType)MatcherTable[MatcherIndex++];

3365

if (VT == MVT::iPTR)

3366

VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;

3367

VTs.push_back(VT);

3368

}

3369

3370

if (EmitNodeInfo & OPFL_Chain)

3371

VTs.push_back(MVT::Other);

3372

if (EmitNodeInfo & OPFL_GlueOutput)

3373

VTs.push_back(MVT::Glue);

3374

3375

// This is hot code, so optimize the two most common cases of 1 and 2

3376

// results.

3377

SDVTList VTList;

3378

if (VTs.size() == 1)

3379

VTList = CurDAG->getVTList(VTs[0]);

3380

else if (VTs.size() == 2)

3381

VTList = CurDAG->getVTList(VTs[0], VTs[1]);

3382

else

3383

VTList = CurDAG->getVTList(VTs);

3384

3385

// Get the operand list.

3386

unsigned NumOps = MatcherTable[MatcherIndex++];

3387

SmallVector<SDValue, 8> Ops;

3388

for (unsigned i = 0; i != NumOps; ++i) {

3389

unsigned RecNo = MatcherTable[MatcherIndex++];

3390

if (RecNo & 128)

3391

RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);

3392

3393

assert(RecNo < RecordedNodes.size() && "Invalid EmitNode")((RecNo < RecordedNodes.size() && "Invalid EmitNode"
) ? static_cast<void> (0) : __assert_fail ("RecNo < RecordedNodes.size() && \"Invalid EmitNode\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3393, __PRETTY_FUNCTION__));

3394

Ops.push_back(RecordedNodes[RecNo].first);

3395

}

3396

3397

// If there are variadic operands to add, handle them now.

3398

if (EmitNodeInfo & OPFL_VariadicInfo) {

3399

// Determine the start index to copy from.

3400

unsigned FirstOpToCopy = getNumFixedFromVariadicInfo(EmitNodeInfo);

3401

FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;

3402

assert(NodeToMatch->getNumOperands() >= FirstOpToCopy &&((NodeToMatch->getNumOperands() >= FirstOpToCopy &&
"Invalid variadic node") ? static_cast<void> (0) : __assert_fail
("NodeToMatch->getNumOperands() >= FirstOpToCopy && \"Invalid variadic node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3403, __PRETTY_FUNCTION__))

3403

"Invalid variadic node")((NodeToMatch->getNumOperands() >= FirstOpToCopy &&
"Invalid variadic node") ? static_cast<void> (0) : __assert_fail
("NodeToMatch->getNumOperands() >= FirstOpToCopy && \"Invalid variadic node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3403, __PRETTY_FUNCTION__));

3404

// Copy all of the variadic operands, not including a potential glue

3405

// input.

3406

for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();

3407

i != e; ++i) {

3408

SDValue V = NodeToMatch->getOperand(i);

3409

if (V.getValueType() == MVT::Glue) break;

3410

Ops.push_back(V);

3411

}

3412

}

3413

3414

// If this has chain/glue inputs, add them.

3415

if (EmitNodeInfo & OPFL_Chain)

3416

Ops.push_back(InputChain);

3417

if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr)

3418

Ops.push_back(InputGlue);

3419

3420

// Create the node.

3421

SDNode *Res = nullptr;

3422

bool IsMorphNodeTo = Opcode == OPC_MorphNodeTo ||

3423

(Opcode >= OPC_MorphNodeTo0 && Opcode <= OPC_MorphNodeTo2);

3424

if (!IsMorphNodeTo) {

3425

// If this is a normal EmitNode command, just create the new node and

3426

// add the results to the RecordedNodes list.

3427

Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),

3428

VTList, Ops);

3429

3430

// Add all the non-glue/non-chain results to the RecordedNodes list.

3431

for (unsigned i = 0, e = VTs.size(); i != e; ++i) {

3432

if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;

3433

RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),

3434

nullptr));

3435

}

3436

} else {

3437

assert(NodeToMatch->getOpcode() != ISD::DELETED_NODE &&((NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
"NodeToMatch was removed partway through selection") ? static_cast
<void> (0) : __assert_fail ("NodeToMatch->getOpcode() != ISD::DELETED_NODE && \"NodeToMatch was removed partway through selection\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3438, __PRETTY_FUNCTION__))

3438

"NodeToMatch was removed partway through selection")((NodeToMatch->getOpcode() != ISD::DELETED_NODE &&
"NodeToMatch was removed partway through selection") ? static_cast
<void> (0) : __assert_fail ("NodeToMatch->getOpcode() != ISD::DELETED_NODE && \"NodeToMatch was removed partway through selection\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3438, __PRETTY_FUNCTION__));

3439

SelectionDAG::DAGNodeDeletedListener NDL(*CurDAG, [&](SDNode *N,

3440

SDNode *E) {

3441

auto &Chain = ChainNodesMatched;

3442

assert((!E || !is_contained(Chain, N)) &&(((!E || !is_contained(Chain, N)) && "Chain node replaced during MorphNode"
) ? static_cast<void> (0) : __assert_fail ("(!E || !is_contained(Chain, N)) && \"Chain node replaced during MorphNode\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3443, __PRETTY_FUNCTION__))

3443

"Chain node replaced during MorphNode")(((!E || !is_contained(Chain, N)) && "Chain node replaced during MorphNode"
) ? static_cast<void> (0) : __assert_fail ("(!E || !is_contained(Chain, N)) && \"Chain node replaced during MorphNode\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3443, __PRETTY_FUNCTION__));

3444

Chain.erase(std::remove(Chain.begin(), Chain.end(), N), Chain.end());

3445

});

3446

Res = MorphNode(NodeToMatch, TargetOpc, VTList, Ops, EmitNodeInfo);

3447

}

3448

3449

// If the node had chain/glue results, update our notion of the current

3450

// chain and glue.

3451

if (EmitNodeInfo & OPFL_GlueOutput) {

3452

InputGlue = SDValue(Res, VTs.size()-1);

3453

if (EmitNodeInfo & OPFL_Chain)

3454

InputChain = SDValue(Res, VTs.size()-2);

3455

} else if (EmitNodeInfo & OPFL_Chain)

3456

InputChain = SDValue(Res, VTs.size()-1);

3457

3458

// If the OPFL_MemRefs glue is set on this node, slap all of the

3459

// accumulated memrefs onto it.

3460

3461

// FIXME: This is vastly incorrect for patterns with multiple outputs

3462

// instructions that access memory and for ComplexPatterns that match

3463

// loads.

3464

if (EmitNodeInfo & OPFL_MemRefs) {

3465

// Only attach load or store memory operands if the generated

3466

// instruction may load or store.

3467

const MCInstrDesc &MCID = TII->get(TargetOpc);

3468

bool mayLoad = MCID.mayLoad();

3469

bool mayStore = MCID.mayStore();

3470

3471

unsigned NumMemRefs = 0;

3472

for (SmallVectorImpl<MachineMemOperand *>::const_iterator I =

3473

MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) {

3474

if ((*I)->isLoad()) {

3475

if (mayLoad)

3476

++NumMemRefs;

3477

} else if ((*I)->isStore()) {

3478

if (mayStore)

3479

++NumMemRefs;

3480

} else {

3481

++NumMemRefs;

3482

}

3483

}

3484

3485

MachineSDNode::mmo_iterator MemRefs =

3486

MF->allocateMemRefsArray(NumMemRefs);

3487

3488

MachineSDNode::mmo_iterator MemRefsPos = MemRefs;

3489

for (SmallVectorImpl<MachineMemOperand *>::const_iterator I =

3490

MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) {

3491

if ((*I)->isLoad()) {

3492

if (mayLoad)

3493

*MemRefsPos++ = *I;

3494

} else if ((*I)->isStore()) {

3495

if (mayStore)

3496

*MemRefsPos++ = *I;

3497

} else {

3498

*MemRefsPos++ = *I;

3499

}

3500

}

3501

3502

cast<MachineSDNode>(Res)

3503

->setMemRefs(MemRefs, MemRefs + NumMemRefs);

3504

}

3505

3506

DEBUG(dbgs() << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " " << (IsMorphNodeTo ? "Morphed"
: "Created") << " node: "; Res->dump(CurDAG); dbgs(
) << "\n"; } } while (false)

3507

<< (IsMorphNodeTo ? "Morphed" : "Created")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " " << (IsMorphNodeTo ? "Morphed"
: "Created") << " node: "; Res->dump(CurDAG); dbgs(
) << "\n"; } } while (false)

3508

<< " node: "; Res->dump(CurDAG); dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " " << (IsMorphNodeTo ? "Morphed"
: "Created") << " node: "; Res->dump(CurDAG); dbgs(
) << "\n"; } } while (false);

3509

3510

// If this was a MorphNodeTo then we're completely done!

3511

if (IsMorphNodeTo) {

3512

// Update chain uses.

3513

UpdateChains(Res, InputChain, ChainNodesMatched, true);

3514

return;

3515

}

3516

continue;

3517

}

3518

3519

case OPC_CompleteMatch: {

3520

// The match has been completed, and any new nodes (if any) have been

3521

// created. Patch up references to the matched dag to use the newly

3522

// created nodes.

3523

unsigned NumResults = MatcherTable[MatcherIndex++];

3524

3525

for (unsigned i = 0; i != NumResults; ++i) {

3526

unsigned ResSlot = MatcherTable[MatcherIndex++];

3527

if (ResSlot & 128)

3528

ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);

3529

3530

assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch")((ResSlot < RecordedNodes.size() && "Invalid CompleteMatch"
) ? static_cast<void> (0) : __assert_fail ("ResSlot < RecordedNodes.size() && \"Invalid CompleteMatch\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3530, __PRETTY_FUNCTION__));

3531

SDValue Res = RecordedNodes[ResSlot].first;

3532

3533

assert(i < NodeToMatch->getNumValues() &&((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3536, __PRETTY_FUNCTION__))

3534

NodeToMatch->getValueType(i) != MVT::Other &&((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3536, __PRETTY_FUNCTION__))

3535

NodeToMatch->getValueType(i) != MVT::Glue &&((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3536, __PRETTY_FUNCTION__))

3536

"Invalid number of results to complete!")((i < NodeToMatch->getNumValues() && NodeToMatch
->getValueType(i) != MVT::Other && NodeToMatch->
getValueType(i) != MVT::Glue && "Invalid number of results to complete!"
) ? static_cast<void> (0) : __assert_fail ("i < NodeToMatch->getNumValues() && NodeToMatch->getValueType(i) != MVT::Other && NodeToMatch->getValueType(i) != MVT::Glue && \"Invalid number of results to complete!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3536, __PRETTY_FUNCTION__));

3537

assert((NodeToMatch->getValueType(i) == Res.getValueType() ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))

3538

NodeToMatch->getValueType(i) == MVT::iPTR ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))

3539

Res.getValueType() == MVT::iPTR ||(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))

3540

NodeToMatch->getValueType(i).getSizeInBits() ==(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))

3541

Res.getValueSizeInBits()) &&(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__))

3542

"invalid replacement")(((NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch
->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT
::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res
.getValueSizeInBits()) && "invalid replacement") ? static_cast
<void> (0) : __assert_fail ("(NodeToMatch->getValueType(i) == Res.getValueType() || NodeToMatch->getValueType(i) == MVT::iPTR || Res.getValueType() == MVT::iPTR || NodeToMatch->getValueType(i).getSizeInBits() == Res.getValueSizeInBits()) && \"invalid replacement\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3542, __PRETTY_FUNCTION__));

3543

CurDAG->ReplaceAllUsesOfValueWith(SDValue(NodeToMatch, i), Res);

3544

}

3545

3546

// Update chain uses.

3547

UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false);

3548

3549

// If the root node defines glue, we need to update it to the glue result.

3550

// TODO: This never happens in our tests and I think it can be removed /

3551

// replaced with an assert, but if we do it this the way the change is

3552

// NFC.

3553

if (NodeToMatch->getValueType(NodeToMatch->getNumValues() - 1) ==

3554

MVT::Glue &&

3555

InputGlue.getNode())

3556

CurDAG->ReplaceAllUsesOfValueWith(

3557

SDValue(NodeToMatch, NodeToMatch->getNumValues() - 1), InputGlue);

3558

3559

assert(NodeToMatch->use_empty() &&((NodeToMatch->use_empty() && "Didn't replace all uses of the node?"
) ? static_cast<void> (0) : __assert_fail ("NodeToMatch->use_empty() && \"Didn't replace all uses of the node?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3560, __PRETTY_FUNCTION__))

3560

"Didn't replace all uses of the node?")((NodeToMatch->use_empty() && "Didn't replace all uses of the node?"
) ? static_cast<void> (0) : __assert_fail ("NodeToMatch->use_empty() && \"Didn't replace all uses of the node?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn302474/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp"
, 3560, __PRETTY_FUNCTION__));

3561

CurDAG->RemoveDeadNode(NodeToMatch);

3562

3563

return;

3564

}

3565

}

3566

3567

// If the code reached this point, then the match failed. See if there is

3568

// another child to try in the current 'Scope', otherwise pop it until we

3569

// find a case to check.

3570

DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Match failed at index " <<
CurrentOpcodeIndex << "\n"; } } while (false);

3571

++NumDAGIselRetries;

3572

while (true) {

3573

if (MatchScopes.empty()) {

3574

CannotYetSelect(NodeToMatch);

3575

return;

3576

}

3577

3578

// Restore the interpreter state back to the point where the scope was

3579

// formed.

3580

MatchScope &LastScope = MatchScopes.back();

3581

RecordedNodes.resize(LastScope.NumRecordedNodes);

3582

NodeStack.clear();

3583

NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());

3584

N = NodeStack.back();

3585

3586

if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())

3587

MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);

3588

MatcherIndex = LastScope.FailIndex;

3589

3590

DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Continuing at " << MatcherIndex
<< "\n"; } } while (false);

3591

3592

InputChain = LastScope.InputChain;

3593

InputGlue = LastScope.InputGlue;

3594

if (!LastScope.HasChainNodesMatched)

3595

ChainNodesMatched.clear();

3596

3597

// Check to see what the offset is at the new MatcherIndex. If it is zero

3598

// we have reached the end of this scope, otherwise we have another child

3599

// in the current scope to try.

3600

unsigned NumToSkip = MatcherTable[MatcherIndex++];

3601

if (NumToSkip & 128)

3602

NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);

3603

3604

// If we have another child in this scope to match, update FailIndex and

3605

// try it.

3606

if (NumToSkip != 0) {

3607

LastScope.FailIndex = MatcherIndex+NumToSkip;

3608

break;

3609

}

3610

3611

// End of this scope, pop it and try the next child in the containing

3612

// scope.

3613

MatchScopes.pop_back();

3614

}

3615

}

3616

}

3617

3618

void SelectionDAGISel::CannotYetSelect(SDNode *N) {

3619

std::string msg;

3620

raw_string_ostream Msg(msg);

3621

Msg << "Cannot select: ";

3622

3623

if (N->getOpcode() != ISD::INTRINSIC_W_CHAIN &&

3624

N->getOpcode() != ISD::INTRINSIC_WO_CHAIN &&

3625

N->getOpcode() != ISD::INTRINSIC_VOID) {

3626

N->printrFull(Msg, CurDAG);

3627

Msg << "\nIn function: " << MF->getName();

3628

} else {

3629

bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;

3630

unsigned iid =

3631

cast<ConstantSDNode>(N->getOperand(HasInputChain))->getZExtValue();

3632

if (iid < Intrinsic::num_intrinsics)

3633

Msg << "intrinsic %" << Intrinsic::getName((Intrinsic::ID)iid, None);

3634

else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())

3635

Msg << "target intrinsic %" << TII->getName(iid);

3636

else

3637

Msg << "unknown intrinsic #" << iid;

3638

}

3639

report_fatal_error(Msg.str());

3640

}

3641

3642

char SelectionDAGISel::ID = 0;