/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp

Bug Summary

File:	lib/Transforms/Vectorize/LoopVectorize.cpp
Warning:	line 6201, column 11 Called C++ object pointer is null

Annotated Source Code

//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//

// The LLVM Compiler Infrastructure

// This file is distributed under the University of Illinois Open Source

// License. See LICENSE.TXT for details.

//===----------------------------------------------------------------------===//

// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops

// and generates target-independent LLVM-IR.

// The vectorizer uses the TargetTransformInfo analysis to estimate the costs

// of instructions in order to estimate the profitability of vectorization.

// The loop vectorizer combines consecutive loop iterations into a single

// 'wide' iteration. After this transformation the index is incremented

// by the SIMD vector width, and not by one.

// This pass has three parts:

// 1. The main loop pass that drives the different parts.

// 2. LoopVectorizationLegality - A unit that checks for the legality

// of the vectorization.

// 3. InnerLoopVectorizer - A unit that performs the actual

// widening of instructions.

// 4. LoopVectorizationCostModel - A unit that checks for the profitability

// of vectorization. It decides on the optimal vector width, which

// can be one, if vectorization is not profitable.

//===----------------------------------------------------------------------===//

// The reduction-variable vectorization is based on the paper:

// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.

// Variable uniformity checks are inspired by:

// Karrenberg, R. and Hack, S. Whole Function Vectorization.

// The interleaved access vectorization is based on the paper:

// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved

// Data for SIMD

// Other ideas/concepts are from:

// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.

// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of

// Vectorizing Compilers.

//===----------------------------------------------------------------------===//

#include "llvm/Transforms/Vectorize/LoopVectorize.h"

#include "llvm/ADT/DenseMap.h"

#include "llvm/ADT/Hashing.h"

#include "llvm/ADT/MapVector.h"

#include "llvm/ADT/Optional.h"

#include "llvm/ADT/SCCIterator.h"

#include "llvm/ADT/SetVector.h"

#include "llvm/ADT/SmallPtrSet.h"

#include "llvm/ADT/SmallSet.h"

#include "llvm/ADT/SmallVector.h"

#include "llvm/ADT/Statistic.h"

#include "llvm/ADT/StringExtras.h"

#include "llvm/Analysis/CodeMetrics.h"

#include "llvm/Analysis/GlobalsModRef.h"

#include "llvm/Analysis/LoopInfo.h"

#include "llvm/Analysis/LoopIterator.h"

#include "llvm/Analysis/LoopPass.h"

#include "llvm/Analysis/ScalarEvolutionExpander.h"

#include "llvm/Analysis/ScalarEvolutionExpressions.h"

#include "llvm/Analysis/ValueTracking.h"

#include "llvm/Analysis/VectorUtils.h"

#include "llvm/IR/Constants.h"

#include "llvm/IR/DataLayout.h"

#include "llvm/IR/DebugInfo.h"

#include "llvm/IR/DerivedTypes.h"

#include "llvm/IR/DiagnosticInfo.h"

#include "llvm/IR/Dominators.h"

#include "llvm/IR/Function.h"

#include "llvm/IR/IRBuilder.h"

#include "llvm/IR/Instructions.h"

#include "llvm/IR/IntrinsicInst.h"

#include "llvm/IR/LLVMContext.h"

#include "llvm/IR/Module.h"

#include "llvm/IR/PatternMatch.h"

#include "llvm/IR/Type.h"

#include "llvm/IR/User.h"

#include "llvm/IR/Value.h"

#include "llvm/IR/ValueHandle.h"

#include "llvm/IR/Verifier.h"

#include "llvm/Pass.h"

#include "llvm/Support/BranchProbability.h"

#include "llvm/Support/CommandLine.h"

#include "llvm/Support/Debug.h"

#include "llvm/Support/raw_ostream.h"

#include "llvm/Transforms/Scalar.h"

#include "llvm/Transforms/Utils/BasicBlockUtils.h"

#include "llvm/Transforms/Utils/Local.h"

#include "llvm/Transforms/Utils/LoopSimplify.h"

#include "llvm/Transforms/Utils/LoopUtils.h"

#include "llvm/Transforms/Utils/LoopVersioning.h"

#include "llvm/Transforms/Vectorize.h"

100

#include <algorithm>

101

#include <map>

102

#include <tuple>

103

104

using namespace llvm;

105

using namespace llvm::PatternMatch;

106

107

#define LV_NAME"loop-vectorize" "loop-vectorize"

108

#define DEBUG_TYPE"loop-vectorize" LV_NAME"loop-vectorize"

109

110

STATISTIC(LoopsVectorized, "Number of loops vectorized")static llvm::Statistic LoopsVectorized = {"loop-vectorize", "LoopsVectorized"
, "Number of loops vectorized", {0}, false};

111

STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization")static llvm::Statistic LoopsAnalyzed = {"loop-vectorize", "LoopsAnalyzed"
, "Number of loops analyzed for vectorization", {0}, false};

112

113

static cl::opt<bool>

114

EnableIfConversion("enable-if-conversion", cl::init(true), cl::Hidden,

115

cl::desc("Enable if-conversion during vectorization."));

116

117

/// We don't vectorize loops with a known constant trip count below this number.

118

static cl::opt<unsigned> TinyTripCountVectorThreshold(

119

"vectorizer-min-trip-count", cl::init(16), cl::Hidden,

120

cl::desc("Don't vectorize loops with a constant "

121

"trip count that is smaller than this "

122

"value."));

123

124

static cl::opt<bool> MaximizeBandwidth(

125

"vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,

126

cl::desc("Maximize bandwidth when selecting vectorization factor which "

127

"will be determined by the smallest type in loop."));

128

129

static cl::opt<bool> EnableInterleavedMemAccesses(

130

"enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,

131

cl::desc("Enable vectorization on interleaved memory accesses in a loop"));

132

133

/// Maximum factor for an interleaved memory access.

134

static cl::opt<unsigned> MaxInterleaveGroupFactor(

135

"max-interleave-group-factor", cl::Hidden,

136

cl::desc("Maximum factor for an interleaved access group (default = 8)"),

137

cl::init(8));

138

139

/// We don't interleave loops with a known constant trip count below this

140

/// number.

141

static const unsigned TinyTripCountInterleaveThreshold = 128;

142

143

static cl::opt<unsigned> ForceTargetNumScalarRegs(

144

"force-target-num-scalar-regs", cl::init(0), cl::Hidden,

145

cl::desc("A flag that overrides the target's number of scalar registers."));

146

147

static cl::opt<unsigned> ForceTargetNumVectorRegs(

148

"force-target-num-vector-regs", cl::init(0), cl::Hidden,

149

cl::desc("A flag that overrides the target's number of vector registers."));

150

151

/// Maximum vectorization interleave count.

152

static const unsigned MaxInterleaveFactor = 16;

153

154

static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(

155

"force-target-max-scalar-interleave", cl::init(0), cl::Hidden,

156

cl::desc("A flag that overrides the target's max interleave factor for "

157

"scalar loops."));

158

159

static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(

160

"force-target-max-vector-interleave", cl::init(0), cl::Hidden,

161

cl::desc("A flag that overrides the target's max interleave factor for "

162

"vectorized loops."));

163

164

static cl::opt<unsigned> ForceTargetInstructionCost(

165

"force-target-instruction-cost", cl::init(0), cl::Hidden,

166

cl::desc("A flag that overrides the target's expected cost for "

167

"an instruction to a single constant value. Mostly "

168

"useful for getting consistent testing."));

169

170

static cl::opt<unsigned> SmallLoopCost(

171

"small-loop-cost", cl::init(20), cl::Hidden,

172

cl::desc(

173

"The cost of a loop that is considered 'small' by the interleaver."));

174

175

static cl::opt<bool> LoopVectorizeWithBlockFrequency(

176

"loop-vectorize-with-block-frequency", cl::init(false), cl::Hidden,

177

cl::desc("Enable the use of the block frequency analysis to access PGO "

178

"heuristics minimizing code growth in cold regions and being more "

179

"aggressive in hot regions."));

180

181

// Runtime interleave loops for load/store throughput.

182

static cl::opt<bool> EnableLoadStoreRuntimeInterleave(

183

"enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,

184

cl::desc(

185

"Enable runtime interleaving until load/store ports are saturated"));

186

187

/// The number of stores in a loop that are allowed to need predication.

188

static cl::opt<unsigned> NumberOfStoresToPredicate(

189

"vectorize-num-stores-pred", cl::init(1), cl::Hidden,

190

cl::desc("Max number of stores to be predicated behind an if."));

191

192

static cl::opt<bool> EnableIndVarRegisterHeur(

193

"enable-ind-var-reg-heur", cl::init(true), cl::Hidden,

194

cl::desc("Count the induction variable only once when interleaving"));

195

196

static cl::opt<bool> EnableCondStoresVectorization(

197

"enable-cond-stores-vec", cl::init(true), cl::Hidden,

198

cl::desc("Enable if predication of stores during vectorization."));

199

200

static cl::opt<unsigned> MaxNestedScalarReductionIC(

201

"max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,

202

cl::desc("The maximum interleave count to use when interleaving a scalar "

203

"reduction in a nested loop."));

204

205

static cl::opt<unsigned> PragmaVectorizeMemoryCheckThreshold(

206

"pragma-vectorize-memory-check-threshold", cl::init(128), cl::Hidden,

207

cl::desc("The maximum allowed number of runtime memory checks with a "

208

"vectorize(enable) pragma."));

209

210

static cl::opt<unsigned> VectorizeSCEVCheckThreshold(

211

"vectorize-scev-check-threshold", cl::init(16), cl::Hidden,

212

cl::desc("The maximum number of SCEV checks allowed."));

213

214

static cl::opt<unsigned> PragmaVectorizeSCEVCheckThreshold(

215

"pragma-vectorize-scev-check-threshold", cl::init(128), cl::Hidden,

216

cl::desc("The maximum number of SCEV checks allowed with a "

217

"vectorize(enable) pragma"));

218

219

/// Create an analysis remark that explains why vectorization failed

220

///

221

/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p

222

/// RemarkName is the identifier for the remark. If \p I is passed it is an

223

/// instruction that prevents vectorization. Otherwise \p TheLoop is used for

224

/// the location of the remark. \return the remark object that can be

225

/// streamed to.

226

static OptimizationRemarkAnalysis

227

createMissedAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop,

228

Instruction *I = nullptr) {

229

Value *CodeRegion = TheLoop->getHeader();

230

DebugLoc DL = TheLoop->getStartLoc();

231

232

if (I) {

233

CodeRegion = I->getParent();

234

// If there is no debug location attached to the instruction, revert back to

235

// using the loop's.

236

if (I->getDebugLoc())

237

DL = I->getDebugLoc();

238

}

239

240

OptimizationRemarkAnalysis R(PassName, RemarkName, DL, CodeRegion);

241

R << "loop not vectorized: ";

242

return R;

243

}

244

245

namespace {

246

247

// Forward declarations.

248

class LoopVectorizeHints;

249

class LoopVectorizationLegality;

250

class LoopVectorizationCostModel;

251

class LoopVectorizationRequirements;

252

253

/// Returns true if the given loop body has a cycle, excluding the loop

254

/// itself.

255

static bool hasCyclesInLoopBody(const Loop &L) {

256

if (!L.empty())

257

return true;

258

259

for (const auto &SCC :

260

make_range(scc_iterator<Loop, LoopBodyTraits>::begin(L),

261

scc_iterator<Loop, LoopBodyTraits>::end(L))) {

262

if (SCC.size() > 1) {

263

DEBUG(dbgs() << "LVL: Detected a cycle in the loop body:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LVL: Detected a cycle in the loop body:\n"
; } } while (false);

264

DEBUG(L.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { L.dump(); } } while (false);

265

return true;

266

}

267

}

268

return false;

269

}

270

271

/// A helper function for converting Scalar types to vector types.

272

/// If the incoming type is void, we return void. If the VF is 1, we return

273

/// the scalar type.

274

static Type *ToVectorTy(Type *Scalar, unsigned VF) {

275

if (Scalar->isVoidTy() || VF == 1)

276

return Scalar;

277

return VectorType::get(Scalar, VF);

278

}

279

280

// FIXME: The following helper functions have multiple implementations

281

// in the project. They can be effectively organized in a common Load/Store

282

// utilities unit.

283

284

/// A helper function that returns the pointer operand of a load or store

285

/// instruction.

286

static Value *getPointerOperand(Value *I) {

287

if (auto *LI = dyn_cast<LoadInst>(I))

288

return LI->getPointerOperand();

289

if (auto *SI = dyn_cast<StoreInst>(I))

290

return SI->getPointerOperand();

291

return nullptr;

292

}

293

294

/// A helper function that returns the type of loaded or stored value.

295

static Type *getMemInstValueType(Value *I) {

296

assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
"Expected Load or Store instruction") ? static_cast<void>
(0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 297, __PRETTY_FUNCTION__))

297

"Expected Load or Store instruction")(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
"Expected Load or Store instruction") ? static_cast<void>
(0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 297, __PRETTY_FUNCTION__));

298

if (auto *LI = dyn_cast<LoadInst>(I))

299

return LI->getType();

300

return cast<StoreInst>(I)->getValueOperand()->getType();

301

}

302

303

/// A helper function that returns the alignment of load or store instruction.

304

static unsigned getMemInstAlignment(Value *I) {

305

306

307

if (auto *LI = dyn_cast<LoadInst>(I))

308

return LI->getAlignment();

309

return cast<StoreInst>(I)->getAlignment();

310

}

311

312

/// A helper function that returns the address space of the pointer operand of

313

/// load or store instruction.

314

static unsigned getMemInstAddressSpace(Value *I) {

315

316

317

if (auto *LI = dyn_cast<LoadInst>(I))

318

return LI->getPointerAddressSpace();

319

return cast<StoreInst>(I)->getPointerAddressSpace();

320

}

321

322

/// A helper function that returns true if the given type is irregular. The

323

/// type is irregular if its allocated size doesn't equal the store size of an

324

/// element of the corresponding vector type at the given vectorization factor.

325

static bool hasIrregularType(Type *Ty, const DataLayout &DL, unsigned VF) {

326

327

// Determine if an array of VF elements of type Ty is "bitcast compatible"

328

// with a <VF x Ty> vector.

329

if (VF > 1) {

330

auto *VectorTy = VectorType::get(Ty, VF);

331

return VF * DL.getTypeAllocSize(Ty) != DL.getTypeStoreSize(VectorTy);

332

}

333

334

// If the vectorization factor is one, we just check if an array of type Ty

335

// requires padding between elements.

336

return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);

337

}

338

339

/// A helper function that returns the reciprocal of the block probability of

340

/// predicated blocks. If we return X, we are assuming the predicated block

341

/// will execute once for for every X iterations of the loop header.

342

///

343

/// TODO: We should use actual block probability here, if available. Currently,

344

/// we always assume predicated blocks have a 50% chance of executing.

345

static unsigned getReciprocalPredBlockProb() { return 2; }

346

347

/// A helper function that adds a 'fast' flag to floating-point operations.

348

static Value *addFastMathFlag(Value *V) {

349

if (isa<FPMathOperator>(V)) {

350

FastMathFlags Flags;

351

Flags.setUnsafeAlgebra();

352

cast<Instruction>(V)->setFastMathFlags(Flags);

353

}

354

return V;

355

}

356

357

/// A helper function that returns an integer or floating-point constant with

358

/// value C.

359

static Constant *getSignedIntOrFpConstant(Type *Ty, int64_t C) {

360

return Ty->isIntegerTy() ? ConstantInt::getSigned(Ty, C)

361

: ConstantFP::get(Ty, C);

362

}

363

364

/// InnerLoopVectorizer vectorizes loops which contain only one basic

365

/// block to a specified vectorization factor (VF).

366

/// This class performs the widening of scalars into vectors, or multiple

367

/// scalars. This class also implements the following features:

368

/// * It inserts an epilogue loop for handling loops that don't have iteration

369

/// counts that are known to be a multiple of the vectorization factor.

370

/// * It handles the code generation for reduction variables.

371

/// * Scalarization (implementation using scalars) of un-vectorizable

372

/// instructions.

373

/// InnerLoopVectorizer does not perform any vectorization-legality

374

/// checks, and relies on the caller to check for the different legality

375

/// aspects. The InnerLoopVectorizer relies on the

376

/// LoopVectorizationLegality class to provide information about the induction

377

/// and reduction variables that were found to a given vectorization factor.

378

class InnerLoopVectorizer {

379

public:

380

InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,

381

LoopInfo *LI, DominatorTree *DT,

382

const TargetLibraryInfo *TLI,

383

const TargetTransformInfo *TTI, AssumptionCache *AC,

384

OptimizationRemarkEmitter *ORE, unsigned VecWidth,

385

unsigned UnrollFactor, LoopVectorizationLegality *LVL,

386

LoopVectorizationCostModel *CM)

387

: OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),

388

AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor),

389

Builder(PSE.getSE()->getContext()), Induction(nullptr),

390

OldInduction(nullptr), VectorLoopValueMap(UnrollFactor, VecWidth),

391

TripCount(nullptr), VectorTripCount(nullptr), Legal(LVL), Cost(CM),

392

AddedSafetyChecks(false) {}

393

394

/// Create a new empty loop. Unlink the old loop and connect the new one.

395

void createVectorizedLoopSkeleton();

396

397

/// Vectorize a single instruction within the innermost loop.

398

void vectorizeInstruction(Instruction &I);

399

400

/// Fix the vectorized code, taking care of header phi's, live-outs, and more.

401

void fixVectorizedLoop();

402

403

// Return true if any runtime check is added.

404

bool areSafetyChecksAdded() { return AddedSafetyChecks; }

405

406

virtual ~InnerLoopVectorizer() {}

407

408

protected:

409

/// A small list of PHINodes.

410

typedef SmallVector<PHINode *, 4> PhiVector;

411

412

/// A type for vectorized values in the new loop. Each value from the

413

/// original loop, when vectorized, is represented by UF vector values in the

414

/// new unrolled loop, where UF is the unroll factor.

415

typedef SmallVector<Value *, 2> VectorParts;

416

417

/// A type for scalarized values in the new loop. Each value from the

418

/// original loop, when scalarized, is represented by UF x VF scalar values

419

/// in the new unrolled loop, where UF is the unroll factor and VF is the

420

/// vectorization factor.

421

typedef SmallVector<SmallVector<Value *, 4>, 2> ScalarParts;

422

423

// When we if-convert we need to create edge masks. We have to cache values

424

// so that we don't end up with exponential recursion/IR.

425

typedef DenseMap<std::pair<BasicBlock *, BasicBlock *>, VectorParts>

426

EdgeMaskCacheTy;

427

typedef DenseMap<BasicBlock *, VectorParts> BlockMaskCacheTy;

428

429

/// Set up the values of the IVs correctly when exiting the vector loop.

430

void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,

431

Value *CountRoundDown, Value *EndValue,

432

BasicBlock *MiddleBlock);

433

434

/// Create a new induction variable inside L.

435

PHINode *createInductionVariable(Loop *L, Value *Start, Value *End,

436

Value *Step, Instruction *DL);

437

438

/// Handle all cross-iteration phis in the header.

439

void fixCrossIterationPHIs();

440

441

/// Fix a first-order recurrence. This is the second phase of vectorizing

442

/// this phi node.

443

void fixFirstOrderRecurrence(PHINode *Phi);

444

445

/// Fix a reduction cross-iteration phi. This is the second phase of

446

/// vectorizing this phi node.

447

void fixReduction(PHINode *Phi);

448

449

/// \brief The Loop exit block may have single value PHI nodes with some

450

/// incoming value. While vectorizing we only handled real values

451

/// that were defined inside the loop and we should have one value for

452

/// each predecessor of its parent basic block. See PR14725.

453

void fixLCSSAPHIs();

454

455

/// Iteratively sink the scalarized operands of a predicated instruction into

456

/// the block that was created for it.

457

void sinkScalarOperands(Instruction *PredInst);

458

459

/// Predicate conditional instructions that require predication on their

460

/// respective conditions.

461

void predicateInstructions();

462

463

/// Shrinks vector element sizes to the smallest bitwidth they can be legally

464

/// represented as.

465

void truncateToMinimalBitwidths();

466

467

/// A helper function that computes the predicate of the block BB, assuming

468

/// that the header block of the loop is set to True. It returns the *entry*

469

/// mask for the block BB.

470

VectorParts createBlockInMask(BasicBlock *BB);

471

/// A helper function that computes the predicate of the edge between SRC

472

/// and DST.

473

VectorParts createEdgeMask(BasicBlock *Src, BasicBlock *Dst);

474

475

/// Vectorize a single PHINode in a block. This method handles the induction

476

/// variable canonicalization. It supports both VF = 1 for unrolled loops and

477

/// arbitrary length vectors.

478

void widenPHIInstruction(Instruction *PN, unsigned UF, unsigned VF);

479

480

/// Insert the new loop to the loop hierarchy and pass manager

481

/// and update the analysis passes.

482

void updateAnalysis();

483

484

/// This instruction is un-vectorizable. Implement it as a sequence

485

/// of scalars. If \p IfPredicateInstr is true we need to 'hide' each

486

/// scalarized instruction behind an if block predicated on the control

487

/// dependence of the instruction.

488

void scalarizeInstruction(Instruction *Instr, bool IfPredicateInstr = false);

489

490

/// Vectorize Load and Store instructions,

491

virtual void vectorizeMemoryInstruction(Instruction *Instr);

492

493

/// Create a broadcast instruction. This method generates a broadcast

494

/// instruction (shuffle) for loop invariant values and for the induction

495

/// value. If this is the induction variable then we extend it to N, N+1, ...

496

/// this is needed because each iteration in the loop corresponds to a SIMD

497

/// element.

498

virtual Value *getBroadcastInstrs(Value *V);

499

500

/// This function adds (StartIdx, StartIdx + Step, StartIdx + 2*Step, ...)

501

/// to each vector element of Val. The sequence starts at StartIndex.

502

/// \p Opcode is relevant for FP induction variable.

503

virtual Value *getStepVector(Value *Val, int StartIdx, Value *Step,

504

Instruction::BinaryOps Opcode =

505

Instruction::BinaryOpsEnd);

506

507

/// Compute scalar induction steps. \p ScalarIV is the scalar induction

508

/// variable on which to base the steps, \p Step is the size of the step, and

509

/// \p EntryVal is the value from the original loop that maps to the steps.

510

/// Note that \p EntryVal doesn't have to be an induction variable (e.g., it

511

/// can be a truncate instruction).

512

void buildScalarSteps(Value *ScalarIV, Value *Step, Value *EntryVal,

513

const InductionDescriptor &ID);

514

515

/// Create a vector induction phi node based on an existing scalar one. \p

516

/// EntryVal is the value from the original loop that maps to the vector phi

517

/// node, and \p Step is the loop-invariant step. If \p EntryVal is a

518

/// truncate instruction, instead of widening the original IV, we widen a

519

/// version of the IV truncated to \p EntryVal's type.

520

void createVectorIntOrFpInductionPHI(const InductionDescriptor &II,

521

Value *Step, Instruction *EntryVal);

522

523

/// Widen an integer or floating-point induction variable \p IV. If \p Trunc

524

/// is provided, the integer induction variable will first be truncated to

525

/// the corresponding type.

526

void widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc = nullptr);

527

528

/// Returns true if an instruction \p I should be scalarized instead of

529

/// vectorized for the chosen vectorization factor.

530

bool shouldScalarizeInstruction(Instruction *I) const;

531

532

/// Returns true if we should generate a scalar version of \p IV.

533

bool needsScalarInduction(Instruction *IV) const;

534

535

/// getOrCreateVectorValue and getOrCreateScalarValue coordinate to generate a

536

/// vector or scalar value on-demand if one is not yet available. When

537

/// vectorizing a loop, we visit the definition of an instruction before its

538

/// uses. When visiting the definition, we either vectorize or scalarize the

539

/// instruction, creating an entry for it in the corresponding map. (In some

540

/// cases, such as induction variables, we will create both vector and scalar

541

/// entries.) Then, as we encounter uses of the definition, we derive values

542

/// for each scalar or vector use unless such a value is already available.

543

/// For example, if we scalarize a definition and one of its uses is vector,

544

/// we build the required vector on-demand with an insertelement sequence

545

/// when visiting the use. Otherwise, if the use is scalar, we can use the

546

/// existing scalar definition.

547

///

548

/// Return a value in the new loop corresponding to \p V from the original

549

/// loop at unroll index \p Part. If the value has already been vectorized,

550

/// the corresponding vector entry in VectorLoopValueMap is returned. If,

551

/// however, the value has a scalar entry in VectorLoopValueMap, we construct

552

/// a new vector value on-demand by inserting the scalar values into a vector

553

/// with an insertelement sequence. If the value has been neither vectorized

554

/// nor scalarized, it must be loop invariant, so we simply broadcast the

555

/// value into a vector.

556

Value *getOrCreateVectorValue(Value *V, unsigned Part);

557

558

/// Return a value in the new loop corresponding to \p V from the original

559

/// loop at unroll index \p Part and vector index \p Lane. If the value has

560

/// been vectorized but not scalarized, the necessary extractelement

561

/// instruction will be generated.

562

Value *getOrCreateScalarValue(Value *V, unsigned Part, unsigned Lane);

563

564

/// Try to vectorize the interleaved access group that \p Instr belongs to.

565

void vectorizeInterleaveGroup(Instruction *Instr);

566

567

/// Generate a shuffle sequence that will reverse the vector Vec.

568

virtual Value *reverseVector(Value *Vec);

569

570

/// Returns (and creates if needed) the original loop trip count.

571

Value *getOrCreateTripCount(Loop *NewLoop);

572

573

/// Returns (and creates if needed) the trip count of the widened loop.

574

Value *getOrCreateVectorTripCount(Loop *NewLoop);

575

576

/// Emit a bypass check to see if the trip count would overflow, or we

577

/// wouldn't have enough iterations to execute one vector loop.

578

void emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass);

579

/// Emit a bypass check to see if the vector trip count is nonzero.

580

void emitVectorLoopEnteredCheck(Loop *L, BasicBlock *Bypass);

581

/// Emit a bypass check to see if all of the SCEV assumptions we've

582

/// had to make are correct.

583

void emitSCEVChecks(Loop *L, BasicBlock *Bypass);

584

/// Emit bypass checks to check any memory assumptions we may have made.

585

void emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass);

586

587

/// Add additional metadata to \p To that was not present on \p Orig.

588

///

589

/// Currently this is used to add the noalias annotations based on the

590

/// inserted memchecks. Use this for instructions that are *cloned* into the

591

/// vector loop.

592

void addNewMetadata(Instruction *To, const Instruction *Orig);

593

594

/// Add metadata from one instruction to another.

595

///

596

/// This includes both the original MDs from \p From and additional ones (\see

597

/// addNewMetadata). Use this for *newly created* instructions in the vector

598

/// loop.

599

void addMetadata(Instruction *To, Instruction *From);

600

601

/// \brief Similar to the previous function but it adds the metadata to a

602

/// vector of instructions.

603

void addMetadata(ArrayRef<Value *> To, Instruction *From);

604

605

/// \brief Set the debug location in the builder using the debug location in

606

/// the instruction.

607

void setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr);

608

609

/// This is a helper class for maintaining vectorization state. It's used for

610

/// mapping values from the original loop to their corresponding values in

611

/// the new loop. Two mappings are maintained: one for vectorized values and

612

/// one for scalarized values. Vectorized values are represented with UF

613

/// vector values in the new loop, and scalarized values are represented with

614

/// UF x VF scalar values in the new loop. UF and VF are the unroll and

615

/// vectorization factors, respectively.

616

///

617

/// Entries can be added to either map with setVectorValue and setScalarValue,

618

/// which assert that an entry was not already added before. If an entry is to

619

/// replace an existing one, call resetVectorValue. This is currently needed

620

/// to modify the mapped values during "fix-up" operations that occur once the

621

/// first phase of widening is complete. These operations include type

622

/// truncation and the second phase of recurrence widening.

623

///

624

/// Entries from either map can be retrieved using the getVectorValue and

625

/// getScalarValue functions, which assert that the desired value exists.

626

627

struct ValueMap {

628

629

/// Construct an empty map with the given unroll and vectorization factors.

630

ValueMap(unsigned UF, unsigned VF) : UF(UF), VF(VF) {}

631

632

/// \return True if the map has any vector entry for \p Key.

633

bool hasAnyVectorValue(Value *Key) const {

634

return VectorMapStorage.count(Key);

635

}

636

637

/// \return True if the map has a vector entry for \p Key and \p Part.

638

bool hasVectorValue(Value *Key, unsigned Part) const {

639

assert(Part < UF && "Queried Vector Part is too large.")((Part < UF && "Queried Vector Part is too large."
) ? static_cast<void> (0) : __assert_fail ("Part < UF && \"Queried Vector Part is too large.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 639, __PRETTY_FUNCTION__));

640

if (!hasAnyVectorValue(Key))

641

return false;

642

const VectorParts &Entry = VectorMapStorage.find(Key)->second;

643

assert(Entry.size() == UF && "VectorParts has wrong dimensions.")((Entry.size() == UF && "VectorParts has wrong dimensions."
) ? static_cast<void> (0) : __assert_fail ("Entry.size() == UF && \"VectorParts has wrong dimensions.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 643, __PRETTY_FUNCTION__));

644

return Entry[Part] != nullptr;

645

}

646

647

/// \return True if the map has any scalar entry for \p Key.

648

bool hasAnyScalarValue(Value *Key) const {

649

return ScalarMapStorage.count(Key);

650

}

651

652

/// \return True if the map has a scalar entry for \p Key, \p Part and

653

/// \p Part.

654

bool hasScalarValue(Value *Key, unsigned Part, unsigned Lane) const {

655

assert(Part < UF && "Queried Scalar Part is too large.")((Part < UF && "Queried Scalar Part is too large."
) ? static_cast<void> (0) : __assert_fail ("Part < UF && \"Queried Scalar Part is too large.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 655, __PRETTY_FUNCTION__));

656

assert(Lane < VF && "Queried Scalar Lane is too large.")((Lane < VF && "Queried Scalar Lane is too large."
) ? static_cast<void> (0) : __assert_fail ("Lane < VF && \"Queried Scalar Lane is too large.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 656, __PRETTY_FUNCTION__));

657

if (!hasAnyScalarValue(Key))

658

return false;

659

const ScalarParts &Entry = ScalarMapStorage.find(Key)->second;

660

assert(Entry.size() == UF && "ScalarParts has wrong dimensions.")((Entry.size() == UF && "ScalarParts has wrong dimensions."
) ? static_cast<void> (0) : __assert_fail ("Entry.size() == UF && \"ScalarParts has wrong dimensions.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 660, __PRETTY_FUNCTION__));

661

assert(Entry[Part].size() == VF && "ScalarParts has wrong dimensions.")((Entry[Part].size() == VF && "ScalarParts has wrong dimensions."
) ? static_cast<void> (0) : __assert_fail ("Entry[Part].size() == VF && \"ScalarParts has wrong dimensions.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 661, __PRETTY_FUNCTION__));

662

return Entry[Part][Lane] != nullptr;

663

}

664

665

/// Retrieve the existing vector value that corresponds to \p Key and

666

/// \p Part.

667

Value *getVectorValue(Value *Key, unsigned Part) {

668

assert(hasVectorValue(Key, Part) && "Getting non-existent value.")((hasVectorValue(Key, Part) && "Getting non-existent value."
) ? static_cast<void> (0) : __assert_fail ("hasVectorValue(Key, Part) && \"Getting non-existent value.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 668, __PRETTY_FUNCTION__));

669

return VectorMapStorage[Key][Part];

670

}

671

672

/// Retrieve the existing scalar value that corresponds to \p Key, \p Part

673

/// and \p Lane.

674

Value *getScalarValue(Value *Key, unsigned Part, unsigned Lane) {

675

assert(hasScalarValue(Key, Part, Lane) && "Getting non-existent value.")((hasScalarValue(Key, Part, Lane) && "Getting non-existent value."
) ? static_cast<void> (0) : __assert_fail ("hasScalarValue(Key, Part, Lane) && \"Getting non-existent value.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 675, __PRETTY_FUNCTION__));

676

return ScalarMapStorage[Key][Part][Lane];

677

}

678

679

/// Set a vector value associated with \p Key and \p Part. Assumes such a

680

/// value is not already set. If it is, use resetVectorValue() instead.

681

void setVectorValue(Value *Key, unsigned Part, Value *Vector) {

682

assert(!hasVectorValue(Key, Part) && "Vector value already set for part")((!hasVectorValue(Key, Part) && "Vector value already set for part"
) ? static_cast<void> (0) : __assert_fail ("!hasVectorValue(Key, Part) && \"Vector value already set for part\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 682, __PRETTY_FUNCTION__));

683

if (!VectorMapStorage.count(Key)) {

684

VectorParts Entry(UF);

685

VectorMapStorage[Key] = Entry;

686

}

687

VectorMapStorage[Key][Part] = Vector;

688

}

689

690

/// Set a scalar value associated with \p Key for \p Part and \p Lane.

691

/// Assumes such a value is not already set.

692

void setScalarValue(Value *Key, unsigned Part, unsigned Lane,

693

Value *Scalar) {

694

assert(!hasScalarValue(Key, Part, Lane) && "Scalar value already set")((!hasScalarValue(Key, Part, Lane) && "Scalar value already set"
) ? static_cast<void> (0) : __assert_fail ("!hasScalarValue(Key, Part, Lane) && \"Scalar value already set\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 694, __PRETTY_FUNCTION__));

695

if (!ScalarMapStorage.count(Key)) {

696

ScalarParts Entry(UF);

697

for (unsigned Part = 0; Part < UF; ++Part)

698

Entry[Part].resize(VF, nullptr);

699

// TODO: Consider storing uniform values only per-part, as they occupy

700

// lane 0 only, keeping the other VF-1 redundant entries null.

701

ScalarMapStorage[Key] = Entry;

702

}

703

ScalarMapStorage[Key][Part][Lane] = Scalar;

704

}

705

706

/// Reset the vector value associated with \p Key for the given \p Part.

707

/// This function can be used to update values that have already been

708

/// vectorized. This is the case for "fix-up" operations including type

709

/// truncation and the second phase of recurrence vectorization.

710

void resetVectorValue(Value *Key, unsigned Part, Value *Vector) {

711

assert(hasVectorValue(Key, Part) && "Vector value not set for part")((hasVectorValue(Key, Part) && "Vector value not set for part"
) ? static_cast<void> (0) : __assert_fail ("hasVectorValue(Key, Part) && \"Vector value not set for part\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 711, __PRETTY_FUNCTION__));

712

VectorMapStorage[Key][Part] = Vector;

713

}

714

715

private:

716

/// The unroll factor. Each entry in the vector map contains UF vector

717

/// values.

718

unsigned UF;

719

720

/// The vectorization factor. Each entry in the scalar map contains UF x VF

721

/// scalar values.

722

unsigned VF;

723

724

/// The vector and scalar map storage. We use std::map and not DenseMap

725

/// because insertions to DenseMap invalidate its iterators.

726

std::map<Value *, VectorParts> VectorMapStorage;

727

std::map<Value *, ScalarParts> ScalarMapStorage;

728

};

729

730

/// The original loop.

731

Loop *OrigLoop;

732

/// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies

733

/// dynamic knowledge to simplify SCEV expressions and converts them to a

734

/// more usable form.

735

PredicatedScalarEvolution &PSE;

736

/// Loop Info.

737

LoopInfo *LI;

738

/// Dominator Tree.

739

DominatorTree *DT;

740

/// Alias Analysis.

741

AliasAnalysis *AA;

742

/// Target Library Info.

743

const TargetLibraryInfo *TLI;

744

/// Target Transform Info.

745

const TargetTransformInfo *TTI;

746

/// Assumption Cache.

747

AssumptionCache *AC;

748

/// Interface to emit optimization remarks.

749

OptimizationRemarkEmitter *ORE;

750

751

/// \brief LoopVersioning. It's only set up (non-null) if memchecks were

752

/// used.

753

///

754

/// This is currently only used to add no-alias metadata based on the

755

/// memchecks. The actually versioning is performed manually.

756

std::unique_ptr<LoopVersioning> LVer;

757

758

/// The vectorization SIMD factor to use. Each vector will have this many

759

/// vector elements.

760

unsigned VF;

761

762

protected:

763

/// The vectorization unroll factor to use. Each scalar is vectorized to this

764

/// many different vector instructions.

765

unsigned UF;

766

767

/// The builder that we use

768

IRBuilder<> Builder;

769

770

// --- Vectorization state ---

771

772

/// The vector-loop preheader.

773

BasicBlock *LoopVectorPreHeader;

774

/// The scalar-loop preheader.

775

BasicBlock *LoopScalarPreHeader;

776

/// Middle Block between the vector and the scalar.

777

BasicBlock *LoopMiddleBlock;

778

/// The ExitBlock of the scalar loop.

779

BasicBlock *LoopExitBlock;

780

/// The vector loop body.

781

BasicBlock *LoopVectorBody;

782

/// The scalar loop body.

783

BasicBlock *LoopScalarBody;

784

/// A list of all bypass blocks. The first block is the entry of the loop.

785

SmallVector<BasicBlock *, 4> LoopBypassBlocks;

786

787

/// The new Induction variable which was added to the new block.

788

PHINode *Induction;

789

/// The induction variable of the old basic block.

790

PHINode *OldInduction;

791

792

/// Maps values from the original loop to their corresponding values in the

793

/// vectorized loop. A key value can map to either vector values, scalar

794

/// values or both kinds of values, depending on whether the key was

795

/// vectorized and scalarized.

796

ValueMap VectorLoopValueMap;

797

798

/// Store instructions that should be predicated, as a pair

799

/// <StoreInst, Predicate>

800

SmallVector<std::pair<Instruction *, Value *>, 4> PredicatedInstructions;

801

EdgeMaskCacheTy EdgeMaskCache;

802

BlockMaskCacheTy BlockMaskCache;

803

/// Trip count of the original loop.

804

Value *TripCount;

805

/// Trip count of the widened loop (TripCount - TripCount % (VF*UF))

806

Value *VectorTripCount;

807

808

/// The legality analysis.

809

LoopVectorizationLegality *Legal;

810

811

/// The profitablity analysis.

812

LoopVectorizationCostModel *Cost;

813

814

// Record whether runtime checks are added.

815

bool AddedSafetyChecks;

816

817

// Holds the end values for each induction variable. We save the end values

818

// so we can later fix-up the external users of the induction variables.

819

DenseMap<PHINode *, Value *> IVEndValues;

820

};

821

822

class InnerLoopUnroller : public InnerLoopVectorizer {

823

public:

824

InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE,

825

LoopInfo *LI, DominatorTree *DT,

826

const TargetLibraryInfo *TLI,

827

const TargetTransformInfo *TTI, AssumptionCache *AC,

828

OptimizationRemarkEmitter *ORE, unsigned UnrollFactor,

829

LoopVectorizationLegality *LVL,

830

LoopVectorizationCostModel *CM)

831

: InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, 1,

832

UnrollFactor, LVL, CM) {}

833

834

private:

835

void vectorizeMemoryInstruction(Instruction *Instr) override;

836

Value *getBroadcastInstrs(Value *V) override;

837

Value *getStepVector(Value *Val, int StartIdx, Value *Step,

838

Instruction::BinaryOps Opcode =

839

Instruction::BinaryOpsEnd) override;

840

Value *reverseVector(Value *Vec) override;

841

};

842

843

/// \brief Look for a meaningful debug location on the instruction or it's

844

/// operands.

845

static Instruction *getDebugLocFromInstOrOperands(Instruction *I) {

846

if (!I)

847

return I;

848

849

DebugLoc Empty;

850

if (I->getDebugLoc() != Empty)

851

return I;

852

853

for (User::op_iterator OI = I->op_begin(), OE = I->op_end(); OI != OE; ++OI) {

854

if (Instruction *OpInst = dyn_cast<Instruction>(*OI))

855

if (OpInst->getDebugLoc() != Empty)

856

return OpInst;

857

}

858

859

return I;

860

}

861

862

void InnerLoopVectorizer::setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr) {

863

if (const Instruction *Inst = dyn_cast_or_null<Instruction>(Ptr)) {

864

const DILocation *DIL = Inst->getDebugLoc();

865

if (DIL && Inst->getFunction()->isDebugInfoForProfiling())

866

B.SetCurrentDebugLocation(DIL->cloneWithDuplicationFactor(UF * VF));

867

else

868

B.SetCurrentDebugLocation(DIL);

869

} else

870

B.SetCurrentDebugLocation(DebugLoc());

871

}

872

873

#ifndef NDEBUG

874

/// \return string containing a file name and a line # for the given loop.

875

static std::string getDebugLocString(const Loop *L) {

876

std::string Result;

877

if (L) {

878

raw_string_ostream OS(Result);

879

if (const DebugLoc LoopDbgLoc = L->getStartLoc())

880

LoopDbgLoc.print(OS);

881

else

882

// Just print the module name.

883

OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier();

884

OS.flush();

885

}

886

return Result;

887

}

888

#endif

889

890

void InnerLoopVectorizer::addNewMetadata(Instruction *To,

891

const Instruction *Orig) {

892

// If the loop was versioned with memchecks, add the corresponding no-alias

893

// metadata.

894

if (LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))

895

LVer->annotateInstWithNoAlias(To, Orig);

896

}

897

898

void InnerLoopVectorizer::addMetadata(Instruction *To,

899

Instruction *From) {

900

propagateMetadata(To, From);

901

addNewMetadata(To, From);

902

}

903

904

void InnerLoopVectorizer::addMetadata(ArrayRef<Value *> To,

905

Instruction *From) {

906

for (Value *V : To) {

907

if (Instruction *I = dyn_cast<Instruction>(V))

908

addMetadata(I, From);

909

}

910

}

911

912

/// \brief The group of interleaved loads/stores sharing the same stride and

913

/// close to each other.

914

///

915

/// Each member in this group has an index starting from 0, and the largest

916

/// index should be less than interleaved factor, which is equal to the absolute

917

/// value of the access's stride.

918

///

919

/// E.g. An interleaved load group of factor 4:

920

/// for (unsigned i = 0; i < 1024; i+=4) {

921

/// a = A[i]; // Member of index 0

922

/// b = A[i+1]; // Member of index 1

923

/// d = A[i+3]; // Member of index 3

924

/// ...

925

/// }

926

///

927

/// An interleaved store group of factor 4:

928

/// for (unsigned i = 0; i < 1024; i+=4) {

929

/// ...

930

/// A[i] = a; // Member of index 0

931

/// A[i+1] = b; // Member of index 1

932

/// A[i+2] = c; // Member of index 2

933

/// A[i+3] = d; // Member of index 3

934

/// }

935

///

936

/// Note: the interleaved load group could have gaps (missing members), but

937

/// the interleaved store group doesn't allow gaps.

938

class InterleaveGroup {

939

public:

940

InterleaveGroup(Instruction *Instr, int Stride, unsigned Align)

941

: Align(Align), SmallestKey(0), LargestKey(0), InsertPos(Instr) {

942

assert(Align && "The alignment should be non-zero")((Align && "The alignment should be non-zero") ? static_cast
<void> (0) : __assert_fail ("Align && \"The alignment should be non-zero\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 942, __PRETTY_FUNCTION__));

943

944

Factor = std::abs(Stride);

945

assert(Factor > 1 && "Invalid interleave factor")((Factor > 1 && "Invalid interleave factor") ? static_cast
<void> (0) : __assert_fail ("Factor > 1 && \"Invalid interleave factor\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 945, __PRETTY_FUNCTION__));

946

947

Reverse = Stride < 0;

948

Members[0] = Instr;

949

}

950

951

bool isReverse() const { return Reverse; }

952

unsigned getFactor() const { return Factor; }

953

unsigned getAlignment() const { return Align; }

954

unsigned getNumMembers() const { return Members.size(); }

955

956

/// \brief Try to insert a new member \p Instr with index \p Index and

957

/// alignment \p NewAlign. The index is related to the leader and it could be

958

/// negative if it is the new leader.

959

///

960

/// \returns false if the instruction doesn't belong to the group.

961

bool insertMember(Instruction *Instr, int Index, unsigned NewAlign) {

962

assert(NewAlign && "The new member's alignment should be non-zero")((NewAlign && "The new member's alignment should be non-zero"
) ? static_cast<void> (0) : __assert_fail ("NewAlign && \"The new member's alignment should be non-zero\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 962, __PRETTY_FUNCTION__));

963

964

int Key = Index + SmallestKey;

965

966

// Skip if there is already a member with the same index.

967

if (Members.count(Key))

968

return false;

969

970

if (Key > LargestKey) {

971

// The largest index is always less than the interleave factor.

972

if (Index >= static_cast<int>(Factor))

973

return false;

974

975

LargestKey = Key;

976

} else if (Key < SmallestKey) {

977

// The largest index is always less than the interleave factor.

978

if (LargestKey - Key >= static_cast<int>(Factor))

979

return false;

980

981

SmallestKey = Key;

982

}

983

984

// It's always safe to select the minimum alignment.

985

Align = std::min(Align, NewAlign);

986

Members[Key] = Instr;

987

return true;

988

}

989

990

/// \brief Get the member with the given index \p Index

991

///

992

/// \returns nullptr if contains no such member.

993

Instruction *getMember(unsigned Index) const {

994

int Key = SmallestKey + Index;

995

if (!Members.count(Key))

996

return nullptr;

997

998

return Members.find(Key)->second;

999

}

1000

1001

/// \brief Get the index for the given member. Unlike the key in the member

1002

/// map, the index starts from 0.

1003

unsigned getIndex(Instruction *Instr) const {

1004

for (auto I : Members)

1005

if (I.second == Instr)

1006

return I.first - SmallestKey;

1007

1008

llvm_unreachable("InterleaveGroup contains no such member")::llvm::llvm_unreachable_internal("InterleaveGroup contains no such member"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1008);

1009

}

1010

1011

Instruction *getInsertPos() const { return InsertPos; }

1012

void setInsertPos(Instruction *Inst) { InsertPos = Inst; }

1013

1014

private:

1015

unsigned Factor; // Interleave Factor.

1016

bool Reverse;

1017

unsigned Align;

1018

DenseMap<int, Instruction *> Members;

1019

int SmallestKey;

1020

int LargestKey;

1021

1022

// To avoid breaking dependences, vectorized instructions of an interleave

1023

// group should be inserted at either the first load or the last store in

1024

// program order.

1025

1026

// E.g. %even = load i32 // Insert Position

1027

// %add = add i32 %even // Use of %even

1028

// %odd = load i32

1029

1030

// store i32 %even

1031

// %odd = add i32 // Def of %odd

1032

// store i32 %odd // Insert Position

1033

Instruction *InsertPos;

1034

};

1035

1036

/// \brief Drive the analysis of interleaved memory accesses in the loop.

1037

///

1038

/// Use this class to analyze interleaved accesses only when we can vectorize

1039

/// a loop. Otherwise it's meaningless to do analysis as the vectorization

1040

/// on interleaved accesses is unsafe.

1041

///

1042

/// The analysis collects interleave groups and records the relationships

1043

/// between the member and the group in a map.

1044

class InterleavedAccessInfo {

1045

public:

1046

InterleavedAccessInfo(PredicatedScalarEvolution &PSE, Loop *L,

1047

DominatorTree *DT, LoopInfo *LI)

1048

: PSE(PSE), TheLoop(L), DT(DT), LI(LI), LAI(nullptr),

1049

RequiresScalarEpilogue(false) {}

1050

1051

~InterleavedAccessInfo() {

1052

SmallSet<InterleaveGroup *, 4> DelSet;

1053

// Avoid releasing a pointer twice.

1054

for (auto &I : InterleaveGroupMap)

1055

DelSet.insert(I.second);

1056

for (auto *Ptr : DelSet)

1057

delete Ptr;

1058

}

1059

1060

/// \brief Analyze the interleaved accesses and collect them in interleave

1061

/// groups. Substitute symbolic strides using \p Strides.

1062

void analyzeInterleaving(const ValueToValueMap &Strides);

1063

1064

/// \brief Check if \p Instr belongs to any interleave group.

1065

bool isInterleaved(Instruction *Instr) const {

1066

return InterleaveGroupMap.count(Instr);

1067

}

1068

1069

/// \brief Return the maximum interleave factor of all interleaved groups.

1070

unsigned getMaxInterleaveFactor() const {

1071

unsigned MaxFactor = 1;

1072

for (auto &Entry : InterleaveGroupMap)

1073

MaxFactor = std::max(MaxFactor, Entry.second->getFactor());

1074

return MaxFactor;

1075

}

1076

1077

/// \brief Get the interleave group that \p Instr belongs to.

1078

///

1079

/// \returns nullptr if doesn't have such group.

1080

InterleaveGroup *getInterleaveGroup(Instruction *Instr) const {

1081

if (InterleaveGroupMap.count(Instr))

1082

return InterleaveGroupMap.find(Instr)->second;

1083

return nullptr;

1084

}

1085

1086

/// \brief Returns true if an interleaved group that may access memory

1087

/// out-of-bounds requires a scalar epilogue iteration for correctness.

1088

bool requiresScalarEpilogue() const { return RequiresScalarEpilogue; }

1089

1090

/// \brief Initialize the LoopAccessInfo used for dependence checking.

1091

void setLAI(const LoopAccessInfo *Info) { LAI = Info; }

1092

1093

private:

1094

/// A wrapper around ScalarEvolution, used to add runtime SCEV checks.

1095

/// Simplifies SCEV expressions in the context of existing SCEV assumptions.

1096

/// The interleaved access analysis can also add new predicates (for example

1097

/// by versioning strides of pointers).

1098

PredicatedScalarEvolution &PSE;

1099

Loop *TheLoop;

1100

DominatorTree *DT;

1101

LoopInfo *LI;

1102

const LoopAccessInfo *LAI;

1103

1104

/// True if the loop may contain non-reversed interleaved groups with

1105

/// out-of-bounds accesses. We ensure we don't speculatively access memory

1106

/// out-of-bounds by executing at least one scalar epilogue iteration.

1107

bool RequiresScalarEpilogue;

1108

1109

/// Holds the relationships between the members and the interleave group.

1110

DenseMap<Instruction *, InterleaveGroup *> InterleaveGroupMap;

1111

1112

/// Holds dependences among the memory accesses in the loop. It maps a source

1113

/// access to a set of dependent sink accesses.

1114

DenseMap<Instruction *, SmallPtrSet<Instruction *, 2>> Dependences;

1115

1116

/// \brief The descriptor for a strided memory access.

1117

struct StrideDescriptor {

1118

StrideDescriptor(int64_t Stride, const SCEV *Scev, uint64_t Size,

1119

unsigned Align)

1120

: Stride(Stride), Scev(Scev), Size(Size), Align(Align) {}

1121

1122

StrideDescriptor() = default;

1123

1124

// The access's stride. It is negative for a reverse access.

1125

int64_t Stride = 0;

1126

const SCEV *Scev = nullptr; // The scalar expression of this access

1127

uint64_t Size = 0; // The size of the memory object.

1128

unsigned Align = 0; // The alignment of this access.

1129

};

1130

1131

/// \brief A type for holding instructions and their stride descriptors.

1132

typedef std::pair<Instruction *, StrideDescriptor> StrideEntry;

1133

1134

/// \brief Create a new interleave group with the given instruction \p Instr,

1135

/// stride \p Stride and alignment \p Align.

1136

///

1137

/// \returns the newly created interleave group.

1138

InterleaveGroup *createInterleaveGroup(Instruction *Instr, int Stride,

1139

unsigned Align) {

1140

assert(!InterleaveGroupMap.count(Instr) &&((!InterleaveGroupMap.count(Instr) && "Already in an interleaved access group"
) ? static_cast<void> (0) : __assert_fail ("!InterleaveGroupMap.count(Instr) && \"Already in an interleaved access group\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1141, __PRETTY_FUNCTION__))

1141

"Already in an interleaved access group")((!InterleaveGroupMap.count(Instr) && "Already in an interleaved access group"
) ? static_cast<void> (0) : __assert_fail ("!InterleaveGroupMap.count(Instr) && \"Already in an interleaved access group\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1141, __PRETTY_FUNCTION__));

1142

InterleaveGroupMap[Instr] = new InterleaveGroup(Instr, Stride, Align);

1143

return InterleaveGroupMap[Instr];

1144

}

1145

1146

/// \brief Release the group and remove all the relationships.

1147

void releaseGroup(InterleaveGroup *Group) {

1148

for (unsigned i = 0; i < Group->getFactor(); i++)

1149

if (Instruction *Member = Group->getMember(i))

1150

InterleaveGroupMap.erase(Member);

1151

1152

delete Group;

1153

}

1154

1155

/// \brief Collect all the accesses with a constant stride in program order.

1156

void collectConstStrideAccesses(

1157

MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,

1158

const ValueToValueMap &Strides);

1159

1160

/// \brief Returns true if \p Stride is allowed in an interleaved group.

1161

static bool isStrided(int Stride) {

1162

unsigned Factor = std::abs(Stride);

1163

return Factor >= 2 && Factor <= MaxInterleaveGroupFactor;

1164

}

1165

1166

/// \brief Returns true if \p BB is a predicated block.

1167

bool isPredicated(BasicBlock *BB) const {

1168

return LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT);

1169

}

1170

1171

/// \brief Returns true if LoopAccessInfo can be used for dependence queries.

1172

bool areDependencesValid() const {

1173

return LAI && LAI->getDepChecker().getDependences();

1174

}

1175

1176

/// \brief Returns true if memory accesses \p A and \p B can be reordered, if

1177

/// necessary, when constructing interleaved groups.

1178

///

1179

/// \p A must precede \p B in program order. We return false if reordering is

1180

/// not necessary or is prevented because \p A and \p B may be dependent.

1181

bool canReorderMemAccessesForInterleavedGroups(StrideEntry *A,

1182

StrideEntry *B) const {

1183

1184

// Code motion for interleaved accesses can potentially hoist strided loads

1185

// and sink strided stores. The code below checks the legality of the

1186

// following two conditions:

1187

1188

// 1. Potentially moving a strided load (B) before any store (A) that

1189

// precedes B, or

1190

1191

// 2. Potentially moving a strided store (A) after any load or store (B)

1192

// that A precedes.

1193

1194

// It's legal to reorder A and B if we know there isn't a dependence from A

1195

// to B. Note that this determination is conservative since some

1196

// dependences could potentially be reordered safely.

1197

1198

// A is potentially the source of a dependence.

1199

auto *Src = A->first;

1200

auto SrcDes = A->second;

1201

1202

// B is potentially the sink of a dependence.

1203

auto *Sink = B->first;

1204

auto SinkDes = B->second;

1205

1206

// Code motion for interleaved accesses can't violate WAR dependences.

1207

// Thus, reordering is legal if the source isn't a write.

1208

if (!Src->mayWriteToMemory())

1209

return true;

1210

1211

// At least one of the accesses must be strided.

1212

if (!isStrided(SrcDes.Stride) && !isStrided(SinkDes.Stride))

1213

return true;

1214

1215

// If dependence information is not available from LoopAccessInfo,

1216

// conservatively assume the instructions can't be reordered.

1217

if (!areDependencesValid())

1218

return false;

1219

1220

// If we know there is a dependence from source to sink, assume the

1221

// instructions can't be reordered. Otherwise, reordering is legal.

1222

return !Dependences.count(Src) || !Dependences.lookup(Src).count(Sink);

1223

}

1224

1225

/// \brief Collect the dependences from LoopAccessInfo.

1226

///

1227

/// We process the dependences once during the interleaved access analysis to

1228

/// enable constant-time dependence queries.

1229

void collectDependences() {

1230

if (!areDependencesValid())

1231

return;

1232

auto *Deps = LAI->getDepChecker().getDependences();

1233

for (auto Dep : *Deps)

1234

Dependences[Dep.getSource(*LAI)].insert(Dep.getDestination(*LAI));

1235

}

1236

};

1237

1238

/// Utility class for getting and setting loop vectorizer hints in the form

1239

/// of loop metadata.

1240

/// This class keeps a number of loop annotations locally (as member variables)

1241

/// and can, upon request, write them back as metadata on the loop. It will

1242

/// initially scan the loop for existing metadata, and will update the local

1243

/// values based on information in the loop.

1244

/// We cannot write all values to metadata, as the mere presence of some info,

1245

/// for example 'force', means a decision has been made. So, we need to be

1246

/// careful NOT to add them if the user hasn't specifically asked so.

1247

class LoopVectorizeHints {

1248

enum HintKind { HK_WIDTH, HK_UNROLL, HK_FORCE };

1249

1250

/// Hint - associates name and validation with the hint value.

1251

struct Hint {

1252

const char *Name;

1253

unsigned Value; // This may have to change for non-numeric values.

1254

HintKind Kind;

1255

1256

Hint(const char *Name, unsigned Value, HintKind Kind)

1257

: Name(Name), Value(Value), Kind(Kind) {}

1258

1259

bool validate(unsigned Val) {

1260

switch (Kind) {

1261

case HK_WIDTH:

1262

return isPowerOf2_32(Val) && Val <= VectorizerParams::MaxVectorWidth;

1263

case HK_UNROLL:

1264

return isPowerOf2_32(Val) && Val <= MaxInterleaveFactor;

1265

case HK_FORCE:

1266

return (Val <= 1);

1267

}

1268

return false;

1269

}

1270

};

1271

1272

/// Vectorization width.

1273

Hint Width;

1274

/// Vectorization interleave factor.

1275

Hint Interleave;

1276

/// Vectorization forced

1277

Hint Force;

1278

1279

/// Return the loop metadata prefix.

1280

static StringRef Prefix() { return "llvm.loop."; }

1281

1282

/// True if there is any unsafe math in the loop.

1283

bool PotentiallyUnsafe;

1284

1285

public:

1286

enum ForceKind {

1287

FK_Undefined = -1, ///< Not selected.

1288

FK_Disabled = 0, ///< Forcing disabled.

1289

FK_Enabled = 1, ///< Forcing enabled.

1290

};

1291

1292

LoopVectorizeHints(const Loop *L, bool DisableInterleaving,

1293

OptimizationRemarkEmitter &ORE)

1294

: Width("vectorize.width", VectorizerParams::VectorizationFactor,

1295

HK_WIDTH),

1296

Interleave("interleave.count", DisableInterleaving, HK_UNROLL),

1297

Force("vectorize.enable", FK_Undefined, HK_FORCE),

1298

PotentiallyUnsafe(false), TheLoop(L), ORE(ORE) {

1299

// Populate values with existing loop metadata.

1300

getHintsFromMetadata();

1301

1302

// force-vector-interleave overrides DisableInterleaving.

1303

if (VectorizerParams::isInterleaveForced())

1304

Interleave.Value = VectorizerParams::VectorizationInterleave;

1305

1306

DEBUG(if (DisableInterleaving && Interleave.Value == 1) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (DisableInterleaving && Interleave
.Value == 1) dbgs() << "LV: Interleaving disabled by the pass manager\n"
; } } while (false)

1307

<< "LV: Interleaving disabled by the pass manager\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (DisableInterleaving && Interleave
.Value == 1) dbgs() << "LV: Interleaving disabled by the pass manager\n"
; } } while (false);

1308

}

1309

1310

/// Mark the loop L as already vectorized by setting the width to 1.

1311

void setAlreadyVectorized() {

1312

Width.Value = Interleave.Value = 1;

1313

Hint Hints[] = {Width, Interleave};

1314

writeHintsToMetadata(Hints);

1315

}

1316

1317

bool allowVectorization(Function *F, Loop *L, bool AlwaysVectorize) const {

1318

if (getForce() == LoopVectorizeHints::FK_Disabled) {

1319

DEBUG(dbgs() << "LV: Not vectorizing: #pragma vectorize disable.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: #pragma vectorize disable.\n"
; } } while (false);

1320

emitRemarkWithHints();

1321

return false;

1322

}

1323

1324

if (!AlwaysVectorize && getForce() != LoopVectorizeHints::FK_Enabled) {

1325

DEBUG(dbgs() << "LV: Not vectorizing: No #pragma vectorize enable.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: No #pragma vectorize enable.\n"
; } } while (false);

1326

emitRemarkWithHints();

1327

return false;

1328

}

1329

1330

if (getWidth() == 1 && getInterleave() == 1) {

1331

// FIXME: Add a separate metadata to indicate when the loop has already

1332

// been vectorized instead of setting width and count to 1.

1333

DEBUG(dbgs() << "LV: Not vectorizing: Disabled/already vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Disabled/already vectorized.\n"
; } } while (false);

1334

// FIXME: Add interleave.disable metadata. This will allow

1335

// vectorize.disable to be used without disabling the pass and errors

1336

// to differentiate between disabled vectorization and a width of 1.

1337

ORE.emit(OptimizationRemarkAnalysis(vectorizeAnalysisPassName(),

1338

"AllDisabled", L->getStartLoc(),

1339

L->getHeader())

1340

<< "loop not vectorized: vectorization and interleaving are "

1341

"explicitly disabled, or vectorize width and interleave "

1342

"count are both set to 1");

1343

return false;

1344

}

1345

1346

return true;

1347

}

1348

1349

/// Dumps all the hint information.

1350

void emitRemarkWithHints() const {

1351

using namespace ore;

1352

if (Force.Value == LoopVectorizeHints::FK_Disabled)

1353

ORE.emit(OptimizationRemarkMissed(LV_NAME"loop-vectorize", "MissedExplicitlyDisabled",

1354

TheLoop->getStartLoc(),

1355

TheLoop->getHeader())

1356

<< "loop not vectorized: vectorization is explicitly disabled");

1357

else {

1358

OptimizationRemarkMissed R(LV_NAME"loop-vectorize", "MissedDetails",

1359

TheLoop->getStartLoc(), TheLoop->getHeader());

1360

R << "loop not vectorized";

1361

if (Force.Value == LoopVectorizeHints::FK_Enabled) {

1362

R << " (Force=" << NV("Force", true);

1363

if (Width.Value != 0)

1364

R << ", Vector Width=" << NV("VectorWidth", Width.Value);

1365

if (Interleave.Value != 0)

1366

R << ", Interleave Count=" << NV("InterleaveCount", Interleave.Value);

1367

R << ")";

1368

}

1369

ORE.emit(R);

1370

}

1371

}

1372

1373

unsigned getWidth() const { return Width.Value; }

1374

unsigned getInterleave() const { return Interleave.Value; }

1375

enum ForceKind getForce() const { return (ForceKind)Force.Value; }

1376

1377

/// \brief If hints are provided that force vectorization, use the AlwaysPrint

1378

/// pass name to force the frontend to print the diagnostic.

1379

const char *vectorizeAnalysisPassName() const {

1380

if (getWidth() == 1)

1381

return LV_NAME"loop-vectorize";

1382

if (getForce() == LoopVectorizeHints::FK_Disabled)

1383

return LV_NAME"loop-vectorize";

1384

if (getForce() == LoopVectorizeHints::FK_Undefined && getWidth() == 0)

1385

return LV_NAME"loop-vectorize";

1386

return OptimizationRemarkAnalysis::AlwaysPrint;

1387

}

1388

1389

bool allowReordering() const {

1390

// When enabling loop hints are provided we allow the vectorizer to change

1391

// the order of operations that is given by the scalar loop. This is not

1392

// enabled by default because can be unsafe or inefficient. For example,

1393

// reordering floating-point operations will change the way round-off

1394

// error accumulates in the loop.

1395

return getForce() == LoopVectorizeHints::FK_Enabled || getWidth() > 1;

1396

}

1397

1398

bool isPotentiallyUnsafe() const {

1399

// Avoid FP vectorization if the target is unsure about proper support.

1400

// This may be related to the SIMD unit in the target not handling

1401

// IEEE 754 FP ops properly, or bad single-to-double promotions.

1402

// Otherwise, a sequence of vectorized loops, even without reduction,

1403

// could lead to different end results on the destination vectors.

1404

return getForce() != LoopVectorizeHints::FK_Enabled && PotentiallyUnsafe;

1405

}

1406

1407

void setPotentiallyUnsafe() { PotentiallyUnsafe = true; }

1408

1409

private:

1410

/// Find hints specified in the loop metadata and update local values.

1411

void getHintsFromMetadata() {

1412

MDNode *LoopID = TheLoop->getLoopID();

1413

if (!LoopID)

1414

return;

1415

1416

// First operand should refer to the loop id itself.

1417

assert(LoopID->getNumOperands() > 0 && "requires at least one operand")((LoopID->getNumOperands() > 0 && "requires at least one operand"
) ? static_cast<void> (0) : __assert_fail ("LoopID->getNumOperands() > 0 && \"requires at least one operand\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1417, __PRETTY_FUNCTION__));

1418

assert(LoopID->getOperand(0) == LoopID && "invalid loop id")((LoopID->getOperand(0) == LoopID && "invalid loop id"
) ? static_cast<void> (0) : __assert_fail ("LoopID->getOperand(0) == LoopID && \"invalid loop id\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1418, __PRETTY_FUNCTION__));

1419

1420

for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {

1421

const MDString *S = nullptr;

1422

SmallVector<Metadata *, 4> Args;

1423

1424

// The expected hint is either a MDString or a MDNode with the first

1425

// operand a MDString.

1426

if (const MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i))) {

1427

if (!MD || MD->getNumOperands() == 0)

1428

continue;

1429

S = dyn_cast<MDString>(MD->getOperand(0));

1430

for (unsigned i = 1, ie = MD->getNumOperands(); i < ie; ++i)

1431

Args.push_back(MD->getOperand(i));

1432

} else {

1433

S = dyn_cast<MDString>(LoopID->getOperand(i));

1434

assert(Args.size() == 0 && "too many arguments for MDString")((Args.size() == 0 && "too many arguments for MDString"
) ? static_cast<void> (0) : __assert_fail ("Args.size() == 0 && \"too many arguments for MDString\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1434, __PRETTY_FUNCTION__));

1435

}

1436

1437

if (!S)

1438

continue;

1439

1440

// Check if the hint starts with the loop metadata prefix.

1441

StringRef Name = S->getString();

1442

if (Args.size() == 1)

1443

setHint(Name, Args[0]);

1444

}

1445

}

1446

1447

/// Checks string hint with one operand and set value if valid.

1448

void setHint(StringRef Name, Metadata *Arg) {

1449

if (!Name.startswith(Prefix()))

1450

return;

1451

Name = Name.substr(Prefix().size(), StringRef::npos);

1452

1453

const ConstantInt *C = mdconst::dyn_extract<ConstantInt>(Arg);

1454

if (!C)

1455

return;

1456

unsigned Val = C->getZExtValue();

1457

1458

Hint *Hints[] = {&Width, &Interleave, &Force};

1459

for (auto H : Hints) {

1460

if (Name == H->Name) {

1461

if (H->validate(Val))

1462

H->Value = Val;

1463

else

1464

DEBUG(dbgs() << "LV: ignoring invalid hint '" << Name << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: ignoring invalid hint '"
<< Name << "'\n"; } } while (false);

1465

break;

1466

}

1467

}

1468

}

1469

1470

/// Create a new hint from name / value pair.

1471

MDNode *createHintMetadata(StringRef Name, unsigned V) const {

1472

LLVMContext &Context = TheLoop->getHeader()->getContext();

1473

Metadata *MDs[] = {MDString::get(Context, Name),

1474

ConstantAsMetadata::get(

1475

ConstantInt::get(Type::getInt32Ty(Context), V))};

1476

return MDNode::get(Context, MDs);

1477

}

1478

1479

/// Matches metadata with hint name.

1480

bool matchesHintMetadataName(MDNode *Node, ArrayRef<Hint> HintTypes) {

1481

MDString *Name = dyn_cast<MDString>(Node->getOperand(0));

1482

if (!Name)

1483

return false;

1484

1485

for (auto H : HintTypes)

1486

if (Name->getString().endswith(H.Name))

1487

return true;

1488

return false;

1489

}

1490

1491

/// Sets current hints into loop metadata, keeping other values intact.

1492

void writeHintsToMetadata(ArrayRef<Hint> HintTypes) {

1493

if (HintTypes.size() == 0)

1494

return;

1495

1496

// Reserve the first element to LoopID (see below).

1497

SmallVector<Metadata *, 4> MDs(1);

1498

// If the loop already has metadata, then ignore the existing operands.

1499

MDNode *LoopID = TheLoop->getLoopID();

1500

if (LoopID) {

1501

for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {

1502

MDNode *Node = cast<MDNode>(LoopID->getOperand(i));

1503

// If node in update list, ignore old value.

1504

if (!matchesHintMetadataName(Node, HintTypes))

1505

MDs.push_back(Node);

1506

}

1507

}

1508

1509

// Now, add the missing hints.

1510

for (auto H : HintTypes)

1511

MDs.push_back(createHintMetadata(Twine(Prefix(), H.Name).str(), H.Value));

1512

1513

// Replace current metadata node with new one.

1514

LLVMContext &Context = TheLoop->getHeader()->getContext();

1515

MDNode *NewLoopID = MDNode::get(Context, MDs);

1516

// Set operand 0 to refer to the loop id itself.

1517

NewLoopID->replaceOperandWith(0, NewLoopID);

1518

1519

TheLoop->setLoopID(NewLoopID);

1520

}

1521

1522

/// The loop these hints belong to.

1523

const Loop *TheLoop;

1524

1525

/// Interface to emit optimization remarks.

1526

OptimizationRemarkEmitter &ORE;

1527

};

1528

1529

static void emitMissedWarning(Function *F, Loop *L,

1530

const LoopVectorizeHints &LH,

1531

OptimizationRemarkEmitter *ORE) {

1532

LH.emitRemarkWithHints();

1533

1534

if (LH.getForce() == LoopVectorizeHints::FK_Enabled) {

1535

if (LH.getWidth() != 1)

1536

ORE->emit(DiagnosticInfoOptimizationFailure(

1537

DEBUG_TYPE"loop-vectorize", "FailedRequestedVectorization",

1538

L->getStartLoc(), L->getHeader())

1539

<< "loop not vectorized: "

1540

<< "failed explicitly specified loop vectorization");

1541

else if (LH.getInterleave() != 1)

1542

ORE->emit(DiagnosticInfoOptimizationFailure(

1543

DEBUG_TYPE"loop-vectorize", "FailedRequestedInterleaving", L->getStartLoc(),

1544

L->getHeader())

1545

<< "loop not interleaved: "

1546

<< "failed explicitly specified loop interleaving");

1547

}

1548

}

1549

1550

/// LoopVectorizationLegality checks if it is legal to vectorize a loop, and

1551

/// to what vectorization factor.

1552

/// This class does not look at the profitability of vectorization, only the

1553

/// legality. This class has two main kinds of checks:

1554

/// * Memory checks - The code in canVectorizeMemory checks if vectorization

1555

/// will change the order of memory accesses in a way that will change the

1556

/// correctness of the program.

1557

/// * Scalars checks - The code in canVectorizeInstrs and canVectorizeMemory

1558

/// checks for a number of different conditions, such as the availability of a

1559

/// single induction variable, that all types are supported and vectorize-able,

1560

/// etc. This code reflects the capabilities of InnerLoopVectorizer.

1561

/// This class is also used by InnerLoopVectorizer for identifying

1562

/// induction variable and the different reduction variables.

1563

class LoopVectorizationLegality {

1564

public:

1565

LoopVectorizationLegality(

1566

Loop *L, PredicatedScalarEvolution &PSE, DominatorTree *DT,

1567

TargetLibraryInfo *TLI, AliasAnalysis *AA, Function *F,

1568

const TargetTransformInfo *TTI,

1569

std::function<const LoopAccessInfo &(Loop &)> *GetLAA, LoopInfo *LI,

1570

OptimizationRemarkEmitter *ORE, LoopVectorizationRequirements *R,

1571

LoopVectorizeHints *H)

1572

: NumPredStores(0), TheLoop(L), PSE(PSE), TLI(TLI), TTI(TTI), DT(DT),

1573

GetLAA(GetLAA), LAI(nullptr), ORE(ORE), InterleaveInfo(PSE, L, DT, LI),

1574

PrimaryInduction(nullptr), WidestIndTy(nullptr), HasFunNoNaNAttr(false),

1575

Requirements(R), Hints(H) {}

1576

1577

/// ReductionList contains the reduction descriptors for all

1578

/// of the reductions that were found in the loop.

1579

typedef DenseMap<PHINode *, RecurrenceDescriptor> ReductionList;

1580

1581

/// InductionList saves induction variables and maps them to the

1582

/// induction descriptor.

1583

typedef MapVector<PHINode *, InductionDescriptor> InductionList;

1584

1585

/// RecurrenceSet contains the phi nodes that are recurrences other than

1586

/// inductions and reductions.

1587

typedef SmallPtrSet<const PHINode *, 8> RecurrenceSet;

1588

1589

/// Returns true if it is legal to vectorize this loop.

1590

/// This does not mean that it is profitable to vectorize this

1591

/// loop, only that it is legal to do so.

1592

bool canVectorize();

1593

1594

/// Returns the primary induction variable.

1595

PHINode *getPrimaryInduction() { return PrimaryInduction; }

1596

1597

/// Returns the reduction variables found in the loop.

1598

ReductionList *getReductionVars() { return &Reductions; }

1599

1600

/// Returns the induction variables found in the loop.

1601

InductionList *getInductionVars() { return &Inductions; }

1602

1603

/// Return the first-order recurrences found in the loop.

1604

RecurrenceSet *getFirstOrderRecurrences() { return &FirstOrderRecurrences; }

1605

1606

/// Returns the widest induction type.

1607

Type *getWidestInductionType() { return WidestIndTy; }

1608

1609

/// Returns True if V is an induction variable in this loop.

1610

bool isInductionVariable(const Value *V);

1611

1612

/// Returns True if PN is a reduction variable in this loop.

1613

bool isReductionVariable(PHINode *PN) { return Reductions.count(PN); }

1614

1615

/// Returns True if Phi is a first-order recurrence in this loop.

1616

bool isFirstOrderRecurrence(const PHINode *Phi);

1617

1618

/// Return true if the block BB needs to be predicated in order for the loop

1619

/// to be vectorized.

1620

bool blockNeedsPredication(BasicBlock *BB);

1621

1622

/// Check if this pointer is consecutive when vectorizing. This happens

1623

/// when the last index of the GEP is the induction variable, or that the

1624

/// pointer itself is an induction variable.

1625

/// This check allows us to vectorize A[idx] into a wide load/store.

1626

/// Returns:

1627

/// 0 - Stride is unknown or non-consecutive.

1628

/// 1 - Address is consecutive.

1629

/// -1 - Address is consecutive, and decreasing.

1630

int isConsecutivePtr(Value *Ptr);

1631

1632

/// Returns true if the value V is uniform within the loop.

1633

bool isUniform(Value *V);

1634

1635

/// Returns the information that we collected about runtime memory check.

1636

const RuntimePointerChecking *getRuntimePointerChecking() const {

1637

return LAI->getRuntimePointerChecking();

1638

}

1639

1640

const LoopAccessInfo *getLAI() const { return LAI; }

1641

1642

/// \brief Check if \p Instr belongs to any interleaved access group.

1643

bool isAccessInterleaved(Instruction *Instr) {

1644

return InterleaveInfo.isInterleaved(Instr);

1645

}

1646

1647

/// \brief Return the maximum interleave factor of all interleaved groups.

1648

unsigned getMaxInterleaveFactor() const {

1649

return InterleaveInfo.getMaxInterleaveFactor();

1650

}

1651

1652

/// \brief Get the interleaved access group that \p Instr belongs to.

1653

const InterleaveGroup *getInterleavedAccessGroup(Instruction *Instr) {

1654

return InterleaveInfo.getInterleaveGroup(Instr);

1655

}

1656

1657

/// \brief Returns true if an interleaved group requires a scalar iteration

1658

/// to handle accesses with gaps.

1659

bool requiresScalarEpilogue() const {

1660

return InterleaveInfo.requiresScalarEpilogue();

1661

}

1662

1663

unsigned getMaxSafeDepDistBytes() { return LAI->getMaxSafeDepDistBytes(); }

1664

1665

bool hasStride(Value *V) { return LAI->hasStride(V); }

1666

1667

/// Returns true if the target machine supports masked store operation

1668

/// for the given \p DataType and kind of access to \p Ptr.

1669

bool isLegalMaskedStore(Type *DataType, Value *Ptr) {

1670

return isConsecutivePtr(Ptr) && TTI->isLegalMaskedStore(DataType);

1671

}

1672

/// Returns true if the target machine supports masked load operation

1673

/// for the given \p DataType and kind of access to \p Ptr.

1674

bool isLegalMaskedLoad(Type *DataType, Value *Ptr) {

1675

return isConsecutivePtr(Ptr) && TTI->isLegalMaskedLoad(DataType);

1676

}

1677

/// Returns true if the target machine supports masked scatter operation

1678

/// for the given \p DataType.

1679

bool isLegalMaskedScatter(Type *DataType) {

1680

return TTI->isLegalMaskedScatter(DataType);

1681

}

1682

/// Returns true if the target machine supports masked gather operation

1683

/// for the given \p DataType.

1684

bool isLegalMaskedGather(Type *DataType) {

1685

return TTI->isLegalMaskedGather(DataType);

1686

}

1687

/// Returns true if the target machine can represent \p V as a masked gather

1688

/// or scatter operation.

1689

bool isLegalGatherOrScatter(Value *V) {

1690

auto *LI = dyn_cast<LoadInst>(V);

1691

auto *SI = dyn_cast<StoreInst>(V);

1692

if (!LI && !SI)

1693

return false;

1694

auto *Ptr = getPointerOperand(V);

1695

auto *Ty = cast<PointerType>(Ptr->getType())->getElementType();

1696

return (LI && isLegalMaskedGather(Ty)) || (SI && isLegalMaskedScatter(Ty));

1697

}

1698

1699

/// Returns true if vector representation of the instruction \p I

1700

/// requires mask.

1701

bool isMaskRequired(const Instruction *I) { return (MaskedOp.count(I) != 0); }

1702

unsigned getNumStores() const { return LAI->getNumStores(); }

1703

unsigned getNumLoads() const { return LAI->getNumLoads(); }

1704

unsigned getNumPredStores() const { return NumPredStores; }

1705

1706

/// Returns true if \p I is an instruction that will be scalarized with

1707

/// predication. Such instructions include conditional stores and

1708

/// instructions that may divide by zero.

1709

bool isScalarWithPredication(Instruction *I);

1710

1711

/// Returns true if \p I is a memory instruction with consecutive memory

1712

/// access that can be widened.

1713

bool memoryInstructionCanBeWidened(Instruction *I, unsigned VF = 1);

1714

1715

// Returns true if the NoNaN attribute is set on the function.

1716

bool hasFunNoNaNAttr() const { return HasFunNoNaNAttr; }

1717

1718

private:

1719

/// Check if a single basic block loop is vectorizable.

1720

/// At this point we know that this is a loop with a constant trip count

1721

/// and we only need to check individual instructions.

1722

bool canVectorizeInstrs();

1723

1724

/// When we vectorize loops we may change the order in which

1725

/// we read and write from memory. This method checks if it is

1726

/// legal to vectorize the code, considering only memory constrains.

1727

/// Returns true if the loop is vectorizable

1728

bool canVectorizeMemory();

1729

1730

/// Return true if we can vectorize this loop using the IF-conversion

1731

/// transformation.

1732

bool canVectorizeWithIfConvert();

1733

1734

/// Return true if all of the instructions in the block can be speculatively

1735

/// executed. \p SafePtrs is a list of addresses that are known to be legal

1736

/// and we know that we can read from them without segfault.

1737

bool blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs);

1738

1739

/// Updates the vectorization state by adding \p Phi to the inductions list.

1740

/// This can set \p Phi as the main induction of the loop if \p Phi is a

1741

/// better choice for the main induction than the existing one.

1742

void addInductionPhi(PHINode *Phi, const InductionDescriptor &ID,

1743

SmallPtrSetImpl<Value *> &AllowedExit);

1744

1745

/// Create an analysis remark that explains why vectorization failed

1746

///

1747

/// \p RemarkName is the identifier for the remark. If \p I is passed it is

1748

/// an instruction that prevents vectorization. Otherwise the loop is used

1749

/// for the location of the remark. \return the remark object that can be

1750

/// streamed to.

1751

OptimizationRemarkAnalysis

1752

createMissedAnalysis(StringRef RemarkName, Instruction *I = nullptr) const {

1753

return ::createMissedAnalysis(Hints->vectorizeAnalysisPassName(),

1754

RemarkName, TheLoop, I);

1755

}

1756

1757

/// \brief If an access has a symbolic strides, this maps the pointer value to

1758

/// the stride symbol.

1759

const ValueToValueMap *getSymbolicStrides() {

1760

// FIXME: Currently, the set of symbolic strides is sometimes queried before

1761

// it's collected. This happens from canVectorizeWithIfConvert, when the

1762

// pointer is checked to reference consecutive elements suitable for a

1763

// masked access.

1764

return LAI ? &LAI->getSymbolicStrides() : nullptr;

1765

}

1766

1767

unsigned NumPredStores;

1768

1769

/// The loop that we evaluate.

1770

Loop *TheLoop;

1771

/// A wrapper around ScalarEvolution used to add runtime SCEV checks.

1772

/// Applies dynamic knowledge to simplify SCEV expressions in the context

1773

/// of existing SCEV assumptions. The analysis will also add a minimal set

1774

/// of new predicates if this is required to enable vectorization and

1775

/// unrolling.

1776

PredicatedScalarEvolution &PSE;

1777

/// Target Library Info.

1778

TargetLibraryInfo *TLI;

1779

/// Target Transform Info

1780

const TargetTransformInfo *TTI;

1781

/// Dominator Tree.

1782

DominatorTree *DT;

1783

// LoopAccess analysis.

1784

std::function<const LoopAccessInfo &(Loop &)> *GetLAA;

1785

// And the loop-accesses info corresponding to this loop. This pointer is

1786

// null until canVectorizeMemory sets it up.

1787

const LoopAccessInfo *LAI;

1788

/// Interface to emit optimization remarks.

1789

OptimizationRemarkEmitter *ORE;

1790

1791

/// The interleave access information contains groups of interleaved accesses

1792

/// with the same stride and close to each other.

1793

InterleavedAccessInfo InterleaveInfo;

1794

1795

// --- vectorization state --- //

1796

1797

/// Holds the primary induction variable. This is the counter of the

1798

/// loop.

1799

PHINode *PrimaryInduction;

1800

/// Holds the reduction variables.

1801

ReductionList Reductions;

1802

/// Holds all of the induction variables that we found in the loop.

1803

/// Notice that inductions don't need to start at zero and that induction

1804

/// variables can be pointers.

1805

InductionList Inductions;

1806

/// Holds the phi nodes that are first-order recurrences.

1807

RecurrenceSet FirstOrderRecurrences;

1808

/// Holds the widest induction type encountered.

1809

Type *WidestIndTy;

1810

1811

/// Allowed outside users. This holds the induction and reduction

1812

/// vars which can be accessed from outside the loop.

1813

SmallPtrSet<Value *, 4> AllowedExit;

1814

1815

/// Can we assume the absence of NaNs.

1816

bool HasFunNoNaNAttr;

1817

1818

/// Vectorization requirements that will go through late-evaluation.

1819

LoopVectorizationRequirements *Requirements;

1820

1821

/// Used to emit an analysis of any legality issues.

1822

LoopVectorizeHints *Hints;

1823

1824

/// While vectorizing these instructions we have to generate a

1825

/// call to the appropriate masked intrinsic

1826

SmallPtrSet<const Instruction *, 8> MaskedOp;

1827

};

1828

1829

/// LoopVectorizationCostModel - estimates the expected speedups due to

1830

/// vectorization.

1831

/// In many cases vectorization is not profitable. This can happen because of

1832

/// a number of reasons. In this class we mainly attempt to predict the

1833

/// expected speedup/slowdowns due to the supported instruction set. We use the

1834

/// TargetTransformInfo to query the different backends for the cost of

1835

/// different operations.

1836

class LoopVectorizationCostModel {

1837

public:

1838

LoopVectorizationCostModel(Loop *L, PredicatedScalarEvolution &PSE,

1839

LoopInfo *LI, LoopVectorizationLegality *Legal,

1840

const TargetTransformInfo &TTI,

1841

const TargetLibraryInfo *TLI, DemandedBits *DB,

1842

AssumptionCache *AC,

1843

OptimizationRemarkEmitter *ORE, const Function *F,

1844

const LoopVectorizeHints *Hints)

1845

: TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), DB(DB),

1846

AC(AC), ORE(ORE), TheFunction(F), Hints(Hints) {}

1847

1848

/// \return An upper bound for the vectorization factor, or None if

1849

/// vectorization should be avoided up front.

1850

Optional<unsigned> computeMaxVF(bool OptForSize);

1851

1852

/// Information about vectorization costs

1853

struct VectorizationFactor {

1854

unsigned Width; // Vector width with best cost

1855

unsigned Cost; // Cost of the loop with that width

1856

};

1857

/// \return The most profitable vectorization factor and the cost of that VF.

1858

/// This method checks every power of two up to MaxVF. If UserVF is not ZERO

1859

/// then this vectorization factor will be selected if vectorization is

1860

/// possible.

1861

VectorizationFactor selectVectorizationFactor(unsigned MaxVF);

1862

1863

/// Setup cost-based decisions for user vectorization factor.

1864

void selectUserVectorizationFactor(unsigned UserVF) {

1865

collectUniformsAndScalars(UserVF);

1866

collectInstsToScalarize(UserVF);

1867

}

1868

1869

/// \return The size (in bits) of the smallest and widest types in the code

1870

/// that needs to be vectorized. We ignore values that remain scalar such as

1871

/// 64 bit loop indices.

1872

std::pair<unsigned, unsigned> getSmallestAndWidestTypes();

1873

1874

/// \return The desired interleave count.

1875

/// If interleave count has been specified by metadata it will be returned.

1876

/// Otherwise, the interleave count is computed and returned. VF and LoopCost

1877

/// are the selected vectorization factor and the cost of the selected VF.

1878

unsigned selectInterleaveCount(bool OptForSize, unsigned VF,

1879

unsigned LoopCost);

1880

1881

/// Memory access instruction may be vectorized in more than one way.

1882

/// Form of instruction after vectorization depends on cost.

1883

/// This function takes cost-based decisions for Load/Store instructions

1884

/// and collects them in a map. This decisions map is used for building

1885

/// the lists of loop-uniform and loop-scalar instructions.

1886

/// The calculated cost is saved with widening decision in order to

1887

/// avoid redundant calculations.

1888

void setCostBasedWideningDecision(unsigned VF);

1889

1890

/// \brief A struct that represents some properties of the register usage

1891

/// of a loop.

1892

struct RegisterUsage {

1893

/// Holds the number of loop invariant values that are used in the loop.

1894

unsigned LoopInvariantRegs;

1895

/// Holds the maximum number of concurrent live intervals in the loop.

1896

unsigned MaxLocalUsers;

1897

/// Holds the number of instructions in the loop.

1898

unsigned NumInstructions;

1899

};

1900

1901

/// \return Returns information about the register usages of the loop for the

1902

/// given vectorization factors.

1903

SmallVector<RegisterUsage, 8> calculateRegisterUsage(ArrayRef<unsigned> VFs);

1904

1905

/// Collect values we want to ignore in the cost model.

1906

void collectValuesToIgnore();

1907

1908

/// \returns The smallest bitwidth each instruction can be represented with.

1909

/// The vector equivalents of these instructions should be truncated to this

1910

/// type.

1911

const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const {

1912

return MinBWs;

1913

}

1914

1915

/// \returns True if it is more profitable to scalarize instruction \p I for

1916

/// vectorization factor \p VF.

1917

bool isProfitableToScalarize(Instruction *I, unsigned VF) const {

1918

auto Scalars = InstsToScalarize.find(VF);

1919

assert(Scalars != InstsToScalarize.end() &&((Scalars != InstsToScalarize.end() && "VF not yet analyzed for scalarization profitability"
) ? static_cast<void> (0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1920, __PRETTY_FUNCTION__))

1920

"VF not yet analyzed for scalarization profitability")((Scalars != InstsToScalarize.end() && "VF not yet analyzed for scalarization profitability"
) ? static_cast<void> (0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1920, __PRETTY_FUNCTION__));

1921

return Scalars->second.count(I);

1922

}

1923

1924

/// Returns true if \p I is known to be uniform after vectorization.

1925

bool isUniformAfterVectorization(Instruction *I, unsigned VF) const {

1926

if (VF == 1)

1927

return true;

1928

assert(Uniforms.count(VF) && "VF not yet analyzed for uniformity")((Uniforms.count(VF) && "VF not yet analyzed for uniformity"
) ? static_cast<void> (0) : __assert_fail ("Uniforms.count(VF) && \"VF not yet analyzed for uniformity\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1928, __PRETTY_FUNCTION__));

1929

auto UniformsPerVF = Uniforms.find(VF);

1930

return UniformsPerVF->second.count(I);

1931

}

1932

1933

/// Returns true if \p I is known to be scalar after vectorization.

1934

bool isScalarAfterVectorization(Instruction *I, unsigned VF) const {

1935

if (VF == 1)

1936

return true;

1937

assert(Scalars.count(VF) && "Scalar values are not calculated for VF")((Scalars.count(VF) && "Scalar values are not calculated for VF"
) ? static_cast<void> (0) : __assert_fail ("Scalars.count(VF) && \"Scalar values are not calculated for VF\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1937, __PRETTY_FUNCTION__));

1938

auto ScalarsPerVF = Scalars.find(VF);

1939

return ScalarsPerVF->second.count(I);

1940

}

1941

1942

/// \returns True if instruction \p I can be truncated to a smaller bitwidth

1943

/// for vectorization factor \p VF.

1944

bool canTruncateToMinimalBitwidth(Instruction *I, unsigned VF) const {

1945

return VF > 1 && MinBWs.count(I) && !isProfitableToScalarize(I, VF) &&

1946

!isScalarAfterVectorization(I, VF);

1947

}

1948

1949

/// Decision that was taken during cost calculation for memory instruction.

1950

enum InstWidening {

1951

CM_Unknown,

1952

CM_Widen,

1953

CM_Interleave,

1954

CM_GatherScatter,

1955

CM_Scalarize

1956

};

1957

1958

/// Save vectorization decision \p W and \p Cost taken by the cost model for

1959

/// instruction \p I and vector width \p VF.

1960

void setWideningDecision(Instruction *I, unsigned VF, InstWidening W,

1961

unsigned Cost) {

1962

assert(VF >= 2 && "Expected VF >=2")((VF >= 2 && "Expected VF >=2") ? static_cast<
void> (0) : __assert_fail ("VF >= 2 && \"Expected VF >=2\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1962, __PRETTY_FUNCTION__));

1963

WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);

1964

}

1965

1966

/// Save vectorization decision \p W and \p Cost taken by the cost model for

1967

/// interleaving group \p Grp and vector width \p VF.

1968

void setWideningDecision(const InterleaveGroup *Grp, unsigned VF,

1969

InstWidening W, unsigned Cost) {

1970

1971

/// Broadcast this decicion to all instructions inside the group.

1972

/// But the cost will be assigned to one instruction only.

1973

for (unsigned i = 0; i < Grp->getFactor(); ++i) {

1974

if (auto *I = Grp->getMember(i)) {

1975

if (Grp->getInsertPos() == I)

1976

WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);

1977

else

1978

WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, 0);

1979

}

1980

}

1981

}

1982

1983

/// Return the cost model decision for the given instruction \p I and vector

1984

/// width \p VF. Return CM_Unknown if this instruction did not pass

1985

/// through the cost modeling.

1986

InstWidening getWideningDecision(Instruction *I, unsigned VF) {

1987

1988

std::pair<Instruction *, unsigned> InstOnVF = std::make_pair(I, VF);

1989

auto Itr = WideningDecisions.find(InstOnVF);

1990

if (Itr == WideningDecisions.end())

1991

return CM_Unknown;

1992

return Itr->second.first;

1993

}

1994

1995

/// Return the vectorization cost for the given instruction \p I and vector

1996

/// width \p VF.

1997

unsigned getWideningCost(Instruction *I, unsigned VF) {

1998

1999

std::pair<Instruction *, unsigned> InstOnVF = std::make_pair(I, VF);

2000

assert(WideningDecisions.count(InstOnVF) && "The cost is not calculated")((WideningDecisions.count(InstOnVF) && "The cost is not calculated"
) ? static_cast<void> (0) : __assert_fail ("WideningDecisions.count(InstOnVF) && \"The cost is not calculated\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2000, __PRETTY_FUNCTION__));

2001

return WideningDecisions[InstOnVF].second;

2002

}

2003

2004

/// Return True if instruction \p I is an optimizable truncate whose operand

2005

/// is an induction variable. Such a truncate will be removed by adding a new

2006

/// induction variable with the destination type.

2007

bool isOptimizableIVTruncate(Instruction *I, unsigned VF) {

2008

2009

// If the instruction is not a truncate, return false.

2010

auto *Trunc = dyn_cast<TruncInst>(I);

2011

if (!Trunc)

2012

return false;

2013

2014

// Get the source and destination types of the truncate.

2015

Type *SrcTy = ToVectorTy(cast<CastInst>(I)->getSrcTy(), VF);

2016

Type *DestTy = ToVectorTy(cast<CastInst>(I)->getDestTy(), VF);

2017

2018

// If the truncate is free for the given types, return false. Replacing a

2019

// free truncate with an induction variable would add an induction variable

2020

// update instruction to each iteration of the loop. We exclude from this

2021

// check the primary induction variable since it will need an update

2022

// instruction regardless.

2023

Value *Op = Trunc->getOperand(0);

2024

if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))

2025

return false;

2026

2027

// If the truncated value is not an induction variable, return false.

2028

return Legal->isInductionVariable(Op);

2029

}

2030

2031

private:

2032

/// \return An upper bound for the vectorization factor, larger than zero.

2033

/// One is returned if vectorization should best be avoided due to cost.

2034

unsigned computeFeasibleMaxVF(bool OptForSize);

2035

2036

/// The vectorization cost is a combination of the cost itself and a boolean

2037

/// indicating whether any of the contributing operations will actually

2038

/// operate on

2039

/// vector values after type legalization in the backend. If this latter value

2040

/// is

2041

/// false, then all operations will be scalarized (i.e. no vectorization has

2042

/// actually taken place).

2043

typedef std::pair<unsigned, bool> VectorizationCostTy;

2044

2045

/// Returns the expected execution cost. The unit of the cost does

2046

/// not matter because we use the 'cost' units to compare different

2047

/// vector widths. The cost that is returned is *not* normalized by

2048

/// the factor width.

2049

VectorizationCostTy expectedCost(unsigned VF);

2050

2051

/// Returns the execution time cost of an instruction for a given vector

2052

/// width. Vector width of one means scalar.

2053

VectorizationCostTy getInstructionCost(Instruction *I, unsigned VF);

2054

2055

/// The cost-computation logic from getInstructionCost which provides

2056

/// the vector type as an output parameter.

2057

unsigned getInstructionCost(Instruction *I, unsigned VF, Type *&VectorTy);

2058

2059

/// Calculate vectorization cost of memory instruction \p I.

2060

unsigned getMemoryInstructionCost(Instruction *I, unsigned VF);

2061

2062

/// The cost computation for scalarized memory instruction.

2063

unsigned getMemInstScalarizationCost(Instruction *I, unsigned VF);

2064

2065

/// The cost computation for interleaving group of memory instructions.

2066

unsigned getInterleaveGroupCost(Instruction *I, unsigned VF);

2067

2068

/// The cost computation for Gather/Scatter instruction.

2069

unsigned getGatherScatterCost(Instruction *I, unsigned VF);

2070

2071

/// The cost computation for widening instruction \p I with consecutive

2072

/// memory access.

2073

unsigned getConsecutiveMemOpCost(Instruction *I, unsigned VF);

2074

2075

/// The cost calculation for Load instruction \p I with uniform pointer -

2076

/// scalar load + broadcast.

2077

unsigned getUniformMemOpCost(Instruction *I, unsigned VF);

2078

2079

/// Returns whether the instruction is a load or store and will be a emitted

2080

/// as a vector operation.

2081

bool isConsecutiveLoadOrStore(Instruction *I);

2082

2083

/// Create an analysis remark that explains why vectorization failed

2084

///

2085

/// \p RemarkName is the identifier for the remark. \return the remark object

2086

/// that can be streamed to.

2087

OptimizationRemarkAnalysis createMissedAnalysis(StringRef RemarkName) {

2088

return ::createMissedAnalysis(Hints->vectorizeAnalysisPassName(),

2089

RemarkName, TheLoop);

2090

}

2091

2092

/// Map of scalar integer values to the smallest bitwidth they can be legally

2093

/// represented as. The vector equivalents of these values should be truncated

2094

/// to this type.

2095

MapVector<Instruction *, uint64_t> MinBWs;

2096

2097

/// A type representing the costs for instructions if they were to be

2098

/// scalarized rather than vectorized. The entries are Instruction-Cost

2099

/// pairs.

2100

typedef DenseMap<Instruction *, unsigned> ScalarCostsTy;

2101

2102

/// A set containing all BasicBlocks that are known to present after

2103

/// vectorization as a predicated block.

2104

SmallPtrSet<BasicBlock *, 4> PredicatedBBsAfterVectorization;

2105

2106

/// A map holding scalar costs for different vectorization factors. The

2107

/// presence of a cost for an instruction in the mapping indicates that the

2108

/// instruction will be scalarized when vectorizing with the associated

2109

/// vectorization factor. The entries are VF-ScalarCostTy pairs.

2110

DenseMap<unsigned, ScalarCostsTy> InstsToScalarize;

2111

2112

/// Holds the instructions known to be uniform after vectorization.

2113

/// The data is collected per VF.

2114

DenseMap<unsigned, SmallPtrSet<Instruction *, 4>> Uniforms;

2115

2116

/// Holds the instructions known to be scalar after vectorization.

2117

/// The data is collected per VF.

2118

DenseMap<unsigned, SmallPtrSet<Instruction *, 4>> Scalars;

2119

2120

/// Holds the instructions (address computations) that are forced to be

2121

/// scalarized.

2122

DenseMap<unsigned, SmallPtrSet<Instruction *, 4>> ForcedScalars;

2123

2124

/// Returns the expected difference in cost from scalarizing the expression

2125

/// feeding a predicated instruction \p PredInst. The instructions to

2126

/// scalarize and their scalar costs are collected in \p ScalarCosts. A

2127

/// non-negative return value implies the expression will be scalarized.

2128

/// Currently, only single-use chains are considered for scalarization.

2129

int computePredInstDiscount(Instruction *PredInst, ScalarCostsTy &ScalarCosts,

2130

unsigned VF);

2131

2132

/// Collects the instructions to scalarize for each predicated instruction in

2133

/// the loop.

2134

void collectInstsToScalarize(unsigned VF);

2135

2136

/// Collect the instructions that are uniform after vectorization. An

2137

/// instruction is uniform if we represent it with a single scalar value in

2138

/// the vectorized loop corresponding to each vector iteration. Examples of

2139

/// uniform instructions include pointer operands of consecutive or

2140

/// interleaved memory accesses. Note that although uniformity implies an

2141

/// instruction will be scalar, the reverse is not true. In general, a

2142

/// scalarized instruction will be represented by VF scalar values in the

2143

/// vectorized loop, each corresponding to an iteration of the original

2144

/// scalar loop.

2145

void collectLoopUniforms(unsigned VF);

2146

2147

/// Collect the instructions that are scalar after vectorization. An

2148

/// instruction is scalar if it is known to be uniform or will be scalarized

2149

/// during vectorization. Non-uniform scalarized instructions will be

2150

/// represented by VF values in the vectorized loop, each corresponding to an

2151

/// iteration of the original scalar loop.

2152

void collectLoopScalars(unsigned VF);

2153

2154

/// Collect Uniform and Scalar values for the given \p VF.

2155

/// The sets depend on CM decision for Load/Store instructions

2156

/// that may be vectorized as interleave, gather-scatter or scalarized.

2157

void collectUniformsAndScalars(unsigned VF) {

2158

// Do the analysis once.

2159

if (VF == 1 || Uniforms.count(VF))

2160

return;

2161

setCostBasedWideningDecision(VF);

2162

collectLoopUniforms(VF);

2163

collectLoopScalars(VF);

2164

}

2165

2166

/// Keeps cost model vectorization decision and cost for instructions.

2167

/// Right now it is used for memory instructions only.

2168

typedef DenseMap<std::pair<Instruction *, unsigned>,

2169

std::pair<InstWidening, unsigned>>

2170

DecisionList;

2171

2172

DecisionList WideningDecisions;

2173

2174

public:

2175

/// The loop that we evaluate.

2176

Loop *TheLoop;

2177

/// Predicated scalar evolution analysis.

2178

PredicatedScalarEvolution &PSE;

2179

/// Loop Info analysis.

2180

LoopInfo *LI;

2181

/// Vectorization legality.

2182

LoopVectorizationLegality *Legal;

2183

/// Vector target information.

2184

const TargetTransformInfo &TTI;

2185

/// Target Library Info.

2186

const TargetLibraryInfo *TLI;

2187

/// Demanded bits analysis.

2188

DemandedBits *DB;

2189

/// Assumption cache.

2190

AssumptionCache *AC;

2191

/// Interface to emit optimization remarks.

2192

OptimizationRemarkEmitter *ORE;

2193

2194

const Function *TheFunction;

2195

/// Loop Vectorize Hint.

2196

const LoopVectorizeHints *Hints;

2197

/// Values to ignore in the cost model.

2198

SmallPtrSet<const Value *, 16> ValuesToIgnore;

2199

/// Values to ignore in the cost model when VF > 1.

2200

SmallPtrSet<const Value *, 16> VecValuesToIgnore;

2201

};

2202

2203

/// LoopVectorizationPlanner - drives the vectorization process after having

2204

/// passed Legality checks.

2205

class LoopVectorizationPlanner {

2206

public:

2207

LoopVectorizationPlanner(Loop *OrigLoop, LoopInfo *LI,

2208

LoopVectorizationLegality *Legal,

2209

LoopVectorizationCostModel &CM)

2210

: OrigLoop(OrigLoop), LI(LI), Legal(Legal), CM(CM) {}

2211

2212

~LoopVectorizationPlanner() {}

2213

2214

/// Plan how to best vectorize, return the best VF and its cost.

2215

LoopVectorizationCostModel::VectorizationFactor plan(bool OptForSize,

2216

unsigned UserVF);

2217

2218

/// Generate the IR code for the vectorized loop.

2219

void executePlan(InnerLoopVectorizer &ILV);

2220

2221

protected:

2222

/// Collect the instructions from the original loop that would be trivially

2223

/// dead in the vectorized loop if generated.

2224

void collectTriviallyDeadInstructions(

2225

SmallPtrSetImpl<Instruction *> &DeadInstructions);

2226

2227

private:

2228

/// The loop that we evaluate.

2229

Loop *OrigLoop;

2230

2231

/// Loop Info analysis.

2232

LoopInfo *LI;

2233

2234

/// The legality analysis.

2235

LoopVectorizationLegality *Legal;

2236

2237

/// The profitablity analysis.

2238

LoopVectorizationCostModel &CM;

2239

};

2240

2241

/// \brief This holds vectorization requirements that must be verified late in

2242

/// the process. The requirements are set by legalize and costmodel. Once

2243

/// vectorization has been determined to be possible and profitable the

2244

/// requirements can be verified by looking for metadata or compiler options.

2245

/// For example, some loops require FP commutativity which is only allowed if

2246

/// vectorization is explicitly specified or if the fast-math compiler option

2247

/// has been provided.

2248

/// Late evaluation of these requirements allows helpful diagnostics to be

2249

/// composed that tells the user what need to be done to vectorize the loop. For

2250

/// example, by specifying #pragma clang loop vectorize or -ffast-math. Late

2251

/// evaluation should be used only when diagnostics can generated that can be

2252

/// followed by a non-expert user.

2253

class LoopVectorizationRequirements {

2254

public:

2255

LoopVectorizationRequirements(OptimizationRemarkEmitter &ORE)

2256

: NumRuntimePointerChecks(0), UnsafeAlgebraInst(nullptr), ORE(ORE) {}

2257

2258

void addUnsafeAlgebraInst(Instruction *I) {

2259

// First unsafe algebra instruction.

2260

if (!UnsafeAlgebraInst)

2261

UnsafeAlgebraInst = I;

2262

}

2263

2264

void addRuntimePointerChecks(unsigned Num) { NumRuntimePointerChecks = Num; }

2265

2266

bool doesNotMeet(Function *F, Loop *L, const LoopVectorizeHints &Hints) {

2267

const char *PassName = Hints.vectorizeAnalysisPassName();

2268

bool Failed = false;

2269

if (UnsafeAlgebraInst && !Hints.allowReordering()) {

2270

ORE.emit(

2271

OptimizationRemarkAnalysisFPCommute(PassName, "CantReorderFPOps",

2272

UnsafeAlgebraInst->getDebugLoc(),

2273

UnsafeAlgebraInst->getParent())

2274

<< "loop not vectorized: cannot prove it is safe to reorder "

2275

"floating-point operations");

2276

Failed = true;

2277

}

2278

2279

// Test if runtime memcheck thresholds are exceeded.

2280

bool PragmaThresholdReached =

2281

NumRuntimePointerChecks > PragmaVectorizeMemoryCheckThreshold;

2282

bool ThresholdReached =

2283

NumRuntimePointerChecks > VectorizerParams::RuntimeMemoryCheckThreshold;

2284

if ((ThresholdReached && !Hints.allowReordering()) ||

2285

PragmaThresholdReached) {

2286

ORE.emit(OptimizationRemarkAnalysisAliasing(PassName, "CantReorderMemOps",

2287

L->getStartLoc(),

2288

L->getHeader())

2289

<< "loop not vectorized: cannot prove it is safe to reorder "

2290

"memory operations");

2291

DEBUG(dbgs() << "LV: Too many memory checks needed.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Too many memory checks needed.\n"
; } } while (false);

2292

Failed = true;

2293

}

2294

2295

return Failed;

2296

}

2297

2298

private:

2299

unsigned NumRuntimePointerChecks;

2300

Instruction *UnsafeAlgebraInst;

2301

2302

/// Interface to emit optimization remarks.

2303

OptimizationRemarkEmitter &ORE;

2304

};

2305

2306

static void addAcyclicInnerLoop(Loop &L, SmallVectorImpl<Loop *> &V) {

2307

if (L.empty()) {

2308

if (!hasCyclesInLoopBody(L))

2309

V.push_back(&L);

2310

return;

2311

}

2312

for (Loop *InnerL : L)

2313

addAcyclicInnerLoop(*InnerL, V);

2314

}

2315

2316

/// The LoopVectorize Pass.

2317

struct LoopVectorize : public FunctionPass {

2318

/// Pass identification, replacement for typeid

2319

static char ID;

2320

2321

explicit LoopVectorize(bool NoUnrolling = false, bool AlwaysVectorize = true)

2322

: FunctionPass(ID) {

2323

Impl.DisableUnrolling = NoUnrolling;

2324

Impl.AlwaysVectorize = AlwaysVectorize;

2325

initializeLoopVectorizePass(*PassRegistry::getPassRegistry());

2326

}

2327

2328

LoopVectorizePass Impl;

2329

2330

bool runOnFunction(Function &F) override {

2331

if (skipFunction(F))

2332

return false;

2333

2334

auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();

2335

auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();

2336

auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);

2337

auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();

2338

auto *BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();

2339

auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();

2340

auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;

2341

auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();

2342

auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);

2343

auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>();

2344

auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();

2345

auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();

2346

2347

std::function<const LoopAccessInfo &(Loop &)> GetLAA =

2348

[&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); };

2349

2350

return Impl.runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC,

2351

GetLAA, *ORE);

2352

}

2353

2354

void getAnalysisUsage(AnalysisUsage &AU) const override {

2355

AU.addRequired<AssumptionCacheTracker>();

2356

AU.addRequired<BlockFrequencyInfoWrapperPass>();

2357

AU.addRequired<DominatorTreeWrapperPass>();

2358

AU.addRequired<LoopInfoWrapperPass>();

2359

AU.addRequired<ScalarEvolutionWrapperPass>();

2360

AU.addRequired<TargetTransformInfoWrapperPass>();

2361

AU.addRequired<AAResultsWrapperPass>();

2362

AU.addRequired<LoopAccessLegacyAnalysis>();

2363

AU.addRequired<DemandedBitsWrapperPass>();

2364

AU.addRequired<OptimizationRemarkEmitterWrapperPass>();

2365

AU.addPreserved<LoopInfoWrapperPass>();

2366

AU.addPreserved<DominatorTreeWrapperPass>();

2367

AU.addPreserved<BasicAAWrapperPass>();

2368

AU.addPreserved<GlobalsAAWrapperPass>();

2369

}

2370

};

2371

2372

} // end anonymous namespace

2373

2374

//===----------------------------------------------------------------------===//

2375

// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and

2376

// LoopVectorizationCostModel and LoopVectorizationPlanner.

2377

//===----------------------------------------------------------------------===//

2378

2379

Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {

2380

// We need to place the broadcast of invariant variables outside the loop.

2381

Instruction *Instr = dyn_cast<Instruction>(V);

2382

bool NewInstr = (Instr && Instr->getParent() == LoopVectorBody);

2383

bool Invariant = OrigLoop->isLoopInvariant(V) && !NewInstr;

2384

2385

// Place the code for broadcasting invariant variables in the new preheader.

2386

IRBuilder<>::InsertPointGuard Guard(Builder);

2387

if (Invariant)

2388

Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());

2389

2390

// Broadcast the scalar into all locations in the vector.

2391

Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");

2392

2393

return Shuf;

2394

}

2395

2396

void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(

2397

const InductionDescriptor &II, Value *Step, Instruction *EntryVal) {

2398

Value *Start = II.getStartValue();

2399

2400

// Construct the initial value of the vector IV in the vector loop preheader

2401

auto CurrIP = Builder.saveIP();

2402

Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());

2403

if (isa<TruncInst>(EntryVal)) {

2404

assert(Start->getType()->isIntegerTy() &&((Start->getType()->isIntegerTy() && "Truncation requires an integer type"
) ? static_cast<void> (0) : __assert_fail ("Start->getType()->isIntegerTy() && \"Truncation requires an integer type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2405, __PRETTY_FUNCTION__))

2405

"Truncation requires an integer type")((Start->getType()->isIntegerTy() && "Truncation requires an integer type"
) ? static_cast<void> (0) : __assert_fail ("Start->getType()->isIntegerTy() && \"Truncation requires an integer type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2405, __PRETTY_FUNCTION__));

2406

auto *TruncType = cast<IntegerType>(EntryVal->getType());

2407

Step = Builder.CreateTrunc(Step, TruncType);

2408

Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);

2409

}

2410

Value *SplatStart = Builder.CreateVectorSplat(VF, Start);

2411

Value *SteppedStart =

2412

getStepVector(SplatStart, 0, Step, II.getInductionOpcode());

2413

2414

// We create vector phi nodes for both integer and floating-point induction

2415

// variables. Here, we determine the kind of arithmetic we will perform.

2416

Instruction::BinaryOps AddOp;

2417

Instruction::BinaryOps MulOp;

2418

if (Step->getType()->isIntegerTy()) {

2419

AddOp = Instruction::Add;

2420

MulOp = Instruction::Mul;

2421

} else {

2422

AddOp = II.getInductionOpcode();

2423

MulOp = Instruction::FMul;

2424

}

2425

2426

// Multiply the vectorization factor by the step using integer or

2427

// floating-point arithmetic as appropriate.

2428

Value *ConstVF = getSignedIntOrFpConstant(Step->getType(), VF);

2429

Value *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, Step, ConstVF));

2430

2431

// Create a vector splat to use in the induction update.

2432

2433

// FIXME: If the step is non-constant, we create the vector splat with

2434

// IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't

2435

// handle a constant vector splat.

2436

Value *SplatVF = isa<Constant>(Mul)

2437

? ConstantVector::getSplat(VF, cast<Constant>(Mul))

2438

: Builder.CreateVectorSplat(VF, Mul);

2439

Builder.restoreIP(CurrIP);

2440

2441

// We may need to add the step a number of times, depending on the unroll

2442

// factor. The last of those goes into the PHI.

2443

PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind",

2444

&*LoopVectorBody->getFirstInsertionPt());

2445

Instruction *LastInduction = VecInd;

2446

for (unsigned Part = 0; Part < UF; ++Part) {

2447

VectorLoopValueMap.setVectorValue(EntryVal, Part, LastInduction);

2448

if (isa<TruncInst>(EntryVal))

2449

addMetadata(LastInduction, EntryVal);

2450

LastInduction = cast<Instruction>(addFastMathFlag(

2451

Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add")));

2452

}

2453

2454

// Move the last step to the end of the latch block. This ensures consistent

2455

// placement of all induction updates.

2456

auto *LoopVectorLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();

2457

auto *Br = cast<BranchInst>(LoopVectorLatch->getTerminator());

2458

auto *ICmp = cast<Instruction>(Br->getCondition());

2459

LastInduction->moveBefore(ICmp);

2460

LastInduction->setName("vec.ind.next");

2461

2462

VecInd->addIncoming(SteppedStart, LoopVectorPreHeader);

2463

VecInd->addIncoming(LastInduction, LoopVectorLatch);

2464

}

2465

2466

bool InnerLoopVectorizer::shouldScalarizeInstruction(Instruction *I) const {

2467

return Cost->isScalarAfterVectorization(I, VF) ||

2468

Cost->isProfitableToScalarize(I, VF);

2469

}

2470

2471

bool InnerLoopVectorizer::needsScalarInduction(Instruction *IV) const {

2472

if (shouldScalarizeInstruction(IV))

2473

return true;

2474

auto isScalarInst = [&](User *U) -> bool {

2475

auto *I = cast<Instruction>(U);

2476

return (OrigLoop->contains(I) && shouldScalarizeInstruction(I));

2477

};

2478

return any_of(IV->users(), isScalarInst);

2479

}

2480

2481

void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) {

2482

2483

assert((IV->getType()->isIntegerTy() || IV != OldInduction) &&(((IV->getType()->isIntegerTy() || IV != OldInduction) &&
"Primary induction variable must have an integer type") ? static_cast
<void> (0) : __assert_fail ("(IV->getType()->isIntegerTy() || IV != OldInduction) && \"Primary induction variable must have an integer type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2484, __PRETTY_FUNCTION__))

2484

"Primary induction variable must have an integer type")(((IV->getType()->isIntegerTy() || IV != OldInduction) &&
"Primary induction variable must have an integer type") ? static_cast
<void> (0) : __assert_fail ("(IV->getType()->isIntegerTy() || IV != OldInduction) && \"Primary induction variable must have an integer type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2484, __PRETTY_FUNCTION__));

2485

2486

auto II = Legal->getInductionVars()->find(IV);

2487

assert(II != Legal->getInductionVars()->end() && "IV is not an induction")((II != Legal->getInductionVars()->end() && "IV is not an induction"
) ? static_cast<void> (0) : __assert_fail ("II != Legal->getInductionVars()->end() && \"IV is not an induction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2487, __PRETTY_FUNCTION__));

2488

2489

auto ID = II->second;

2490

assert(IV->getType() == ID.getStartValue()->getType() && "Types must match")((IV->getType() == ID.getStartValue()->getType() &&
"Types must match") ? static_cast<void> (0) : __assert_fail
("IV->getType() == ID.getStartValue()->getType() && \"Types must match\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2490, __PRETTY_FUNCTION__));

2491

2492

// The scalar value to broadcast. This will be derived from the canonical

2493

// induction variable.

2494

Value *ScalarIV = nullptr;

2495

2496

// The value from the original loop to which we are mapping the new induction

2497

// variable.

2498

Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;

2499

2500

// True if we have vectorized the induction variable.

2501

auto VectorizedIV = false;

2502

2503

// Determine if we want a scalar version of the induction variable. This is

2504

// true if the induction variable itself is not widened, or if it has at

2505

// least one user in the loop that is not widened.

2506

auto NeedsScalarIV = VF > 1 && needsScalarInduction(EntryVal);

2507

2508

// Generate code for the induction step. Note that induction steps are

2509

// required to be loop-invariant

2510

assert(PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) &&((PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) &&
"Induction step should be loop invariant") ? static_cast<
void> (0) : __assert_fail ("PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) && \"Induction step should be loop invariant\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2511, __PRETTY_FUNCTION__))

2511

"Induction step should be loop invariant")((PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) &&
"Induction step should be loop invariant") ? static_cast<
void> (0) : __assert_fail ("PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) && \"Induction step should be loop invariant\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2511, __PRETTY_FUNCTION__));

2512

auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout();

2513

Value *Step = nullptr;

2514

if (PSE.getSE()->isSCEVable(IV->getType())) {

2515

SCEVExpander Exp(*PSE.getSE(), DL, "induction");

2516

Step = Exp.expandCodeFor(ID.getStep(), ID.getStep()->getType(),

2517

LoopVectorPreHeader->getTerminator());

2518

} else {

2519

Step = cast<SCEVUnknown>(ID.getStep())->getValue();

2520

}

2521

2522

// Try to create a new independent vector induction variable. If we can't

2523

// create the phi node, we will splat the scalar induction variable in each

2524

// loop iteration.

2525

if (VF > 1 && !shouldScalarizeInstruction(EntryVal)) {

2526

createVectorIntOrFpInductionPHI(ID, Step, EntryVal);

2527

VectorizedIV = true;

2528

}

2529

2530

// If we haven't yet vectorized the induction variable, or if we will create

2531

// a scalar one, we need to define the scalar induction variable and step

2532

// values. If we were given a truncation type, truncate the canonical

2533

// induction variable and step. Otherwise, derive these values from the

2534

// induction descriptor.

2535

if (!VectorizedIV || NeedsScalarIV) {

2536

ScalarIV = Induction;

2537

if (IV != OldInduction) {

2538

ScalarIV = IV->getType()->isIntegerTy()

2539

? Builder.CreateSExtOrTrunc(Induction, IV->getType())

2540

: Builder.CreateCast(Instruction::SIToFP, Induction,

2541

IV->getType());

2542

ScalarIV = ID.transform(Builder, ScalarIV, PSE.getSE(), DL);

2543

ScalarIV->setName("offset.idx");

2544

}

2545

if (Trunc) {

2546

auto *TruncType = cast<IntegerType>(Trunc->getType());

2547

assert(Step->getType()->isIntegerTy() &&((Step->getType()->isIntegerTy() && "Truncation requires an integer step"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isIntegerTy() && \"Truncation requires an integer step\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2548, __PRETTY_FUNCTION__))

2548

"Truncation requires an integer step")((Step->getType()->isIntegerTy() && "Truncation requires an integer step"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isIntegerTy() && \"Truncation requires an integer step\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2548, __PRETTY_FUNCTION__));

2549

ScalarIV = Builder.CreateTrunc(ScalarIV, TruncType);

2550

Step = Builder.CreateTrunc(Step, TruncType);

2551

}

2552

}

2553

2554

// If we haven't yet vectorized the induction variable, splat the scalar

2555

// induction variable, and build the necessary step vectors.

2556

if (!VectorizedIV) {

2557

Value *Broadcasted = getBroadcastInstrs(ScalarIV);

2558

for (unsigned Part = 0; Part < UF; ++Part) {

2559

Value *EntryPart =

2560

getStepVector(Broadcasted, VF * Part, Step, ID.getInductionOpcode());

2561

VectorLoopValueMap.setVectorValue(EntryVal, Part, EntryPart);

2562

if (Trunc)

2563

addMetadata(EntryPart, Trunc);

2564

}

2565

}

2566

2567

// If an induction variable is only used for counting loop iterations or

2568

// calculating addresses, it doesn't need to be widened. Create scalar steps

2569

// that can be used by instructions we will later scalarize. Note that the

2570

// addition of the scalar steps will not increase the number of instructions

2571

// in the loop in the common case prior to InstCombine. We will be trading

2572

// one vector extract for each scalar step.

2573

if (NeedsScalarIV)

2574

buildScalarSteps(ScalarIV, Step, EntryVal, ID);

2575

}

2576

2577

Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx, Value *Step,

2578

Instruction::BinaryOps BinOp) {

2579

// Create and check the types.

2580

assert(Val->getType()->isVectorTy() && "Must be a vector")((Val->getType()->isVectorTy() && "Must be a vector"
) ? static_cast<void> (0) : __assert_fail ("Val->getType()->isVectorTy() && \"Must be a vector\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2580, __PRETTY_FUNCTION__));

2581

int VLen = Val->getType()->getVectorNumElements();

2582

2583

Type *STy = Val->getType()->getScalarType();

2584

assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&(((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
"Induction Step must be an integer or FP") ? static_cast<
void> (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2585, __PRETTY_FUNCTION__))

2585

"Induction Step must be an integer or FP")(((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
"Induction Step must be an integer or FP") ? static_cast<
void> (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2585, __PRETTY_FUNCTION__));

2586

assert(Step->getType() == STy && "Step has wrong type")((Step->getType() == STy && "Step has wrong type")
? static_cast<void> (0) : __assert_fail ("Step->getType() == STy && \"Step has wrong type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2586, __PRETTY_FUNCTION__));

2587

2588

SmallVector<Constant *, 8> Indices;

2589

2590

if (STy->isIntegerTy()) {

2591

// Create a vector of consecutive numbers from zero to VF.

2592

for (int i = 0; i < VLen; ++i)

2593

Indices.push_back(ConstantInt::get(STy, StartIdx + i));

2594

2595

// Add the consecutive indices to the vector value.

2596

Constant *Cv = ConstantVector::get(Indices);

2597

assert(Cv->getType() == Val->getType() && "Invalid consecutive vec")((Cv->getType() == Val->getType() && "Invalid consecutive vec"
) ? static_cast<void> (0) : __assert_fail ("Cv->getType() == Val->getType() && \"Invalid consecutive vec\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2597, __PRETTY_FUNCTION__));

2598

Step = Builder.CreateVectorSplat(VLen, Step);

2599

assert(Step->getType() == Val->getType() && "Invalid step vec")((Step->getType() == Val->getType() && "Invalid step vec"
) ? static_cast<void> (0) : __assert_fail ("Step->getType() == Val->getType() && \"Invalid step vec\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2599, __PRETTY_FUNCTION__));

2600

// FIXME: The newly created binary instructions should contain nsw/nuw flags,

2601

// which can be found from the original scalar operations.

2602

Step = Builder.CreateMul(Cv, Step);

2603

return Builder.CreateAdd(Val, Step, "induction");

2604

}

2605

2606

// Floating point induction.

2607

assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&(((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
"Binary Opcode should be specified for FP induction") ? static_cast
<void> (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2608, __PRETTY_FUNCTION__))

2608

"Binary Opcode should be specified for FP induction")(((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
"Binary Opcode should be specified for FP induction") ? static_cast
<void> (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2608, __PRETTY_FUNCTION__));

2609

// Create a vector of consecutive numbers from zero to VF.

2610

for (int i = 0; i < VLen; ++i)

2611

Indices.push_back(ConstantFP::get(STy, (double)(StartIdx + i)));

2612

2613

// Add the consecutive indices to the vector value.

2614

Constant *Cv = ConstantVector::get(Indices);

2615

2616

Step = Builder.CreateVectorSplat(VLen, Step);

2617

2618

// Floating point operations had to be 'fast' to enable the induction.

2619

FastMathFlags Flags;

2620

Flags.setUnsafeAlgebra();

2621

2622

Value *MulOp = Builder.CreateFMul(Cv, Step);

2623

if (isa<Instruction>(MulOp))

2624

// Have to check, MulOp may be a constant

2625

cast<Instruction>(MulOp)->setFastMathFlags(Flags);

2626

2627

Value *BOp = Builder.CreateBinOp(BinOp, Val, MulOp, "induction");

2628

if (isa<Instruction>(BOp))

2629

cast<Instruction>(BOp)->setFastMathFlags(Flags);

2630

return BOp;

2631

}

2632

2633

void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,

2634

Value *EntryVal,

2635

const InductionDescriptor &ID) {

2636

2637

// We shouldn't have to build scalar steps if we aren't vectorizing.

2638

assert(VF > 1 && "VF should be greater than one")((VF > 1 && "VF should be greater than one") ? static_cast
<void> (0) : __assert_fail ("VF > 1 && \"VF should be greater than one\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2638, __PRETTY_FUNCTION__));

2639

2640

// Get the value type and ensure it and the step have the same integer type.

2641

Type *ScalarIVTy = ScalarIV->getType()->getScalarType();

2642

assert(ScalarIVTy == Step->getType() &&((ScalarIVTy == Step->getType() && "Val and Step should have the same type"
) ? static_cast<void> (0) : __assert_fail ("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2643, __PRETTY_FUNCTION__))

2643

"Val and Step should have the same type")((ScalarIVTy == Step->getType() && "Val and Step should have the same type"
) ? static_cast<void> (0) : __assert_fail ("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2643, __PRETTY_FUNCTION__));

2644

2645

// We build scalar steps for both integer and floating-point induction

2646

// variables. Here, we determine the kind of arithmetic we will perform.

2647

Instruction::BinaryOps AddOp;

2648

Instruction::BinaryOps MulOp;

2649

if (ScalarIVTy->isIntegerTy()) {

2650

AddOp = Instruction::Add;

2651

MulOp = Instruction::Mul;

2652

} else {

2653

AddOp = ID.getInductionOpcode();

2654

MulOp = Instruction::FMul;

2655

}

2656

2657

// Determine the number of scalars we need to generate for each unroll

2658

// iteration. If EntryVal is uniform, we only need to generate the first

2659

// lane. Otherwise, we generate all VF values.

2660

unsigned Lanes =

2661

Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF) ? 1 : VF;

2662

2663

// Compute the scalar steps and save the results in VectorLoopValueMap.

2664

for (unsigned Part = 0; Part < UF; ++Part) {

2665

for (unsigned Lane = 0; Lane < Lanes; ++Lane) {

2666

auto *StartIdx = getSignedIntOrFpConstant(ScalarIVTy, VF * Part + Lane);

2667

auto *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, StartIdx, Step));

2668

auto *Add = addFastMathFlag(Builder.CreateBinOp(AddOp, ScalarIV, Mul));

2669

VectorLoopValueMap.setScalarValue(EntryVal, Part, Lane, Add);

2670

}

2671

}

2672

}

2673

2674

int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) {

2675

2676

const ValueToValueMap &Strides = getSymbolicStrides() ? *getSymbolicStrides() :

2677

ValueToValueMap();

2678

2679

int Stride = getPtrStride(PSE, Ptr, TheLoop, Strides, true, false);

2680

if (Stride == 1 || Stride == -1)

2681

return Stride;

2682

return 0;

2683

}

2684

2685

bool LoopVectorizationLegality::isUniform(Value *V) {

2686

return LAI->isUniform(V);

2687

}

2688

2689

Value *InnerLoopVectorizer::getOrCreateVectorValue(Value *V, unsigned Part) {

2690

assert(V != Induction && "The new induction variable should not be used.")((V != Induction && "The new induction variable should not be used."
) ? static_cast<void> (0) : __assert_fail ("V != Induction && \"The new induction variable should not be used.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2690, __PRETTY_FUNCTION__));

2691

assert(!V->getType()->isVectorTy() && "Can't widen a vector")((!V->getType()->isVectorTy() && "Can't widen a vector"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVectorTy() && \"Can't widen a vector\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2691, __PRETTY_FUNCTION__));

2692

assert(!V->getType()->isVoidTy() && "Type does not produce a value")((!V->getType()->isVoidTy() && "Type does not produce a value"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVoidTy() && \"Type does not produce a value\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2692, __PRETTY_FUNCTION__));

2693

2694

// If we have a stride that is replaced by one, do it here.

2695

if (Legal->hasStride(V))

2696

V = ConstantInt::get(V->getType(), 1);

2697

2698

// If we have a vector mapped to this value, return it.

2699

if (VectorLoopValueMap.hasVectorValue(V, Part))

2700

return VectorLoopValueMap.getVectorValue(V, Part);

2701

2702

// If the value has not been vectorized, check if it has been scalarized

2703

// instead. If it has been scalarized, and we actually need the value in

2704

// vector form, we will construct the vector values on demand.

2705

if (VectorLoopValueMap.hasAnyScalarValue(V)) {

2706

2707

Value *ScalarValue = VectorLoopValueMap.getScalarValue(V, Part, 0);

2708

2709

// If we've scalarized a value, that value should be an instruction.

2710

auto *I = cast<Instruction>(V);

2711

2712

// If we aren't vectorizing, we can just copy the scalar map values over to

2713

// the vector map.

2714

if (VF == 1) {

2715

VectorLoopValueMap.setVectorValue(V, Part, ScalarValue);

2716

return ScalarValue;

2717

}

2718

2719

// Get the last scalar instruction we generated for V. If the value is

2720

// known to be uniform after vectorization, this corresponds to lane zero

2721

// of the last unroll iteration. Otherwise, the last instruction is the one

2722

// we created for the last vector lane of the last unroll iteration.

2723

unsigned LastLane = Cost->isUniformAfterVectorization(I, VF) ? 0 : VF - 1;

2724

auto *LastInst =

2725

cast<Instruction>(getOrCreateScalarValue(V, UF - 1, LastLane));

2726

2727

// Set the insert point after the last scalarized instruction. This ensures

2728

// the insertelement sequence will directly follow the scalar definitions.

2729

auto OldIP = Builder.saveIP();

2730

auto NewIP = std::next(BasicBlock::iterator(LastInst));

2731

Builder.SetInsertPoint(&*NewIP);

2732

2733

// However, if we are vectorizing, we need to construct the vector values.

2734

// If the value is known to be uniform after vectorization, we can just

2735

// broadcast the scalar value corresponding to lane zero for each unroll

2736

// iteration. Otherwise, we construct the vector values using insertelement

2737

// instructions. Since the resulting vectors are stored in

2738

// VectorLoopValueMap, we will only generate the insertelements once.

2739

Value *VectorValue = nullptr;

2740

if (Cost->isUniformAfterVectorization(I, VF)) {

2741

VectorValue = getBroadcastInstrs(ScalarValue);

2742

} else {

2743

VectorValue = UndefValue::get(VectorType::get(V->getType(), VF));

2744

for (unsigned Lane = 0; Lane < VF; ++Lane)

2745

VectorValue = Builder.CreateInsertElement(

2746

VectorValue, getOrCreateScalarValue(V, Part, Lane),

2747

Builder.getInt32(Lane));

2748

}

2749

VectorLoopValueMap.setVectorValue(V, Part, VectorValue);

2750

Builder.restoreIP(OldIP);

2751

return VectorValue;

2752

}

2753

2754

// If this scalar is unknown, assume that it is a constant or that it is

2755

// loop invariant. Broadcast V and save the value for future uses.

2756

Value *B = getBroadcastInstrs(V);

2757

VectorLoopValueMap.setVectorValue(V, Part, B);

2758

return B;

2759

}

2760

2761

Value *InnerLoopVectorizer::getOrCreateScalarValue(Value *V, unsigned Part,

2762

unsigned Lane) {

2763

2764

// If the value is not an instruction contained in the loop, it should

2765

// already be scalar.

2766

if (OrigLoop->isLoopInvariant(V))

2767

return V;

2768

2769

assert(Lane > 0 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF)((Lane > 0 ? !Cost->isUniformAfterVectorization(cast<
Instruction>(V), VF) : true && "Uniform values only have lane zero"
) ? static_cast<void> (0) : __assert_fail ("Lane > 0 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF) : true && \"Uniform values only have lane zero\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2770, __PRETTY_FUNCTION__))

2770

: true && "Uniform values only have lane zero")((Lane > 0 ? !Cost->isUniformAfterVectorization(cast<
Instruction>(V), VF) : true && "Uniform values only have lane zero"
) ? static_cast<void> (0) : __assert_fail ("Lane > 0 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF) : true && \"Uniform values only have lane zero\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2770, __PRETTY_FUNCTION__));

2771

2772

// If the value from the original loop has not been vectorized, it is

2773

// represented by UF x VF scalar values in the new loop. Return the requested

2774

// scalar value.

2775

if (VectorLoopValueMap.hasScalarValue(V, Part, Lane))

2776

return VectorLoopValueMap.getScalarValue(V, Part, Lane);

2777

2778

// If the value has not been scalarized, get its entry in VectorLoopValueMap

2779

// for the given unroll part. If this entry is not a vector type (i.e., the

2780

// vectorization factor is one), there is no need to generate an

2781

// extractelement instruction.

2782

auto *U = getOrCreateVectorValue(V, Part);

2783

if (!U->getType()->isVectorTy()) {

2784

assert(VF == 1 && "Value not scalarized has non-vector type")((VF == 1 && "Value not scalarized has non-vector type"
) ? static_cast<void> (0) : __assert_fail ("VF == 1 && \"Value not scalarized has non-vector type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2784, __PRETTY_FUNCTION__));

2785

return U;

2786

}

2787

2788

// Otherwise, the value from the original loop has been vectorized and is

2789

// represented by UF vector values. Extract and return the requested scalar

2790

// value from the appropriate vector lane.

2791

return Builder.CreateExtractElement(U, Builder.getInt32(Lane));

2792

}

2793

2794

Value *InnerLoopVectorizer::reverseVector(Value *Vec) {

2795

assert(Vec->getType()->isVectorTy() && "Invalid type")((Vec->getType()->isVectorTy() && "Invalid type"
) ? static_cast<void> (0) : __assert_fail ("Vec->getType()->isVectorTy() && \"Invalid type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2795, __PRETTY_FUNCTION__));

2796

SmallVector<Constant *, 8> ShuffleMask;

2797

for (unsigned i = 0; i < VF; ++i)

2798

ShuffleMask.push_back(Builder.getInt32(VF - i - 1));

2799

2800

return Builder.CreateShuffleVector(Vec, UndefValue::get(Vec->getType()),

2801

ConstantVector::get(ShuffleMask),

2802

"reverse");

2803

}

2804

2805

// Try to vectorize the interleave group that \p Instr belongs to.

2806

2807

// E.g. Translate following interleaved load group (factor = 3):

2808

// for (i = 0; i < N; i+=3) {

2809

// R = Pic[i]; // Member of index 0

2810

// G = Pic[i+1]; // Member of index 1

2811

// B = Pic[i+2]; // Member of index 2

2812

// ... // do something to R, G, B

2813

// }

2814

// To:

2815

// %wide.vec = load <12 x i32> ; Read 4 tuples of R,G,B

2816

// %R.vec = shuffle %wide.vec, undef, <0, 3, 6, 9> ; R elements

2817

// %G.vec = shuffle %wide.vec, undef, <1, 4, 7, 10> ; G elements

2818

// %B.vec = shuffle %wide.vec, undef, <2, 5, 8, 11> ; B elements

2819

2820

// Or translate following interleaved store group (factor = 3):

2821

// for (i = 0; i < N; i+=3) {

2822

// ... do something to R, G, B

2823

// Pic[i] = R; // Member of index 0

2824

// Pic[i+1] = G; // Member of index 1

2825

// Pic[i+2] = B; // Member of index 2

2826

// }

2827

// To:

2828

// %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>

2829

// %B_U.vec = shuffle %B.vec, undef, <0, 1, 2, 3, u, u, u, u>

2830

// %interleaved.vec = shuffle %R_G.vec, %B_U.vec,

2831

// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements

2832

// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B

2833

void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr) {

2834

const InterleaveGroup *Group = Legal->getInterleavedAccessGroup(Instr);

2835

assert(Group && "Fail to get an interleaved access group.")((Group && "Fail to get an interleaved access group."
) ? static_cast<void> (0) : __assert_fail ("Group && \"Fail to get an interleaved access group.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2835, __PRETTY_FUNCTION__));

2836

2837

// Skip if current instruction is not the insert position.

2838

if (Instr != Group->getInsertPos())

2839

return;

2840

2841

Value *Ptr = getPointerOperand(Instr);

2842

2843

// Prepare for the vector type of the interleaved load/store.

2844

Type *ScalarTy = getMemInstValueType(Instr);

2845

unsigned InterleaveFactor = Group->getFactor();

2846

Type *VecTy = VectorType::get(ScalarTy, InterleaveFactor * VF);

2847

Type *PtrTy = VecTy->getPointerTo(getMemInstAddressSpace(Instr));

2848

2849

// Prepare for the new pointers.

2850

setDebugLocFromInst(Builder, Ptr);

2851

SmallVector<Value *, 2> NewPtrs;

2852

unsigned Index = Group->getIndex(Instr);

2853

2854

// If the group is reverse, adjust the index to refer to the last vector lane

2855

// instead of the first. We adjust the index from the first vector lane,

2856

// rather than directly getting the pointer for lane VF - 1, because the

2857

// pointer operand of the interleaved access is supposed to be uniform. For

2858

// uniform instructions, we're only required to generate a value for the

2859

// first vector lane in each unroll iteration.

2860

if (Group->isReverse())

2861

Index += (VF - 1) * Group->getFactor();

2862

2863

for (unsigned Part = 0; Part < UF; Part++) {

2864

Value *NewPtr = getOrCreateScalarValue(Ptr, Part, 0);

2865

2866

// Notice current instruction could be any index. Need to adjust the address

2867

// to the member of index 0.

2868

2869

// E.g. a = A[i+1]; // Member of index 1 (Current instruction)

2870

// b = A[i]; // Member of index 0

2871

// Current pointer is pointed to A[i+1], adjust it to A[i].

2872

2873

// E.g. A[i+1] = a; // Member of index 1

2874

// A[i] = b; // Member of index 0

2875

// A[i+2] = c; // Member of index 2 (Current instruction)

2876

// Current pointer is pointed to A[i+2], adjust it to A[i].

2877

NewPtr = Builder.CreateGEP(NewPtr, Builder.getInt32(-Index));

2878

2879

// Cast to the vector pointer type.

2880

NewPtrs.push_back(Builder.CreateBitCast(NewPtr, PtrTy));

2881

}

2882

2883

setDebugLocFromInst(Builder, Instr);

2884

Value *UndefVec = UndefValue::get(VecTy);

2885

2886

// Vectorize the interleaved load group.

2887

if (isa<LoadInst>(Instr)) {

2888

2889

// For each unroll part, create a wide load for the group.

2890

SmallVector<Value *, 2> NewLoads;

2891

for (unsigned Part = 0; Part < UF; Part++) {

2892

auto *NewLoad = Builder.CreateAlignedLoad(

2893

NewPtrs[Part], Group->getAlignment(), "wide.vec");

2894

addMetadata(NewLoad, Instr);

2895

NewLoads.push_back(NewLoad);

2896

}

2897

2898

// For each member in the group, shuffle out the appropriate data from the

2899

// wide loads.

2900

for (unsigned I = 0; I < InterleaveFactor; ++I) {

2901

Instruction *Member = Group->getMember(I);

2902

2903

// Skip the gaps in the group.

2904

if (!Member)

2905

continue;

2906

2907

Constant *StrideMask = createStrideMask(Builder, I, InterleaveFactor, VF);

2908

for (unsigned Part = 0; Part < UF; Part++) {

2909

Value *StridedVec = Builder.CreateShuffleVector(

2910

NewLoads[Part], UndefVec, StrideMask, "strided.vec");

2911

2912

// If this member has different type, cast the result type.

2913

if (Member->getType() != ScalarTy) {

2914

VectorType *OtherVTy = VectorType::get(Member->getType(), VF);

2915

StridedVec = Builder.CreateBitOrPointerCast(StridedVec, OtherVTy);

2916

}

2917

2918

if (Group->isReverse())

2919

StridedVec = reverseVector(StridedVec);

2920

2921

VectorLoopValueMap.setVectorValue(Member, Part, StridedVec);

2922

}

2923

}

2924

return;

2925

}

2926

2927

// The sub vector type for current instruction.

2928

VectorType *SubVT = VectorType::get(ScalarTy, VF);

2929

2930

// Vectorize the interleaved store group.

2931

for (unsigned Part = 0; Part < UF; Part++) {

2932

// Collect the stored vector from each member.

2933

SmallVector<Value *, 4> StoredVecs;

2934

for (unsigned i = 0; i < InterleaveFactor; i++) {

2935

// Interleaved store group doesn't allow a gap, so each index has a member

2936

Instruction *Member = Group->getMember(i);

2937

assert(Member && "Fail to get a member from an interleaved store group")((Member && "Fail to get a member from an interleaved store group"
) ? static_cast<void> (0) : __assert_fail ("Member && \"Fail to get a member from an interleaved store group\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2937, __PRETTY_FUNCTION__));

2938

2939

Value *StoredVec = getOrCreateVectorValue(

2940

cast<StoreInst>(Member)->getValueOperand(), Part);

2941

if (Group->isReverse())

2942

StoredVec = reverseVector(StoredVec);

2943

2944

// If this member has different type, cast it to an unified type.

2945

if (StoredVec->getType() != SubVT)

2946

StoredVec = Builder.CreateBitOrPointerCast(StoredVec, SubVT);

2947

2948

StoredVecs.push_back(StoredVec);

2949

}

2950

2951

// Concatenate all vectors into a wide vector.

2952

Value *WideVec = concatenateVectors(Builder, StoredVecs);

2953

2954

// Interleave the elements in the wide vector.

2955

Constant *IMask = createInterleaveMask(Builder, VF, InterleaveFactor);

2956

Value *IVec = Builder.CreateShuffleVector(WideVec, UndefVec, IMask,

2957

"interleaved.vec");

2958

2959

Instruction *NewStoreInstr =

2960

Builder.CreateAlignedStore(IVec, NewPtrs[Part], Group->getAlignment());

2961

addMetadata(NewStoreInstr, Instr);

2962

}

2963

}

2964

2965

void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {

2966

// Attempt to issue a wide load.

2967

LoadInst *LI = dyn_cast<LoadInst>(Instr);

2968

StoreInst *SI = dyn_cast<StoreInst>(Instr);

2969

2970

assert((LI || SI) && "Invalid Load/Store instruction")(((LI || SI) && "Invalid Load/Store instruction") ? static_cast
<void> (0) : __assert_fail ("(LI || SI) && \"Invalid Load/Store instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2970, __PRETTY_FUNCTION__));

2971

2972

LoopVectorizationCostModel::InstWidening Decision =

2973

Cost->getWideningDecision(Instr, VF);

2974

assert(Decision != LoopVectorizationCostModel::CM_Unknown &&((Decision != LoopVectorizationCostModel::CM_Unknown &&
"CM decision should be taken at this point") ? static_cast<
void> (0) : __assert_fail ("Decision != LoopVectorizationCostModel::CM_Unknown && \"CM decision should be taken at this point\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2975, __PRETTY_FUNCTION__))

2975

"CM decision should be taken at this point")((Decision != LoopVectorizationCostModel::CM_Unknown &&
"CM decision should be taken at this point") ? static_cast<
void> (0) : __assert_fail ("Decision != LoopVectorizationCostModel::CM_Unknown && \"CM decision should be taken at this point\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2975, __PRETTY_FUNCTION__));

2976

if (Decision == LoopVectorizationCostModel::CM_Interleave)

2977

return vectorizeInterleaveGroup(Instr);

2978

2979

Type *ScalarDataTy = getMemInstValueType(Instr);

2980

Type *DataTy = VectorType::get(ScalarDataTy, VF);

2981

Value *Ptr = getPointerOperand(Instr);

2982

unsigned Alignment = getMemInstAlignment(Instr);

2983

// An alignment of 0 means target abi alignment. We need to use the scalar's

2984

// target abi alignment in such a case.

2985

const DataLayout &DL = Instr->getModule()->getDataLayout();

2986

if (!Alignment)

2987

Alignment = DL.getABITypeAlignment(ScalarDataTy);

2988

unsigned AddressSpace = getMemInstAddressSpace(Instr);

2989

2990

// Scalarize the memory instruction if necessary.

2991

if (Decision == LoopVectorizationCostModel::CM_Scalarize)

2992

return scalarizeInstruction(Instr, Legal->isScalarWithPredication(Instr));

2993

2994

// Determine if the pointer operand of the access is either consecutive or

2995

// reverse consecutive.

2996

int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);

2997

bool Reverse = ConsecutiveStride < 0;

2998

bool CreateGatherScatter =

2999

(Decision == LoopVectorizationCostModel::CM_GatherScatter);

3000

3001

// Either Ptr feeds a vector load/store, or a vector GEP should feed a vector

3002

// gather/scatter. Otherwise Decision should have been to Scalarize.

3003

assert((ConsecutiveStride || CreateGatherScatter) &&(((ConsecutiveStride || CreateGatherScatter) && "The instruction should be scalarized"
) ? static_cast<void> (0) : __assert_fail ("(ConsecutiveStride || CreateGatherScatter) && \"The instruction should be scalarized\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3004, __PRETTY_FUNCTION__))

3004

"The instruction should be scalarized")(((ConsecutiveStride || CreateGatherScatter) && "The instruction should be scalarized"
) ? static_cast<void> (0) : __assert_fail ("(ConsecutiveStride || CreateGatherScatter) && \"The instruction should be scalarized\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3004, __PRETTY_FUNCTION__));

3005

3006

// Handle consecutive loads/stores.

3007

if (ConsecutiveStride)

3008

Ptr = getOrCreateScalarValue(Ptr, 0, 0);

3009

3010

VectorParts Mask = createBlockInMask(Instr->getParent());

3011

// Handle Stores:

3012

if (SI) {

3013

assert(!Legal->isUniform(SI->getPointerOperand()) &&((!Legal->isUniform(SI->getPointerOperand()) &&
"We do not allow storing to uniform addresses") ? static_cast
<void> (0) : __assert_fail ("!Legal->isUniform(SI->getPointerOperand()) && \"We do not allow storing to uniform addresses\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3014, __PRETTY_FUNCTION__))

3014

"We do not allow storing to uniform addresses")((!Legal->isUniform(SI->getPointerOperand()) &&
"We do not allow storing to uniform addresses") ? static_cast
<void> (0) : __assert_fail ("!Legal->isUniform(SI->getPointerOperand()) && \"We do not allow storing to uniform addresses\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3014, __PRETTY_FUNCTION__));

3015

setDebugLocFromInst(Builder, SI);

3016

3017

for (unsigned Part = 0; Part < UF; ++Part) {

3018

Instruction *NewSI = nullptr;

3019

Value *StoredVal = getOrCreateVectorValue(SI->getValueOperand(), Part);

3020

if (CreateGatherScatter) {

3021

Value *MaskPart = Legal->isMaskRequired(SI) ? Mask[Part] : nullptr;

3022

Value *VectorGep = getOrCreateVectorValue(Ptr, Part);

3023

NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,

3024

MaskPart);

3025

} else {

3026

// Calculate the pointer for the specific unroll-part.

3027

Value *PartPtr =

3028

Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(Part * VF));

3029

3030

if (Reverse) {

3031

// If we store to reverse consecutive memory locations, then we need

3032

// to reverse the order of elements in the stored value.

3033

StoredVal = reverseVector(StoredVal);

3034

// We don't want to update the value in the map as it might be used in

3035

// another expression. So don't call resetVectorValue(StoredVal).

3036

3037

// If the address is consecutive but reversed, then the

3038

// wide store needs to start at the last vector element.

3039

PartPtr =

3040

Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(-Part * VF));

3041

PartPtr =

3042

Builder.CreateGEP(nullptr, PartPtr, Builder.getInt32(1 - VF));

3043

Mask[Part] = reverseVector(Mask[Part]);

3044

}

3045

3046

Value *VecPtr =

3047

Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace));

3048

3049

if (Legal->isMaskRequired(SI))

3050

NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,

3051

Mask[Part]);

3052

else

3053

NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);

3054

}

3055

addMetadata(NewSI, SI);

3056

}

3057

return;

3058

}

3059

3060

// Handle loads.

3061

assert(LI && "Must have a load instruction")((LI && "Must have a load instruction") ? static_cast
<void> (0) : __assert_fail ("LI && \"Must have a load instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3061, __PRETTY_FUNCTION__));

3062

setDebugLocFromInst(Builder, LI);

3063

for (unsigned Part = 0; Part < UF; ++Part) {

3064

Value *NewLI;

3065

if (CreateGatherScatter) {

3066

Value *MaskPart = Legal->isMaskRequired(LI) ? Mask[Part] : nullptr;

3067

Value *VectorGep = getOrCreateVectorValue(Ptr, Part);

3068

NewLI = Builder.CreateMaskedGather(VectorGep, Alignment, MaskPart,

3069

nullptr, "wide.masked.gather");

3070

addMetadata(NewLI, LI);

3071

} else {

3072

// Calculate the pointer for the specific unroll-part.

3073

Value *PartPtr =

3074

Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(Part * VF));

3075

3076

if (Reverse) {

3077

// If the address is consecutive but reversed, then the

3078

// wide load needs to start at the last vector element.

3079

PartPtr = Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(-Part * VF));

3080

PartPtr = Builder.CreateGEP(nullptr, PartPtr, Builder.getInt32(1 - VF));

3081

Mask[Part] = reverseVector(Mask[Part]);

3082

}

3083

3084

Value *VecPtr =

3085

Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace));

3086

if (Legal->isMaskRequired(LI))

3087

NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment, Mask[Part],

3088

UndefValue::get(DataTy),

3089

"wide.masked.load");

3090

else

3091

NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load");

3092

3093

// Add metadata to the load, but setVectorValue to the reverse shuffle.

3094

addMetadata(NewLI, LI);

3095

if (Reverse)

3096

NewLI = reverseVector(NewLI);

3097

}

3098

VectorLoopValueMap.setVectorValue(Instr, Part, NewLI);

3099

}

3100

}

3101

3102

void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr,

3103

bool IfPredicateInstr) {

3104

assert(!Instr->getType()->isAggregateType() && "Can't handle vectors")((!Instr->getType()->isAggregateType() && "Can't handle vectors"
) ? static_cast<void> (0) : __assert_fail ("!Instr->getType()->isAggregateType() && \"Can't handle vectors\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3104, __PRETTY_FUNCTION__));

3105

DEBUG(dbgs() << "LV: Scalarizing"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Scalarizing" <<
(IfPredicateInstr ? " and predicating:" : ":") << *Instr
<< '\n'; } } while (false)

3106

<< (IfPredicateInstr ? " and predicating:" : ":") << *Instrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Scalarizing" <<
(IfPredicateInstr ? " and predicating:" : ":") << *Instr
<< '\n'; } } while (false)

3107

<< '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Scalarizing" <<
(IfPredicateInstr ? " and predicating:" : ":") << *Instr
<< '\n'; } } while (false);

3108

// Holds vector parameters or scalars, in case of uniform vals.

3109

SmallVector<VectorParts, 4> Params;

3110

3111

setDebugLocFromInst(Builder, Instr);

3112

3113

// Does this instruction return a value ?

3114

bool IsVoidRetTy = Instr->getType()->isVoidTy();

3115

3116

VectorParts Cond;

3117

if (IfPredicateInstr)

3118

Cond = createBlockInMask(Instr->getParent());

3119

3120

// Determine the number of scalars we need to generate for each unroll

3121

// iteration. If the instruction is uniform, we only need to generate the

3122

// first lane. Otherwise, we generate all VF values.

3123

unsigned Lanes = Cost->isUniformAfterVectorization(Instr, VF) ? 1 : VF;

3124

3125

// For each vector unroll 'part':

3126

for (unsigned Part = 0; Part < UF; ++Part) {

3127

// For each scalar that we create:

3128

for (unsigned Lane = 0; Lane < Lanes; ++Lane) {

3129

3130

// Start if-block.

3131

Value *Cmp = nullptr;

3132

if (IfPredicateInstr) {

3133

Cmp = Cond[Part];

3134

if (Cmp->getType()->isVectorTy())

3135

Cmp = Builder.CreateExtractElement(Cmp, Builder.getInt32(Lane));

3136

Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Cmp,

3137

ConstantInt::get(Cmp->getType(), 1));

3138

}

3139

3140

Instruction *Cloned = Instr->clone();

3141

if (!IsVoidRetTy)

3142

Cloned->setName(Instr->getName() + ".cloned");

3143

3144

// Replace the operands of the cloned instructions with their scalar

3145

// equivalents in the new loop.

3146

for (unsigned op = 0, e = Instr->getNumOperands(); op != e; ++op) {

3147

auto *NewOp = getOrCreateScalarValue(Instr->getOperand(op), Part, Lane);

3148

Cloned->setOperand(op, NewOp);

3149

}

3150

addNewMetadata(Cloned, Instr);

3151

3152

// Place the cloned scalar in the new loop.

3153

Builder.Insert(Cloned);

3154

3155

// Add the cloned scalar to the scalar map entry.

3156

VectorLoopValueMap.setScalarValue(Instr, Part, Lane, Cloned);

3157

3158

// If we just cloned a new assumption, add it the assumption cache.

3159

if (auto *II = dyn_cast<IntrinsicInst>(Cloned))

3160

if (II->getIntrinsicID() == Intrinsic::assume)

3161

AC->registerAssumption(II);

3162

3163

// End if-block.

3164

if (IfPredicateInstr)

3165

PredicatedInstructions.push_back(std::make_pair(Cloned, Cmp));

3166

}

3167

}

3168

}

3169

3170

PHINode *InnerLoopVectorizer::createInductionVariable(Loop *L, Value *Start,

3171

Value *End, Value *Step,

3172

Instruction *DL) {

3173

BasicBlock *Header = L->getHeader();

3174

BasicBlock *Latch = L->getLoopLatch();

3175

// As we're just creating this loop, it's possible no latch exists

3176

// yet. If so, use the header as this will be a single block loop.

3177

if (!Latch)

3178

Latch = Header;

3179

3180

IRBuilder<> Builder(&*Header->getFirstInsertionPt());

3181

Instruction *OldInst = getDebugLocFromInstOrOperands(OldInduction);

3182

setDebugLocFromInst(Builder, OldInst);

3183

auto *Induction = Builder.CreatePHI(Start->getType(), 2, "index");

3184

3185

Builder.SetInsertPoint(Latch->getTerminator());

3186

setDebugLocFromInst(Builder, OldInst);

3187

3188

// Create i+1 and fill the PHINode.

3189

Value *Next = Builder.CreateAdd(Induction, Step, "index.next");

3190

Induction->addIncoming(Start, L->getLoopPreheader());

3191

Induction->addIncoming(Next, Latch);

3192

// Create the compare.

3193

Value *ICmp = Builder.CreateICmpEQ(Next, End);

3194

Builder.CreateCondBr(ICmp, L->getExitBlock(), Header);

3195

3196

// Now we have two terminators. Remove the old one from the block.

3197

Latch->getTerminator()->eraseFromParent();

3198

3199

return Induction;

3200

}

3201

3202

Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {

3203

if (TripCount)

3204

return TripCount;

3205

3206

IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());

3207

// Find the loop boundaries.

3208

ScalarEvolution *SE = PSE.getSE();

3209

const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();

3210

assert(BackedgeTakenCount != SE->getCouldNotCompute() &&((BackedgeTakenCount != SE->getCouldNotCompute() &&
"Invalid loop count") ? static_cast<void> (0) : __assert_fail
("BackedgeTakenCount != SE->getCouldNotCompute() && \"Invalid loop count\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3211, __PRETTY_FUNCTION__))

3211

"Invalid loop count")((BackedgeTakenCount != SE->getCouldNotCompute() &&
"Invalid loop count") ? static_cast<void> (0) : __assert_fail
("BackedgeTakenCount != SE->getCouldNotCompute() && \"Invalid loop count\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3211, __PRETTY_FUNCTION__));

3212

3213

Type *IdxTy = Legal->getWidestInductionType();

3214

3215

// The exit count might have the type of i64 while the phi is i32. This can

3216

// happen if we have an induction variable that is sign extended before the

3217

// compare. The only way that we get a backedge taken count is that the

3218

// induction variable was signed and as such will not overflow. In such a case

3219

// truncation is legal.

3220

if (BackedgeTakenCount->getType()->getPrimitiveSizeInBits() >

3221

IdxTy->getPrimitiveSizeInBits())

3222

BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy);

3223

BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);

3224

3225

// Get the total trip count from the count by adding 1.

3226

const SCEV *ExitCount = SE->getAddExpr(

3227

BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));

3228

3229

const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();

3230

3231

// Expand the trip count and place the new instructions in the preheader.

3232

// Notice that the pre-header does not change, only the loop body.

3233

SCEVExpander Exp(*SE, DL, "induction");

3234

3235

// Count holds the overall loop count (N).

3236

TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(),

3237

L->getLoopPreheader()->getTerminator());

3238

3239

if (TripCount->getType()->isPointerTy())

3240

TripCount =

3241

CastInst::CreatePointerCast(TripCount, IdxTy, "exitcount.ptrcnt.to.int",

3242

L->getLoopPreheader()->getTerminator());

3243

3244

return TripCount;

3245

}

3246

3247

Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {

3248

if (VectorTripCount)

3249

return VectorTripCount;

3250

3251

Value *TC = getOrCreateTripCount(L);

3252

IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());

3253

3254

// Now we need to generate the expression for the part of the loop that the

3255

// vectorized body will execute. This is equal to N - (N % Step) if scalar

3256

// iterations are not required for correctness, or N - Step, otherwise. Step

3257

// is equal to the vectorization factor (number of SIMD elements) times the

3258

// unroll factor (number of SIMD instructions).

3259

Constant *Step = ConstantInt::get(TC->getType(), VF * UF);

3260

Value *R = Builder.CreateURem(TC, Step, "n.mod.vf");

3261

3262

// If there is a non-reversed interleaved group that may speculatively access

3263

// memory out-of-bounds, we need to ensure that there will be at least one

3264

// iteration of the scalar epilogue loop. Thus, if the step evenly divides

3265

// the trip count, we set the remainder to be equal to the step. If the step

3266

// does not evenly divide the trip count, no adjustment is necessary since

3267

// there will already be scalar iterations. Note that the minimum iterations

3268

// check ensures that N >= Step.

3269

if (VF > 1 && Legal->requiresScalarEpilogue()) {

3270

auto *IsZero = Builder.CreateICmpEQ(R, ConstantInt::get(R->getType(), 0));

3271

R = Builder.CreateSelect(IsZero, Step, R);

3272

}

3273

3274

VectorTripCount = Builder.CreateSub(TC, R, "n.vec");

3275

3276

return VectorTripCount;

3277

}

3278

3279

void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,

3280

BasicBlock *Bypass) {

3281

Value *Count = getOrCreateTripCount(L);

3282

BasicBlock *BB = L->getLoopPreheader();

3283

IRBuilder<> Builder(BB->getTerminator());

3284

3285

// Generate code to check that the loop's trip count that we computed by

3286

// adding one to the backedge-taken count will not overflow.

3287

Value *CheckMinIters = Builder.CreateICmpULT(

3288

Count, ConstantInt::get(Count->getType(), VF * UF), "min.iters.check");

3289

3290

BasicBlock *NewBB =

3291

BB->splitBasicBlock(BB->getTerminator(), "min.iters.checked");

3292

// Update dominator tree immediately if the generated block is a

3293

// LoopBypassBlock because SCEV expansions to generate loop bypass

3294

// checks may query it before the current function is finished.

3295

DT->addNewBlock(NewBB, BB);

3296

if (L->getParentLoop())

3297

L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);

3298

ReplaceInstWithInst(BB->getTerminator(),

3299

BranchInst::Create(Bypass, NewBB, CheckMinIters));

3300

LoopBypassBlocks.push_back(BB);

3301

}

3302

3303

void InnerLoopVectorizer::emitVectorLoopEnteredCheck(Loop *L,

3304

BasicBlock *Bypass) {

3305

Value *TC = getOrCreateVectorTripCount(L);

3306

BasicBlock *BB = L->getLoopPreheader();

3307

IRBuilder<> Builder(BB->getTerminator());

3308

3309

// Now, compare the new count to zero. If it is zero skip the vector loop and

3310

// jump to the scalar loop.

3311

Value *Cmp = Builder.CreateICmpEQ(TC, Constant::getNullValue(TC->getType()),

3312

"cmp.zero");

3313

3314

// Generate code to check that the loop's trip count that we computed by

3315

// adding one to the backedge-taken count will not overflow.

3316

BasicBlock *NewBB = BB->splitBasicBlock(BB->getTerminator(), "vector.ph");

3317

// Update dominator tree immediately if the generated block is a

3318

// LoopBypassBlock because SCEV expansions to generate loop bypass

3319

// checks may query it before the current function is finished.

3320

DT->addNewBlock(NewBB, BB);

3321

if (L->getParentLoop())

3322

L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);

3323

ReplaceInstWithInst(BB->getTerminator(),

3324

BranchInst::Create(Bypass, NewBB, Cmp));

3325

LoopBypassBlocks.push_back(BB);

3326

}

3327

3328

void InnerLoopVectorizer::emitSCEVChecks(Loop *L, BasicBlock *Bypass) {

3329

BasicBlock *BB = L->getLoopPreheader();

3330

3331

// Generate the code to check that the SCEV assumptions that we made.

3332

// We want the new basic block to start at the first instruction in a

3333

// sequence of instructions that form a check.

3334

SCEVExpander Exp(*PSE.getSE(), Bypass->getModule()->getDataLayout(),

3335

"scev.check");

3336

Value *SCEVCheck =

3337

Exp.expandCodeForPredicate(&PSE.getUnionPredicate(), BB->getTerminator());

3338

3339

if (auto *C = dyn_cast<ConstantInt>(SCEVCheck))

3340

if (C->isZero())

3341

return;

3342

3343

// Create a new block containing the stride check.

3344

BB->setName("vector.scevcheck");

3345

auto *NewBB = BB->splitBasicBlock(BB->getTerminator(), "vector.ph");

3346

// Update dominator tree immediately if the generated block is a

3347

// LoopBypassBlock because SCEV expansions to generate loop bypass

3348

// checks may query it before the current function is finished.

3349

DT->addNewBlock(NewBB, BB);

3350

if (L->getParentLoop())

3351

L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);

3352

ReplaceInstWithInst(BB->getTerminator(),

3353

BranchInst::Create(Bypass, NewBB, SCEVCheck));

3354

LoopBypassBlocks.push_back(BB);

3355

AddedSafetyChecks = true;

3356

}

3357

3358

void InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass) {

3359

BasicBlock *BB = L->getLoopPreheader();

3360

3361

// Generate the code that checks in runtime if arrays overlap. We put the

3362

// checks into a separate block to make the more common case of few elements

3363

// faster.

3364

Instruction *FirstCheckInst;

3365

Instruction *MemRuntimeCheck;

3366

std::tie(FirstCheckInst, MemRuntimeCheck) =

3367

Legal->getLAI()->addRuntimeChecks(BB->getTerminator());

3368

if (!MemRuntimeCheck)

3369

return;

3370

3371

// Create a new block containing the memory check.

3372

BB->setName("vector.memcheck");

3373

auto *NewBB = BB->splitBasicBlock(BB->getTerminator(), "vector.ph");

3374

// Update dominator tree immediately if the generated block is a

3375

// LoopBypassBlock because SCEV expansions to generate loop bypass

3376

// checks may query it before the current function is finished.

3377

DT->addNewBlock(NewBB, BB);

3378

if (L->getParentLoop())

3379

L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);

3380

ReplaceInstWithInst(BB->getTerminator(),

3381

BranchInst::Create(Bypass, NewBB, MemRuntimeCheck));

3382

LoopBypassBlocks.push_back(BB);

3383

AddedSafetyChecks = true;

3384

3385

// We currently don't use LoopVersioning for the actual loop cloning but we

3386

// still use it to add the noalias metadata.

3387

LVer = llvm::make_unique<LoopVersioning>(*Legal->getLAI(), OrigLoop, LI, DT,

3388

PSE.getSE());

3389

LVer->prepareNoAliasMetadata();

3390

}

3391

3392

void InnerLoopVectorizer::createVectorizedLoopSkeleton() {

3393

3394

In this function we generate a new loop. The new loop will contain

3395

the vectorized instructions while the old loop will continue to run the

3396

scalar remainder.

3397

3398

[ ] <-- loop iteration number check.

3399

/ |

3400

/ v

3401

| [ ] <-- vector loop bypass (may consist of multiple blocks).

3402

| / |

3403

| / v

3404

|| [ ] <-- vector pre header.

3405

|/ |

3406

| v

3407

| [ ] \

3408

| [ ]_| <-- vector loop.

3409

| |

3410

| v

3411

| -[ ] <--- middle-block.

3412

| / |

3413

| / v

3414

-|- >[ ] <--- new preheader.

3415

| |

3416

| v

3417

| [ ] \

3418

| [ ]_| <-- old scalar loop to handle remainder.

3419

\ |

3420

\ v

3421

>[ ] <-- exit block.

3422

...

3423

3424

3425

BasicBlock *OldBasicBlock = OrigLoop->getHeader();

3426

BasicBlock *VectorPH = OrigLoop->getLoopPreheader();

3427

BasicBlock *ExitBlock = OrigLoop->getExitBlock();

3428

assert(VectorPH && "Invalid loop structure")((VectorPH && "Invalid loop structure") ? static_cast
<void> (0) : __assert_fail ("VectorPH && \"Invalid loop structure\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3428, __PRETTY_FUNCTION__));

3429

assert(ExitBlock && "Must have an exit block")((ExitBlock && "Must have an exit block") ? static_cast
<void> (0) : __assert_fail ("ExitBlock && \"Must have an exit block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3429, __PRETTY_FUNCTION__));

3430

3431

// Some loops have a single integer induction variable, while other loops

3432

// don't. One example is c++ iterators that often have multiple pointer

3433

// induction variables. In the code below we also support a case where we

3434

// don't have a single induction variable.

3435

3436

// We try to obtain an induction variable from the original loop as hard

3437

// as possible. However if we don't find one that:

3438

// - is an integer

3439

// - counts from zero, stepping by one

3440

// - is the size of the widest induction variable type

3441

// then we create a new one.

3442

OldInduction = Legal->getPrimaryInduction();

3443

Type *IdxTy = Legal->getWidestInductionType();

3444

3445

// Split the single block loop into the two loop structure described above.

3446

BasicBlock *VecBody =

3447

VectorPH->splitBasicBlock(VectorPH->getTerminator(), "vector.body");

3448

BasicBlock *MiddleBlock =

3449

VecBody->splitBasicBlock(VecBody->getTerminator(), "middle.block");

3450

BasicBlock *ScalarPH =

3451

MiddleBlock->splitBasicBlock(MiddleBlock->getTerminator(), "scalar.ph");

3452

3453

// Create and register the new vector loop.

3454

Loop *Lp = new Loop();

3455

Loop *ParentLoop = OrigLoop->getParentLoop();

3456

3457

// Insert the new loop into the loop nest and register the new basic blocks

3458

// before calling any utilities such as SCEV that require valid LoopInfo.

3459

if (ParentLoop) {

3460

ParentLoop->addChildLoop(Lp);

3461

ParentLoop->addBasicBlockToLoop(ScalarPH, *LI);

3462

ParentLoop->addBasicBlockToLoop(MiddleBlock, *LI);

3463

} else {

3464

LI->addTopLevelLoop(Lp);

3465

}

3466

Lp->addBasicBlockToLoop(VecBody, *LI);

3467

3468

// Find the loop boundaries.

3469

Value *Count = getOrCreateTripCount(Lp);

3470

3471

Value *StartIdx = ConstantInt::get(IdxTy, 0);

3472

3473

// We need to test whether the backedge-taken count is uint##_max. Adding one

3474

// to it will cause overflow and an incorrect loop trip count in the vector

3475

// body. In case of overflow we want to directly jump to the scalar remainder

3476

// loop.

3477

emitMinimumIterationCountCheck(Lp, ScalarPH);

3478

// Now, compare the new count to zero. If it is zero skip the vector loop and

3479

// jump to the scalar loop.

3480

emitVectorLoopEnteredCheck(Lp, ScalarPH);

3481

// Generate the code to check any assumptions that we've made for SCEV

3482

// expressions.

3483

emitSCEVChecks(Lp, ScalarPH);

3484

3485

// Generate the code that checks in runtime if arrays overlap. We put the

3486

// checks into a separate block to make the more common case of few elements

3487

// faster.

3488

emitMemRuntimeChecks(Lp, ScalarPH);

3489

3490

// Generate the induction variable.

3491

// The loop step is equal to the vectorization factor (num of SIMD elements)

3492

// times the unroll factor (num of SIMD instructions).

3493

Value *CountRoundDown = getOrCreateVectorTripCount(Lp);

3494

Constant *Step = ConstantInt::get(IdxTy, VF * UF);

3495

Induction =

3496

createInductionVariable(Lp, StartIdx, CountRoundDown, Step,

3497

getDebugLocFromInstOrOperands(OldInduction));

3498

3499

// We are going to resume the execution of the scalar loop.

3500

// Go over all of the induction variables that we found and fix the

3501

// PHIs that are left in the scalar version of the loop.

3502

// The starting values of PHI nodes depend on the counter of the last

3503

// iteration in the vectorized loop.

3504

// If we come from a bypass edge then we need to start from the original

3505

// start value.

3506

3507

// This variable saves the new starting index for the scalar loop. It is used

3508

// to test if there are any tail iterations left once the vector loop has

3509

// completed.

3510

LoopVectorizationLegality::InductionList *List = Legal->getInductionVars();

3511

for (auto &InductionEntry : *List) {

3512

PHINode *OrigPhi = InductionEntry.first;

3513

InductionDescriptor II = InductionEntry.second;

3514

3515

// Create phi nodes to merge from the backedge-taken check block.

3516

PHINode *BCResumeVal = PHINode::Create(

3517

OrigPhi->getType(), 3, "bc.resume.val", ScalarPH->getTerminator());

3518

Value *&EndValue = IVEndValues[OrigPhi];

3519

if (OrigPhi == OldInduction) {

3520

// We know what the end value is.

3521

EndValue = CountRoundDown;

3522

} else {

3523

IRBuilder<> B(LoopBypassBlocks.back()->getTerminator());

3524

Type *StepType = II.getStep()->getType();

3525

Instruction::CastOps CastOp =

3526

CastInst::getCastOpcode(CountRoundDown, true, StepType, true);

3527

Value *CRD = B.CreateCast(CastOp, CountRoundDown, StepType, "cast.crd");

3528

const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout();

3529

EndValue = II.transform(B, CRD, PSE.getSE(), DL);

3530

EndValue->setName("ind.end");

3531

}

3532

3533

// The new PHI merges the original incoming value, in case of a bypass,

3534

// or the value at the end of the vectorized loop.

3535

BCResumeVal->addIncoming(EndValue, MiddleBlock);

3536

3537

// Fix the scalar body counter (PHI node).

3538

unsigned BlockIdx = OrigPhi->getBasicBlockIndex(ScalarPH);

3539

3540

// The old induction's phi node in the scalar body needs the truncated

3541

// value.

3542

for (BasicBlock *BB : LoopBypassBlocks)

3543

BCResumeVal->addIncoming(II.getStartValue(), BB);

3544

OrigPhi->setIncomingValue(BlockIdx, BCResumeVal);

3545

}

3546

3547

// Add a check in the middle block to see if we have completed

3548

// all of the iterations in the first vector loop.

3549

// If (N - N%VF) == N, then we *don't* need to run the remainder.

3550

Value *CmpN =

3551

CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, Count,

3552

CountRoundDown, "cmp.n", MiddleBlock->getTerminator());

3553

ReplaceInstWithInst(MiddleBlock->getTerminator(),

3554

BranchInst::Create(ExitBlock, ScalarPH, CmpN));

3555

3556

// Get ready to start creating new instructions into the vectorized body.

3557

Builder.SetInsertPoint(&*VecBody->getFirstInsertionPt());

3558

3559

// Save the state.

3560

LoopVectorPreHeader = Lp->getLoopPreheader();

3561

LoopScalarPreHeader = ScalarPH;

3562

LoopMiddleBlock = MiddleBlock;

3563

LoopExitBlock = ExitBlock;

3564

LoopVectorBody = VecBody;

3565

LoopScalarBody = OldBasicBlock;

3566

3567

// Keep all loop hints from the original loop on the vector loop (we'll

3568

// replace the vectorizer-specific hints below).

3569

if (MDNode *LID = OrigLoop->getLoopID())

3570

Lp->setLoopID(LID);

3571

3572

LoopVectorizeHints Hints(Lp, true, *ORE);

3573

Hints.setAlreadyVectorized();

3574

}

3575

3576

// Fix up external users of the induction variable. At this point, we are

3577

// in LCSSA form, with all external PHIs that use the IV having one input value,

3578

// coming from the remainder loop. We need those PHIs to also have a correct

3579

// value for the IV when arriving directly from the middle block.

3580

void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,

3581

const InductionDescriptor &II,

3582

Value *CountRoundDown, Value *EndValue,

3583

BasicBlock *MiddleBlock) {

3584

// There are two kinds of external IV usages - those that use the value

3585

// computed in the last iteration (the PHI) and those that use the penultimate

3586

// value (the value that feeds into the phi from the loop latch).

3587

// We allow both, but they, obviously, have different values.

3588

3589

assert(OrigLoop->getExitBlock() && "Expected a single exit block")((OrigLoop->getExitBlock() && "Expected a single exit block"
) ? static_cast<void> (0) : __assert_fail ("OrigLoop->getExitBlock() && \"Expected a single exit block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3589, __PRETTY_FUNCTION__));

3590

3591

DenseMap<Value *, Value *> MissingVals;

3592

3593

// An external user of the last iteration's value should see the value that

3594

// the remainder loop uses to initialize its own IV.

3595

Value *PostInc = OrigPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch());

3596

for (User *U : PostInc->users()) {

3597

Instruction *UI = cast<Instruction>(U);

3598

if (!OrigLoop->contains(UI)) {

3599

assert(isa<PHINode>(UI) && "Expected LCSSA form")((isa<PHINode>(UI) && "Expected LCSSA form") ? static_cast
<void> (0) : __assert_fail ("isa<PHINode>(UI) && \"Expected LCSSA form\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3599, __PRETTY_FUNCTION__));

3600

MissingVals[UI] = EndValue;

3601

}

3602

}

3603

3604

// An external user of the penultimate value need to see EndValue - Step.

3605

// The simplest way to get this is to recompute it from the constituent SCEVs,

3606

// that is Start + (Step * (CRD - 1)).

3607

for (User *U : OrigPhi->users()) {

3608

auto *UI = cast<Instruction>(U);

3609

if (!OrigLoop->contains(UI)) {

3610

const DataLayout &DL =

3611

OrigLoop->getHeader()->getModule()->getDataLayout();

3612

3613

3614

IRBuilder<> B(MiddleBlock->getTerminator());

3615

Value *CountMinusOne = B.CreateSub(

3616

CountRoundDown, ConstantInt::get(CountRoundDown->getType(), 1));

3617

Value *CMO =

3618

!II.getStep()->getType()->isIntegerTy()

3619

? B.CreateCast(Instruction::SIToFP, CountMinusOne,

3620

II.getStep()->getType())

3621

: B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType());

3622

CMO->setName("cast.cmo");

3623

Value *Escape = II.transform(B, CMO, PSE.getSE(), DL);

3624

Escape->setName("ind.escape");

3625

MissingVals[UI] = Escape;

3626

}

3627

}

3628

3629

for (auto &I : MissingVals) {

3630

PHINode *PHI = cast<PHINode>(I.first);

3631

// One corner case we have to handle is two IVs "chasing" each-other,

3632

// that is %IV2 = phi [...], [ %IV1, %latch ]

3633

// In this case, if IV1 has an external use, we need to avoid adding both

3634

// "last value of IV1" and "penultimate value of IV2". So, verify that we

3635

// don't already have an incoming value for the middle block.

3636

if (PHI->getBasicBlockIndex(MiddleBlock) == -1)

3637

PHI->addIncoming(I.second, MiddleBlock);

3638

}

3639

}

3640

3641

namespace {

3642

struct CSEDenseMapInfo {

3643

static bool canHandle(const Instruction *I) {

3644

return isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||

3645

isa<ShuffleVectorInst>(I) || isa<GetElementPtrInst>(I);

3646

}

3647

static inline Instruction *getEmptyKey() {

3648

return DenseMapInfo<Instruction *>::getEmptyKey();

3649

}

3650

static inline Instruction *getTombstoneKey() {

3651

return DenseMapInfo<Instruction *>::getTombstoneKey();

3652

}

3653

static unsigned getHashValue(const Instruction *I) {

3654

assert(canHandle(I) && "Unknown instruction!")((canHandle(I) && "Unknown instruction!") ? static_cast
<void> (0) : __assert_fail ("canHandle(I) && \"Unknown instruction!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3654, __PRETTY_FUNCTION__));

3655

return hash_combine(I->getOpcode(), hash_combine_range(I->value_op_begin(),

3656

I->value_op_end()));

3657

}

3658

static bool isEqual(const Instruction *LHS, const Instruction *RHS) {

3659

if (LHS == getEmptyKey() || RHS == getEmptyKey() ||

3660

LHS == getTombstoneKey() || RHS == getTombstoneKey())

3661

return LHS == RHS;

3662

return LHS->isIdenticalTo(RHS);

3663

}

3664

};

3665

}

3666

3667

///\brief Perform cse of induction variable instructions.

3668

static void cse(BasicBlock *BB) {

3669

// Perform simple cse.

3670

SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;

3671

for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {

3672

Instruction *In = &*I++;

3673

3674

if (!CSEDenseMapInfo::canHandle(In))

3675

continue;

3676

3677

// Check if we can replace this instruction with any of the

3678

// visited instructions.

3679

if (Instruction *V = CSEMap.lookup(In)) {

3680

In->replaceAllUsesWith(V);

3681

In->eraseFromParent();

3682

continue;

3683

}

3684

3685

CSEMap[In] = In;

3686

}

3687

}

3688

3689

/// \brief Estimate the overhead of scalarizing an instruction. This is a

3690

/// convenience wrapper for the type-based getScalarizationOverhead API.

3691

static unsigned getScalarizationOverhead(Instruction *I, unsigned VF,

3692

const TargetTransformInfo &TTI) {

3693

if (VF == 1)

3694

return 0;

3695

3696

unsigned Cost = 0;

3697

Type *RetTy = ToVectorTy(I->getType(), VF);

3698

if (!RetTy->isVoidTy() &&

3699

(!isa<LoadInst>(I) ||

3700

!TTI.supportsEfficientVectorElementLoadStore()))

3701

Cost += TTI.getScalarizationOverhead(RetTy, true, false);

3702

3703

if (CallInst *CI = dyn_cast<CallInst>(I)) {

3704

SmallVector<const Value *, 4> Operands(CI->arg_operands());

3705

Cost += TTI.getOperandsScalarizationOverhead(Operands, VF);

3706

}

3707

else if (!isa<StoreInst>(I) ||

3708

!TTI.supportsEfficientVectorElementLoadStore()) {

3709

SmallVector<const Value *, 4> Operands(I->operand_values());

3710

Cost += TTI.getOperandsScalarizationOverhead(Operands, VF);

3711

}

3712

3713

return Cost;

3714

}

3715

3716

// Estimate cost of a call instruction CI if it were vectorized with factor VF.

3717

// Return the cost of the instruction, including scalarization overhead if it's

3718

// needed. The flag NeedToScalarize shows if the call needs to be scalarized -

3719

// i.e. either vector version isn't available, or is too expensive.

3720

static unsigned getVectorCallCost(CallInst *CI, unsigned VF,

3721

const TargetTransformInfo &TTI,

3722

const TargetLibraryInfo *TLI,

3723

bool &NeedToScalarize) {

3724

Function *F = CI->getCalledFunction();

3725

StringRef FnName = CI->getCalledFunction()->getName();

3726

Type *ScalarRetTy = CI->getType();

3727

SmallVector<Type *, 4> Tys, ScalarTys;

3728

for (auto &ArgOp : CI->arg_operands())

3729

ScalarTys.push_back(ArgOp->getType());

3730

3731

// Estimate cost of scalarized vector call. The source operands are assumed

3732

// to be vectors, so we need to extract individual elements from there,

3733

// execute VF scalar calls, and then gather the result into the vector return

3734

// value.

3735

unsigned ScalarCallCost = TTI.getCallInstrCost(F, ScalarRetTy, ScalarTys);

3736

if (VF == 1)

3737

return ScalarCallCost;

3738

3739

// Compute corresponding vector type for return value and arguments.

3740

Type *RetTy = ToVectorTy(ScalarRetTy, VF);

3741

for (Type *ScalarTy : ScalarTys)

3742

Tys.push_back(ToVectorTy(ScalarTy, VF));

3743

3744

// Compute costs of unpacking argument values for the scalar calls and

3745

// packing the return values to a vector.

3746

unsigned ScalarizationCost = getScalarizationOverhead(CI, VF, TTI);

3747

3748

unsigned Cost = ScalarCallCost * VF + ScalarizationCost;

3749

3750

// If we can't emit a vector call for this function, then the currently found

3751

// cost is the cost we need to return.

3752

NeedToScalarize = true;

3753

if (!TLI || !TLI->isFunctionVectorizable(FnName, VF) || CI->isNoBuiltin())

3754

return Cost;

3755

3756

// If the corresponding vector cost is cheaper, return its cost.

3757

unsigned VectorCallCost = TTI.getCallInstrCost(nullptr, RetTy, Tys);

3758

if (VectorCallCost < Cost) {

3759

NeedToScalarize = false;

3760

return VectorCallCost;

3761

}

3762

return Cost;

3763

}

3764

3765

// Estimate cost of an intrinsic call instruction CI if it were vectorized with

3766

// factor VF. Return the cost of the instruction, including scalarization

3767

// overhead if it's needed.

3768

static unsigned getVectorIntrinsicCost(CallInst *CI, unsigned VF,

3769

const TargetTransformInfo &TTI,

3770

const TargetLibraryInfo *TLI) {

3771

Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);

3772

assert(ID && "Expected intrinsic call!")((ID && "Expected intrinsic call!") ? static_cast<
void> (0) : __assert_fail ("ID && \"Expected intrinsic call!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3772, __PRETTY_FUNCTION__));

3773

3774

FastMathFlags FMF;

3775

if (auto *FPMO = dyn_cast<FPMathOperator>(CI))

3776

FMF = FPMO->getFastMathFlags();

3777

3778

SmallVector<Value *, 4> Operands(CI->arg_operands());

3779

return TTI.getIntrinsicInstrCost(ID, CI->getType(), Operands, FMF, VF);

3780

}

3781

3782

static Type *smallestIntegerVectorType(Type *T1, Type *T2) {

3783

auto *I1 = cast<IntegerType>(T1->getVectorElementType());

3784

auto *I2 = cast<IntegerType>(T2->getVectorElementType());

3785

return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2;

3786

}

3787

static Type *largestIntegerVectorType(Type *T1, Type *T2) {

3788

auto *I1 = cast<IntegerType>(T1->getVectorElementType());

3789

auto *I2 = cast<IntegerType>(T2->getVectorElementType());

3790

return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2;

3791

}

3792

3793

void InnerLoopVectorizer::truncateToMinimalBitwidths() {

3794

// For every instruction `I` in MinBWs, truncate the operands, create a

3795

// truncated version of `I` and reextend its result. InstCombine runs

3796

// later and will remove any ext/trunc pairs.

3797

3798

SmallPtrSet<Value *, 4> Erased;

3799

for (const auto &KV : Cost->getMinimalBitwidths()) {

3800

// If the value wasn't vectorized, we must maintain the original scalar

3801

// type. The absence of the value from VectorLoopValueMap indicates that it

3802

// wasn't vectorized.

3803

if (!VectorLoopValueMap.hasAnyVectorValue(KV.first))

3804

continue;

3805

for (unsigned Part = 0; Part < UF; ++Part) {

3806

Value *I = getOrCreateVectorValue(KV.first, Part);

3807

if (Erased.count(I) || I->use_empty() || !isa<Instruction>(I))

3808

continue;

3809

Type *OriginalTy = I->getType();

3810

Type *ScalarTruncatedTy =

3811

IntegerType::get(OriginalTy->getContext(), KV.second);

3812

Type *TruncatedTy = VectorType::get(ScalarTruncatedTy,

3813

OriginalTy->getVectorNumElements());

3814

if (TruncatedTy == OriginalTy)

3815

continue;

3816

3817

IRBuilder<> B(cast<Instruction>(I));

3818

auto ShrinkOperand = [&](Value *V) -> Value * {

3819

if (auto *ZI = dyn_cast<ZExtInst>(V))

3820

if (ZI->getSrcTy() == TruncatedTy)

3821

return ZI->getOperand(0);

3822

return B.CreateZExtOrTrunc(V, TruncatedTy);

3823

};

3824

3825

// The actual instruction modification depends on the instruction type,

3826

// unfortunately.

3827

Value *NewI = nullptr;

3828

if (auto *BO = dyn_cast<BinaryOperator>(I)) {

3829

NewI = B.CreateBinOp(BO->getOpcode(), ShrinkOperand(BO->getOperand(0)),

3830

ShrinkOperand(BO->getOperand(1)));

3831

3832

// Any wrapping introduced by shrinking this operation shouldn't be

3833

// considered undefined behavior. So, we can't unconditionally copy

3834

// arithmetic wrapping flags to NewI.

3835

cast<BinaryOperator>(NewI)->copyIRFlags(I, /*IncludeWrapFlags=*/false);

3836

} else if (auto *CI = dyn_cast<ICmpInst>(I)) {

3837

NewI =

3838

B.CreateICmp(CI->getPredicate(), ShrinkOperand(CI->getOperand(0)),

3839

ShrinkOperand(CI->getOperand(1)));

3840

} else if (auto *SI = dyn_cast<SelectInst>(I)) {

3841

NewI = B.CreateSelect(SI->getCondition(),

3842

ShrinkOperand(SI->getTrueValue()),

3843

ShrinkOperand(SI->getFalseValue()));

3844

} else if (auto *CI = dyn_cast<CastInst>(I)) {

3845

switch (CI->getOpcode()) {

3846

default:

3847

llvm_unreachable("Unhandled cast!")::llvm::llvm_unreachable_internal("Unhandled cast!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3847);

3848

case Instruction::Trunc:

3849

NewI = ShrinkOperand(CI->getOperand(0));

3850

break;

3851

case Instruction::SExt:

3852

NewI = B.CreateSExtOrTrunc(

3853

CI->getOperand(0),

3854

smallestIntegerVectorType(OriginalTy, TruncatedTy));

3855

break;

3856

case Instruction::ZExt:

3857

NewI = B.CreateZExtOrTrunc(

3858

CI->getOperand(0),

3859

smallestIntegerVectorType(OriginalTy, TruncatedTy));

3860

break;

3861

}

3862

} else if (auto *SI = dyn_cast<ShuffleVectorInst>(I)) {

3863

auto Elements0 = SI->getOperand(0)->getType()->getVectorNumElements();

3864

auto *O0 = B.CreateZExtOrTrunc(

3865

SI->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements0));

3866

auto Elements1 = SI->getOperand(1)->getType()->getVectorNumElements();

3867

auto *O1 = B.CreateZExtOrTrunc(

3868

SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));

3869

3870

NewI = B.CreateShuffleVector(O0, O1, SI->getMask());

3871

} else if (isa<LoadInst>(I)) {

3872

// Don't do anything with the operands, just extend the result.

3873

continue;

3874

} else if (auto *IE = dyn_cast<InsertElementInst>(I)) {

3875

auto Elements = IE->getOperand(0)->getType()->getVectorNumElements();

3876

auto *O0 = B.CreateZExtOrTrunc(

3877

IE->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements));

3878

auto *O1 = B.CreateZExtOrTrunc(IE->getOperand(1), ScalarTruncatedTy);

3879

NewI = B.CreateInsertElement(O0, O1, IE->getOperand(2));

3880

} else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {

3881

auto Elements = EE->getOperand(0)->getType()->getVectorNumElements();

3882

auto *O0 = B.CreateZExtOrTrunc(

3883

EE->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements));

3884

NewI = B.CreateExtractElement(O0, EE->getOperand(2));

3885

} else {

3886

llvm_unreachable("Unhandled instruction type!")::llvm::llvm_unreachable_internal("Unhandled instruction type!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3886);

3887

}

3888

3889

// Lastly, extend the result.

3890

NewI->takeName(cast<Instruction>(I));

3891

Value *Res = B.CreateZExtOrTrunc(NewI, OriginalTy);

3892

I->replaceAllUsesWith(Res);

3893

cast<Instruction>(I)->eraseFromParent();

3894

Erased.insert(I);

3895

VectorLoopValueMap.resetVectorValue(KV.first, Part, Res);

3896

}

3897

}

3898

3899

// We'll have created a bunch of ZExts that are now parentless. Clean up.

3900

for (const auto &KV : Cost->getMinimalBitwidths()) {

3901

// If the value wasn't vectorized, we must maintain the original scalar

3902

// type. The absence of the value from VectorLoopValueMap indicates that it

3903

// wasn't vectorized.

3904

if (!VectorLoopValueMap.hasAnyVectorValue(KV.first))

3905

continue;

3906

for (unsigned Part = 0; Part < UF; ++Part) {

3907

Value *I = getOrCreateVectorValue(KV.first, Part);

3908

ZExtInst *Inst = dyn_cast<ZExtInst>(I);

3909

if (Inst && Inst->use_empty()) {

3910

Value *NewI = Inst->getOperand(0);

3911

Inst->eraseFromParent();

3912

VectorLoopValueMap.resetVectorValue(KV.first, Part, NewI);

3913

}

3914

}

3915

}

3916

}

3917

3918

void InnerLoopVectorizer::fixVectorizedLoop() {

3919

// Insert truncates and extends for any truncated instructions as hints to

3920

// InstCombine.

3921

if (VF > 1)

3922

truncateToMinimalBitwidths();

3923

3924

// At this point every instruction in the original loop is widened to a

3925

// vector form. Now we need to fix the recurrences in the loop. These PHI

3926

// nodes are currently empty because we did not want to introduce cycles.

3927

// This is the second stage of vectorizing recurrences.

3928

fixCrossIterationPHIs();

3929

3930

// Update the dominator tree.

3931

3932

// FIXME: After creating the structure of the new loop, the dominator tree is

3933

// no longer up-to-date, and it remains that way until we update it

3934

// here. An out-of-date dominator tree is problematic for SCEV,

3935

// because SCEVExpander uses it to guide code generation. The

3936

// vectorizer use SCEVExpanders in several places. Instead, we should

3937

// keep the dominator tree up-to-date as we go.

3938

updateAnalysis();

3939

3940

// Fix-up external users of the induction variables.

3941

for (auto &Entry : *Legal->getInductionVars())

3942

fixupIVUsers(Entry.first, Entry.second,

3943

getOrCreateVectorTripCount(LI->getLoopFor(LoopVectorBody)),

3944

IVEndValues[Entry.first], LoopMiddleBlock);

3945

3946

fixLCSSAPHIs();

3947

predicateInstructions();

3948

3949

// Remove redundant induction instructions.

3950

cse(LoopVectorBody);

3951

}

3952

3953

void InnerLoopVectorizer::fixCrossIterationPHIs() {

3954

// In order to support recurrences we need to be able to vectorize Phi nodes.

3955

// Phi nodes have cycles, so we need to vectorize them in two stages. This is

3956

// stage #2: We now need to fix the recurrences by adding incoming edges to

3957

// the currently empty PHI nodes. At this point every instruction in the

3958

// original loop is widened to a vector form so we can use them to construct

3959

// the incoming edges.

3960

for (Instruction &I : *OrigLoop->getHeader()) {

3961

PHINode *Phi = dyn_cast<PHINode>(&I);

3962

if (!Phi)

3963

break;

3964

// Handle first-order recurrences and reductions that need to be fixed.

3965

if (Legal->isFirstOrderRecurrence(Phi))

3966

fixFirstOrderRecurrence(Phi);

3967

else if (Legal->isReductionVariable(Phi))

3968

fixReduction(Phi);

3969

}

3970

}

3971

3972

void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi) {

3973

3974

// This is the second phase of vectorizing first-order recurrences. An

3975

// overview of the transformation is described below. Suppose we have the

3976

// following loop.

3977

3978

// for (int i = 0; i < n; ++i)

3979

// b[i] = a[i] - a[i - 1];

3980

3981

// There is a first-order recurrence on "a". For this loop, the shorthand

3982

// scalar IR looks like:

3983

3984

// scalar.ph:

3985

// s_init = a[-1]

3986

// br scalar.body

3987

3988

// scalar.body:

3989

// i = phi [0, scalar.ph], [i+1, scalar.body]

3990

// s1 = phi [s_init, scalar.ph], [s2, scalar.body]

3991

// s2 = a[i]

3992

// b[i] = s2 - s1

3993

// br cond, scalar.body, ...

3994

3995

// In this example, s1 is a recurrence because it's value depends on the

3996

// previous iteration. In the first phase of vectorization, we created a

3997

// temporary value for s1. We now complete the vectorization and produce the

3998

// shorthand vector IR shown below (for VF = 4, UF = 1).

3999

4000

// vector.ph:

4001

// v_init = vector(..., ..., ..., a[-1])

4002

// br vector.body

4003

4004

// vector.body

4005

// i = phi [0, vector.ph], [i+4, vector.body]

4006

// v1 = phi [v_init, vector.ph], [v2, vector.body]

4007

// v2 = a[i, i+1, i+2, i+3];

4008

// v3 = vector(v1(3), v2(0, 1, 2))

4009

// b[i, i+1, i+2, i+3] = v2 - v3

4010

// br cond, vector.body, middle.block

4011

4012

// middle.block:

4013

// x = v2(3)

4014

// br scalar.ph

4015

4016

// scalar.ph:

4017

// s_init = phi [x, middle.block], [a[-1], otherwise]

4018

// br scalar.body

4019

4020

// After execution completes the vector loop, we extract the next value of

4021

// the recurrence (x) to use as the initial value in the scalar loop.

4022

4023

// Get the original loop preheader and single loop latch.

4024

auto *Preheader = OrigLoop->getLoopPreheader();

4025

auto *Latch = OrigLoop->getLoopLatch();

4026

4027

// Get the initial and previous values of the scalar recurrence.

4028

auto *ScalarInit = Phi->getIncomingValueForBlock(Preheader);

4029

auto *Previous = Phi->getIncomingValueForBlock(Latch);

4030

4031

// Create a vector from the initial value.

4032

auto *VectorInit = ScalarInit;

4033

if (VF > 1) {

4034

Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());

4035

VectorInit = Builder.CreateInsertElement(

4036

UndefValue::get(VectorType::get(VectorInit->getType(), VF)), VectorInit,

4037

Builder.getInt32(VF - 1), "vector.recur.init");

4038

}

4039

4040

// We constructed a temporary phi node in the first phase of vectorization.

4041

// This phi node will eventually be deleted.

4042

Builder.SetInsertPoint(

4043

cast<Instruction>(VectorLoopValueMap.getVectorValue(Phi, 0)));

4044

4045

// Create a phi node for the new recurrence. The current value will either be

4046

// the initial value inserted into a vector or loop-varying vector value.

4047

auto *VecPhi = Builder.CreatePHI(VectorInit->getType(), 2, "vector.recur");

4048

VecPhi->addIncoming(VectorInit, LoopVectorPreHeader);

4049

4050

// Get the vectorized previous value.

4051

Value *PreviousLastPart = getOrCreateVectorValue(Previous, UF - 1);

4052

4053

// Set the insertion point after the previous value if it is an instruction.

4054

// Note that the previous value may have been constant-folded so it is not

4055

// guaranteed to be an instruction in the vector loop. Also, if the previous

4056

// value is a phi node, we should insert after all the phi nodes to avoid

4057

// breaking basic block verification.

4058

if (LI->getLoopFor(LoopVectorBody)->isLoopInvariant(PreviousLastPart) ||

4059

isa<PHINode>(PreviousLastPart))

4060

Builder.SetInsertPoint(&*LoopVectorBody->getFirstInsertionPt());

4061

else

4062

Builder.SetInsertPoint(

4063

&*++BasicBlock::iterator(cast<Instruction>(PreviousLastPart)));

4064

4065

// We will construct a vector for the recurrence by combining the values for

4066

// the current and previous iterations. This is the required shuffle mask.

4067

SmallVector<Constant *, 8> ShuffleMask(VF);

4068

ShuffleMask[0] = Builder.getInt32(VF - 1);

4069

for (unsigned I = 1; I < VF; ++I)

4070

ShuffleMask[I] = Builder.getInt32(I + VF - 1);

4071

4072

// The vector from which to take the initial value for the current iteration

4073

// (actual or unrolled). Initially, this is the vector phi node.

4074

Value *Incoming = VecPhi;

4075

4076

// Shuffle the current and previous vector and update the vector parts.

4077

for (unsigned Part = 0; Part < UF; ++Part) {

4078

Value *PreviousPart = getOrCreateVectorValue(Previous, Part);

4079

Value *PhiPart = VectorLoopValueMap.getVectorValue(Phi, Part);

4080

auto *Shuffle =

4081

VF > 1 ? Builder.CreateShuffleVector(Incoming, PreviousPart,

4082

ConstantVector::get(ShuffleMask))

4083

: Incoming;

4084

PhiPart->replaceAllUsesWith(Shuffle);

4085

cast<Instruction>(PhiPart)->eraseFromParent();

4086

VectorLoopValueMap.resetVectorValue(Phi, Part, Shuffle);

4087

Incoming = PreviousPart;

4088

}

4089

4090

// Fix the latch value of the new recurrence in the vector loop.

4091

VecPhi->addIncoming(Incoming, LI->getLoopFor(LoopVectorBody)->getLoopLatch());

4092

4093

// Extract the last vector element in the middle block. This will be the

4094

// initial value for the recurrence when jumping to the scalar loop.

4095

auto *ExtractForScalar = Incoming;

4096

if (VF > 1) {

4097

Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());

4098

ExtractForScalar = Builder.CreateExtractElement(

4099

ExtractForScalar, Builder.getInt32(VF - 1), "vector.recur.extract");

4100

}

4101

// Extract the second last element in the middle block if the

4102

// Phi is used outside the loop. We need to extract the phi itself

4103

// and not the last element (the phi update in the current iteration). This

4104

// will be the value when jumping to the exit block from the LoopMiddleBlock,

4105

// when the scalar loop is not run at all.

4106

Value *ExtractForPhiUsedOutsideLoop = nullptr;

4107

if (VF > 1)

4108

ExtractForPhiUsedOutsideLoop = Builder.CreateExtractElement(

4109

Incoming, Builder.getInt32(VF - 2), "vector.recur.extract.for.phi");

4110

// When loop is unrolled without vectorizing, initialize

4111

// ExtractForPhiUsedOutsideLoop with the value just prior to unrolled value of

4112

// `Incoming`. This is analogous to the vectorized case above: extracting the

4113

// second last element when VF > 1.

4114

else if (UF > 1)

4115

ExtractForPhiUsedOutsideLoop = getOrCreateVectorValue(Previous, UF - 2);

4116

4117

// Fix the initial value of the original recurrence in the scalar loop.

4118

Builder.SetInsertPoint(&*LoopScalarPreHeader->begin());

4119

auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init");

4120

for (auto *BB : predecessors(LoopScalarPreHeader)) {

4121

auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit;

4122

Start->addIncoming(Incoming, BB);

4123

}

4124

4125

Phi->setIncomingValue(Phi->getBasicBlockIndex(LoopScalarPreHeader), Start);

4126

Phi->setName("scalar.recur");

4127

4128

// Finally, fix users of the recurrence outside the loop. The users will need

4129

// either the last value of the scalar recurrence or the last value of the

4130

// vector recurrence we extracted in the middle block. Since the loop is in

4131

// LCSSA form, we just need to find the phi node for the original scalar

4132

// recurrence in the exit block, and then add an edge for the middle block.

4133

for (auto &I : *LoopExitBlock) {

4134

auto *LCSSAPhi = dyn_cast<PHINode>(&I);

4135

if (!LCSSAPhi)

4136

break;

4137

if (LCSSAPhi->getIncomingValue(0) == Phi) {

4138

LCSSAPhi->addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);

4139

break;

4140

}

4141

}

4142

}

4143

4144

void InnerLoopVectorizer::fixReduction(PHINode *Phi) {

4145

Constant *Zero = Builder.getInt32(0);

4146

4147

// Get it's reduction variable descriptor.

4148

assert(Legal->isReductionVariable(Phi) &&((Legal->isReductionVariable(Phi) && "Unable to find the reduction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->isReductionVariable(Phi) && \"Unable to find the reduction variable\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4149, __PRETTY_FUNCTION__))

4149

"Unable to find the reduction variable")((Legal->isReductionVariable(Phi) && "Unable to find the reduction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->isReductionVariable(Phi) && \"Unable to find the reduction variable\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4149, __PRETTY_FUNCTION__));

4150

RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[Phi];

4151

4152

RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();

4153

TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();

4154

Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();

4155

RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind =

4156

RdxDesc.getMinMaxRecurrenceKind();

4157

setDebugLocFromInst(Builder, ReductionStartValue);

4158

4159

// We need to generate a reduction vector from the incoming scalar.

4160

// To do so, we need to generate the 'identity' vector and override

4161

// one of the elements with the incoming scalar reduction. We need

4162

// to do it in the vector-loop preheader.

4163

Builder.SetInsertPoint(LoopBypassBlocks[1]->getTerminator());

4164

4165

// This is the vector-clone of the value that leaves the loop.

4166

Type *VecTy = getOrCreateVectorValue(LoopExitInst, 0)->getType();

4167

4168

// Find the reduction identity variable. Zero for addition, or, xor,

4169

// one for multiplication, -1 for And.

4170

Value *Identity;

4171

Value *VectorStart;

4172

if (RK == RecurrenceDescriptor::RK_IntegerMinMax ||

4173

RK == RecurrenceDescriptor::RK_FloatMinMax) {

4174

// MinMax reduction have the start value as their identify.

4175

if (VF == 1) {

4176

VectorStart = Identity = ReductionStartValue;

4177

} else {

4178

VectorStart = Identity =

4179

Builder.CreateVectorSplat(VF, ReductionStartValue, "minmax.ident");

4180

}

4181

} else {

4182

// Handle other reduction kinds:

4183

Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(

4184

RK, VecTy->getScalarType());

4185

if (VF == 1) {

4186

Identity = Iden;

4187

// This vector is the Identity vector where the first element is the

4188

// incoming scalar reduction.

4189

VectorStart = ReductionStartValue;

4190

} else {

4191

Identity = ConstantVector::getSplat(VF, Iden);

4192

4193

// This vector is the Identity vector where the first element is the

4194

// incoming scalar reduction.

4195

VectorStart =

4196

Builder.CreateInsertElement(Identity, ReductionStartValue, Zero);

4197

}

4198

}

4199

4200

// Fix the vector-loop phi.

4201

4202

// Reductions do not have to start at zero. They can start with

4203

// any loop invariant values.

4204

BasicBlock *Latch = OrigLoop->getLoopLatch();

4205

Value *LoopVal = Phi->getIncomingValueForBlock(Latch);

4206

for (unsigned Part = 0; Part < UF; ++Part) {

4207

Value *VecRdxPhi = getOrCreateVectorValue(Phi, Part);

4208

Value *Val = getOrCreateVectorValue(LoopVal, Part);

4209

// Make sure to add the reduction stat value only to the

4210

// first unroll part.

4211

Value *StartVal = (Part == 0) ? VectorStart : Identity;

4212

cast<PHINode>(VecRdxPhi)->addIncoming(StartVal, LoopVectorPreHeader);

4213

cast<PHINode>(VecRdxPhi)

4214

->addIncoming(Val, LI->getLoopFor(LoopVectorBody)->getLoopLatch());

4215

}

4216

4217

// Before each round, move the insertion point right between

4218

// the PHIs and the values we are going to write.

4219

// This allows us to write both PHINodes and the extractelement

4220

// instructions.

4221

Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());

4222

4223

setDebugLocFromInst(Builder, LoopExitInst);

4224

4225

// If the vector reduction can be performed in a smaller type, we truncate

4226

// then extend the loop exit value to enable InstCombine to evaluate the

4227

// entire expression in the smaller type.

4228

if (VF > 1 && Phi->getType() != RdxDesc.getRecurrenceType()) {

4229

Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);

4230

Builder.SetInsertPoint(LoopVectorBody->getTerminator());

4231

VectorParts RdxParts(UF);

4232

for (unsigned Part = 0; Part < UF; ++Part) {

4233

RdxParts[Part] = VectorLoopValueMap.getVectorValue(LoopExitInst, Part);

4234

Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);

4235

Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy)

4236

: Builder.CreateZExt(Trunc, VecTy);

4237

for (Value::user_iterator UI = RdxParts[Part]->user_begin();

4238

UI != RdxParts[Part]->user_end();)

4239

if (*UI != Trunc) {

4240

(*UI++)->replaceUsesOfWith(RdxParts[Part], Extnd);

4241

RdxParts[Part] = Extnd;

4242

} else {

4243

++UI;

4244

}

4245

}

4246

Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());

4247

for (unsigned Part = 0; Part < UF; ++Part) {

4248

RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);

4249

VectorLoopValueMap.resetVectorValue(LoopExitInst, Part, RdxParts[Part]);

4250

}

4251

}

4252

4253

// Reduce all of the unrolled parts into a single vector.

4254

Value *ReducedPartRdx = VectorLoopValueMap.getVectorValue(LoopExitInst, 0);

4255

unsigned Op = RecurrenceDescriptor::getRecurrenceBinOp(RK);

4256

setDebugLocFromInst(Builder, ReducedPartRdx);

4257

for (unsigned Part = 1; Part < UF; ++Part) {

4258

Value *RdxPart = VectorLoopValueMap.getVectorValue(LoopExitInst, Part);

4259

if (Op != Instruction::ICmp && Op != Instruction::FCmp)

4260

// Floating point operations had to be 'fast' to enable the reduction.

4261

ReducedPartRdx = addFastMathFlag(

4262

Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxPart,

4263

ReducedPartRdx, "bin.rdx"));

4264

else

4265

ReducedPartRdx = RecurrenceDescriptor::createMinMaxOp(

4266

Builder, MinMaxKind, ReducedPartRdx, RdxPart);

4267

}

4268

4269

if (VF > 1) {

4270

bool NoNaN = Legal->hasFunNoNaNAttr();

4271

ReducedPartRdx =

4272

createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx, NoNaN);

4273

// If the reduction can be performed in a smaller type, we need to extend

4274

// the reduction to the wider type before we branch to the original loop.

4275

if (Phi->getType() != RdxDesc.getRecurrenceType())

4276

ReducedPartRdx =

4277

RdxDesc.isSigned()

4278

? Builder.CreateSExt(ReducedPartRdx, Phi->getType())

4279

: Builder.CreateZExt(ReducedPartRdx, Phi->getType());

4280

}

4281

4282

// Create a phi node that merges control-flow from the backedge-taken check

4283

// block and the middle block.

4284

PHINode *BCBlockPhi = PHINode::Create(Phi->getType(), 2, "bc.merge.rdx",

4285

LoopScalarPreHeader->getTerminator());

4286

for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I)

4287

BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[I]);

4288

BCBlockPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);

4289

4290

// Now, we need to fix the users of the reduction variable

4291

// inside and outside of the scalar remainder loop.

4292

// We know that the loop is in LCSSA form. We need to update the

4293

// PHI nodes in the exit blocks.

4294

for (BasicBlock::iterator LEI = LoopExitBlock->begin(),

4295

LEE = LoopExitBlock->end();

4296

LEI != LEE; ++LEI) {

4297

PHINode *LCSSAPhi = dyn_cast<PHINode>(LEI);

4298

if (!LCSSAPhi)

4299

break;

4300

4301

// All PHINodes need to have a single entry edge, or two if

4302

// we already fixed them.

4303

assert(LCSSAPhi->getNumIncomingValues() < 3 && "Invalid LCSSA PHI")((LCSSAPhi->getNumIncomingValues() < 3 && "Invalid LCSSA PHI"
) ? static_cast<void> (0) : __assert_fail ("LCSSAPhi->getNumIncomingValues() < 3 && \"Invalid LCSSA PHI\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4303, __PRETTY_FUNCTION__));

4304

4305

// We found a reduction value exit-PHI. Update it with the

4306

// incoming bypass edge.

4307

if (LCSSAPhi->getIncomingValue(0) == LoopExitInst)

4308

LCSSAPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);

4309

} // end of the LCSSA phi scan.

4310

4311

// Fix the scalar loop reduction variable with the incoming reduction sum

4312

// from the vector body and from the backedge value.

4313

int IncomingEdgeBlockIdx =

4314

Phi->getBasicBlockIndex(OrigLoop->getLoopLatch());

4315

assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index")((IncomingEdgeBlockIdx >= 0 && "Invalid block index"
) ? static_cast<void> (0) : __assert_fail ("IncomingEdgeBlockIdx >= 0 && \"Invalid block index\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4315, __PRETTY_FUNCTION__));

4316

// Pick the other block.

4317

int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);

4318

Phi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);

4319

Phi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);

4320

}

4321

4322

void InnerLoopVectorizer::fixLCSSAPHIs() {

4323

for (Instruction &LEI : *LoopExitBlock) {

4324

auto *LCSSAPhi = dyn_cast<PHINode>(&LEI);

4325

if (!LCSSAPhi)

4326

break;

4327

if (LCSSAPhi->getNumIncomingValues() == 1) {

4328

assert(OrigLoop->isLoopInvariant(LCSSAPhi->getIncomingValue(0)) &&((OrigLoop->isLoopInvariant(LCSSAPhi->getIncomingValue(
0)) && "Incoming value isn't loop invariant") ? static_cast
<void> (0) : __assert_fail ("OrigLoop->isLoopInvariant(LCSSAPhi->getIncomingValue(0)) && \"Incoming value isn't loop invariant\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4329, __PRETTY_FUNCTION__))

4329

"Incoming value isn't loop invariant")((OrigLoop->isLoopInvariant(LCSSAPhi->getIncomingValue(
0)) && "Incoming value isn't loop invariant") ? static_cast
<void> (0) : __assert_fail ("OrigLoop->isLoopInvariant(LCSSAPhi->getIncomingValue(0)) && \"Incoming value isn't loop invariant\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4329, __PRETTY_FUNCTION__));

4330

LCSSAPhi->addIncoming(LCSSAPhi->getIncomingValue(0), LoopMiddleBlock);

4331

}

4332

}

4333

}

4334

4335

void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {

4336

4337

// The basic block and loop containing the predicated instruction.

4338

auto *PredBB = PredInst->getParent();

4339

auto *VectorLoop = LI->getLoopFor(PredBB);

4340

4341

// Initialize a worklist with the operands of the predicated instruction.

4342

SetVector<Value *> Worklist(PredInst->op_begin(), PredInst->op_end());

4343

4344

// Holds instructions that we need to analyze again. An instruction may be

4345

// reanalyzed if we don't yet know if we can sink it or not.

4346

SmallVector<Instruction *, 8> InstsToReanalyze;

4347

4348

// Returns true if a given use occurs in the predicated block. Phi nodes use

4349

// their operands in their corresponding predecessor blocks.

4350

auto isBlockOfUsePredicated = [&](Use &U) -> bool {

4351

auto *I = cast<Instruction>(U.getUser());

4352

BasicBlock *BB = I->getParent();

4353

if (auto *Phi = dyn_cast<PHINode>(I))

4354

BB = Phi->getIncomingBlock(

4355

PHINode::getIncomingValueNumForOperand(U.getOperandNo()));

4356

return BB == PredBB;

4357

};

4358

4359

// Iteratively sink the scalarized operands of the predicated instruction

4360

// into the block we created for it. When an instruction is sunk, it's

4361

// operands are then added to the worklist. The algorithm ends after one pass

4362

// through the worklist doesn't sink a single instruction.

4363

bool Changed;

4364

do {

4365

4366

// Add the instructions that need to be reanalyzed to the worklist, and

4367

// reset the changed indicator.

4368

Worklist.insert(InstsToReanalyze.begin(), InstsToReanalyze.end());

4369

InstsToReanalyze.clear();

4370

Changed = false;

4371

4372

while (!Worklist.empty()) {

4373

auto *I = dyn_cast<Instruction>(Worklist.pop_back_val());

4374

4375

// We can't sink an instruction if it is a phi node, is already in the

4376

// predicated block, is not in the loop, or may have side effects.

4377

if (!I || isa<PHINode>(I) || I->getParent() == PredBB ||

4378

!VectorLoop->contains(I) || I->mayHaveSideEffects())

4379

continue;

4380

4381

// It's legal to sink the instruction if all its uses occur in the

4382

// predicated block. Otherwise, there's nothing to do yet, and we may

4383

// need to reanalyze the instruction.

4384

if (!all_of(I->uses(), isBlockOfUsePredicated)) {

4385

InstsToReanalyze.push_back(I);

4386

continue;

4387

}

4388

4389

// Move the instruction to the beginning of the predicated block, and add

4390

// it's operands to the worklist.

4391

I->moveBefore(&*PredBB->getFirstInsertionPt());

4392

Worklist.insert(I->op_begin(), I->op_end());

4393

4394

// The sinking may have enabled other instructions to be sunk, so we will

4395

// need to iterate.

4396

Changed = true;

4397

}

4398

} while (Changed);

4399

}

4400

4401

void InnerLoopVectorizer::predicateInstructions() {

4402

4403

// For each instruction I marked for predication on value C, split I into its

4404

// own basic block to form an if-then construct over C. Since I may be fed by

4405

// an extractelement instruction or other scalar operand, we try to

4406

// iteratively sink its scalar operands into the predicated block. If I feeds

4407

// an insertelement instruction, we try to move this instruction into the

4408

// predicated block as well. For non-void types, a phi node will be created

4409

// for the resulting value (either vector or scalar).

4410

4411

// So for some predicated instruction, e.g. the conditional sdiv in:

4412

4413

// for.body:

4414

// ...

4415

// %add = add nsw i32 %mul, %0

4416

// %cmp5 = icmp sgt i32 %2, 7

4417

// br i1 %cmp5, label %if.then, label %if.end

4418

4419

// if.then:

4420

// %div = sdiv i32 %0, %1

4421

// br label %if.end

4422

4423

// if.end:

4424

// %x.0 = phi i32 [ %div, %if.then ], [ %add, %for.body ]

4425

4426

// the sdiv at this point is scalarized and if-converted using a select.

4427

// The inactive elements in the vector are not used, but the predicated

4428

// instruction is still executed for all vector elements, essentially:

4429

4430

// vector.body:

4431

// ...

4432

// %17 = add nsw <2 x i32> %16, %wide.load

4433

// %29 = extractelement <2 x i32> %wide.load, i32 0

4434

// %30 = extractelement <2 x i32> %wide.load51, i32 0

4435

// %31 = sdiv i32 %29, %30

4436

// %32 = insertelement <2 x i32> undef, i32 %31, i32 0

4437

// %35 = extractelement <2 x i32> %wide.load, i32 1

4438

// %36 = extractelement <2 x i32> %wide.load51, i32 1

4439

// %37 = sdiv i32 %35, %36

4440

// %38 = insertelement <2 x i32> %32, i32 %37, i32 1

4441

// %predphi = select <2 x i1> %26, <2 x i32> %38, <2 x i32> %17

4442

4443

// Predication will now re-introduce the original control flow to avoid false

4444

// side-effects by the sdiv instructions on the inactive elements, yielding

4445

// (after cleanup):

4446

4447

// vector.body:

4448

// ...

4449

// %5 = add nsw <2 x i32> %4, %wide.load

4450

// %8 = icmp sgt <2 x i32> %wide.load52, <i32 7, i32 7>

4451

// %9 = extractelement <2 x i1> %8, i32 0

4452

// br i1 %9, label %pred.sdiv.if, label %pred.sdiv.continue

4453

4454

// pred.sdiv.if:

4455

// %10 = extractelement <2 x i32> %wide.load, i32 0

4456

// %11 = extractelement <2 x i32> %wide.load51, i32 0

4457

// %12 = sdiv i32 %10, %11

4458

// %13 = insertelement <2 x i32> undef, i32 %12, i32 0

4459

// br label %pred.sdiv.continue

4460

4461

// pred.sdiv.continue:

4462

// %14 = phi <2 x i32> [ undef, %vector.body ], [ %13, %pred.sdiv.if ]

4463

// %15 = extractelement <2 x i1> %8, i32 1

4464

// br i1 %15, label %pred.sdiv.if54, label %pred.sdiv.continue55

4465

4466

// pred.sdiv.if54:

4467

// %16 = extractelement <2 x i32> %wide.load, i32 1

4468

// %17 = extractelement <2 x i32> %wide.load51, i32 1

4469

// %18 = sdiv i32 %16, %17

4470

// %19 = insertelement <2 x i32> %14, i32 %18, i32 1

4471

// br label %pred.sdiv.continue55

4472

4473

// pred.sdiv.continue55:

4474

// %20 = phi <2 x i32> [ %14, %pred.sdiv.continue ], [ %19, %pred.sdiv.if54 ]

4475

// %predphi = select <2 x i1> %8, <2 x i32> %20, <2 x i32> %5

4476

4477

for (auto KV : PredicatedInstructions) {

4478

BasicBlock::iterator I(KV.first);

4479

BasicBlock *Head = I->getParent();

4480

auto *T = SplitBlockAndInsertIfThen(KV.second, &*I, /*Unreachable=*/false,

4481

/*BranchWeights=*/nullptr, DT, LI);

4482

I->moveBefore(T);

4483

sinkScalarOperands(&*I);

4484

4485

BasicBlock *PredicatedBlock = I->getParent();

4486

Twine BBNamePrefix = Twine("pred.") + I->getOpcodeName();

4487

PredicatedBlock->setName(BBNamePrefix + ".if");

4488

PredicatedBlock->getSingleSuccessor()->setName(BBNamePrefix + ".continue");

4489

4490

// If the instruction is non-void create a Phi node at reconvergence point.

4491

if (!I->getType()->isVoidTy()) {

4492

Value *IncomingTrue = nullptr;

4493

Value *IncomingFalse = nullptr;

4494

4495

if (I->hasOneUse() && isa<InsertElementInst>(*I->user_begin())) {

4496

// If the predicated instruction is feeding an insert-element, move it

4497

// into the Then block; Phi node will be created for the vector.

4498

InsertElementInst *IEI = cast<InsertElementInst>(*I->user_begin());

4499

IEI->moveBefore(T);

4500

IncomingTrue = IEI; // the new vector with the inserted element.

4501

IncomingFalse = IEI->getOperand(0); // the unmodified vector

4502

} else {

4503

// Phi node will be created for the scalar predicated instruction.

4504

IncomingTrue = &*I;

4505

IncomingFalse = UndefValue::get(I->getType());

4506

}

4507

4508

BasicBlock *PostDom = I->getParent()->getSingleSuccessor();

4509

assert(PostDom && "Then block has multiple successors")((PostDom && "Then block has multiple successors") ? static_cast
<void> (0) : __assert_fail ("PostDom && \"Then block has multiple successors\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4509, __PRETTY_FUNCTION__));

4510

PHINode *Phi =

4511

PHINode::Create(IncomingTrue->getType(), 2, "", &PostDom->front());

4512

IncomingTrue->replaceAllUsesWith(Phi);

4513

Phi->addIncoming(IncomingFalse, Head);

4514

Phi->addIncoming(IncomingTrue, I->getParent());

4515

}

4516

}

4517

4518

DEBUG(DT->verifyDomTree())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { DT->verifyDomTree(); } } while (false
);

4519

}

4520

4521

InnerLoopVectorizer::VectorParts

4522

InnerLoopVectorizer::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) {

4523

assert(is_contained(predecessors(Dst), Src) && "Invalid edge")((is_contained(predecessors(Dst), Src) && "Invalid edge"
) ? static_cast<void> (0) : __assert_fail ("is_contained(predecessors(Dst), Src) && \"Invalid edge\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4523, __PRETTY_FUNCTION__));

4524

4525

// Look for cached value.

4526

std::pair<BasicBlock *, BasicBlock *> Edge(Src, Dst);

4527

EdgeMaskCacheTy::iterator ECEntryIt = EdgeMaskCache.find(Edge);

4528

if (ECEntryIt != EdgeMaskCache.end())

4529

return ECEntryIt->second;

4530

4531

VectorParts SrcMask = createBlockInMask(Src);

4532

4533

// The terminator has to be a branch inst!

4534

BranchInst *BI = dyn_cast<BranchInst>(Src->getTerminator());

4535

assert(BI && "Unexpected terminator found")((BI && "Unexpected terminator found") ? static_cast<
void> (0) : __assert_fail ("BI && \"Unexpected terminator found\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4535, __PRETTY_FUNCTION__));

4536

4537

if (BI->isConditional()) {

4538

4539

VectorParts EdgeMask(UF);

4540

for (unsigned Part = 0; Part < UF; ++Part) {

4541

auto *EdgeMaskPart = getOrCreateVectorValue(BI->getCondition(), Part);

4542

if (BI->getSuccessor(0) != Dst)

4543

EdgeMaskPart = Builder.CreateNot(EdgeMaskPart);

4544

4545

EdgeMaskPart = Builder.CreateAnd(EdgeMaskPart, SrcMask[Part]);

4546

EdgeMask[Part] = EdgeMaskPart;

4547

}

4548

4549

EdgeMaskCache[Edge] = EdgeMask;

4550

return EdgeMask;

4551

}

4552

4553

EdgeMaskCache[Edge] = SrcMask;

4554

return SrcMask;

4555

}

4556

4557

InnerLoopVectorizer::VectorParts

4558

InnerLoopVectorizer::createBlockInMask(BasicBlock *BB) {

4559

assert(OrigLoop->contains(BB) && "Block is not a part of a loop")((OrigLoop->contains(BB) && "Block is not a part of a loop"
) ? static_cast<void> (0) : __assert_fail ("OrigLoop->contains(BB) && \"Block is not a part of a loop\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4559, __PRETTY_FUNCTION__));

4560

4561

// Look for cached value.

4562

BlockMaskCacheTy::iterator BCEntryIt = BlockMaskCache.find(BB);

4563

if (BCEntryIt != BlockMaskCache.end())

4564

return BCEntryIt->second;

4565

4566

VectorParts BlockMask(UF);

4567

4568

// Loop incoming mask is all-one.

4569

if (OrigLoop->getHeader() == BB) {

4570

Value *C = ConstantInt::get(IntegerType::getInt1Ty(BB->getContext()), 1);

4571

for (unsigned Part = 0; Part < UF; ++Part)

4572

BlockMask[Part] = getOrCreateVectorValue(C, Part);

4573

BlockMaskCache[BB] = BlockMask;

4574

return BlockMask;

4575

}

4576

4577

// This is the block mask. We OR all incoming edges, and with zero.

4578

Value *Zero = ConstantInt::get(IntegerType::getInt1Ty(BB->getContext()), 0);

4579

for (unsigned Part = 0; Part < UF; ++Part)

4580

BlockMask[Part] = getOrCreateVectorValue(Zero, Part);

4581

4582

// For each pred:

4583

for (pred_iterator It = pred_begin(BB), E = pred_end(BB); It != E; ++It) {

4584

VectorParts EM = createEdgeMask(*It, BB);

4585

for (unsigned Part = 0; Part < UF; ++Part)

4586

BlockMask[Part] = Builder.CreateOr(BlockMask[Part], EM[Part]);

4587

}

4588

4589

BlockMaskCache[BB] = BlockMask;

4590

return BlockMask;

4591

}

4592

4593

void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN, unsigned UF,

4594

unsigned VF) {

4595

PHINode *P = cast<PHINode>(PN);

4596

// In order to support recurrences we need to be able to vectorize Phi nodes.

4597

// Phi nodes have cycles, so we need to vectorize them in two stages. This is

4598

// stage #1: We create a new vector PHI node with no incoming edges. We'll use

4599

// this value when we vectorize all of the instructions that use the PHI.

4600

if (Legal->isReductionVariable(P) || Legal->isFirstOrderRecurrence(P)) {

4601

for (unsigned Part = 0; Part < UF; ++Part) {

4602

// This is phase one of vectorizing PHIs.

4603

Type *VecTy =

4604

(VF == 1) ? PN->getType() : VectorType::get(PN->getType(), VF);

4605

Value *EntryPart = PHINode::Create(

4606

VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());

4607

VectorLoopValueMap.setVectorValue(P, Part, EntryPart);

4608

}

4609

return;

4610

}

4611

4612

setDebugLocFromInst(Builder, P);

4613

// Check for PHI nodes that are lowered to vector selects.

4614

if (P->getParent() != OrigLoop->getHeader()) {

4615

// We know that all PHIs in non-header blocks are converted into

4616

// selects, so we don't have to worry about the insertion order and we

4617

// can just use the builder.

4618

// At this point we generate the predication tree. There may be

4619

// duplications since this is a simple recursive scan, but future

4620

// optimizations will clean it up.

4621

4622

unsigned NumIncoming = P->getNumIncomingValues();

4623

4624

// Generate a sequence of selects of the form:

4625

// SELECT(Mask3, In3,

4626

// SELECT(Mask2, In2,

4627

// ( ...)))

4628

VectorParts Entry(UF);

4629

for (unsigned In = 0; In < NumIncoming; In++) {

4630

VectorParts Cond =

4631

createEdgeMask(P->getIncomingBlock(In), P->getParent());

4632

4633

for (unsigned Part = 0; Part < UF; ++Part) {

4634

Value *In0 = getOrCreateVectorValue(P->getIncomingValue(In), Part);

4635

// We might have single edge PHIs (blocks) - use an identity

4636

// 'select' for the first PHI operand.

4637

if (In == 0)

4638

Entry[Part] = Builder.CreateSelect(Cond[Part], In0, In0);

4639

else

4640

// Select between the current value and the previous incoming edge

4641

// based on the incoming mask.

4642

Entry[Part] = Builder.CreateSelect(Cond[Part], In0, Entry[Part],

4643

"predphi");

4644

}

4645

}

4646

for (unsigned Part = 0; Part < UF; ++Part)

4647

VectorLoopValueMap.setVectorValue(P, Part, Entry[Part]);

4648

return;

4649

}

4650

4651

// This PHINode must be an induction variable.

4652

// Make sure that we know about it.

4653

assert(Legal->getInductionVars()->count(P) && "Not an induction variable")((Legal->getInductionVars()->count(P) && "Not an induction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->getInductionVars()->count(P) && \"Not an induction variable\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4653, __PRETTY_FUNCTION__));

4654

4655

InductionDescriptor II = Legal->getInductionVars()->lookup(P);

4656

const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout();

4657

4658

// FIXME: The newly created binary instructions should contain nsw/nuw flags,

4659

// which can be found from the original scalar operations.

4660

switch (II.getKind()) {

4661

case InductionDescriptor::IK_NoInduction:

4662

llvm_unreachable("Unknown induction")::llvm::llvm_unreachable_internal("Unknown induction", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4662);

4663

case InductionDescriptor::IK_IntInduction:

4664

case InductionDescriptor::IK_FpInduction:

4665

return widenIntOrFpInduction(P);

4666

case InductionDescriptor::IK_PtrInduction: {

4667

// Handle the pointer induction variable case.

4668

assert(P->getType()->isPointerTy() && "Unexpected type.")((P->getType()->isPointerTy() && "Unexpected type."
) ? static_cast<void> (0) : __assert_fail ("P->getType()->isPointerTy() && \"Unexpected type.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4668, __PRETTY_FUNCTION__));

4669

// This is the normalized GEP that starts counting at zero.

4670

Value *PtrInd = Induction;

4671

PtrInd = Builder.CreateSExtOrTrunc(PtrInd, II.getStep()->getType());

4672

// Determine the number of scalars we need to generate for each unroll

4673

// iteration. If the instruction is uniform, we only need to generate the

4674

// first lane. Otherwise, we generate all VF values.

4675

unsigned Lanes = Cost->isUniformAfterVectorization(P, VF) ? 1 : VF;

4676

// These are the scalar results. Notice that we don't generate vector GEPs

4677

// because scalar GEPs result in better code.

4678

for (unsigned Part = 0; Part < UF; ++Part) {

4679

for (unsigned Lane = 0; Lane < Lanes; ++Lane) {

4680

Constant *Idx = ConstantInt::get(PtrInd->getType(), Lane + Part * VF);

4681

Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);

4682

Value *SclrGep = II.transform(Builder, GlobalIdx, PSE.getSE(), DL);

4683

SclrGep->setName("next.gep");

4684

VectorLoopValueMap.setScalarValue(P, Part, Lane, SclrGep);

4685

}

4686

}

4687

return;

4688

}

4689

}

4690

}

4691

4692

/// A helper function for checking whether an integer division-related

4693

/// instruction may divide by zero (in which case it must be predicated if

4694

/// executed conditionally in the scalar code).

4695

/// TODO: It may be worthwhile to generalize and check isKnownNonZero().

4696

/// Non-zero divisors that are non compile-time constants will not be

4697

/// converted into multiplication, so we will still end up scalarizing

4698

/// the division, but can do so w/o predication.

4699

static bool mayDivideByZero(Instruction &I) {

4700

assert((I.getOpcode() == Instruction::UDiv ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4704, __PRETTY_FUNCTION__))

4701

I.getOpcode() == Instruction::SDiv ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4704, __PRETTY_FUNCTION__))

4702

I.getOpcode() == Instruction::URem ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4704, __PRETTY_FUNCTION__))

4703

I.getOpcode() == Instruction::SRem) &&(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4704, __PRETTY_FUNCTION__))

4704

"Unexpected instruction")(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4704, __PRETTY_FUNCTION__));

4705

Value *Divisor = I.getOperand(1);

4706

auto *CInt = dyn_cast<ConstantInt>(Divisor);

4707

return !CInt || CInt->isZero();

4708

}

4709

4710

void InnerLoopVectorizer::vectorizeInstruction(Instruction &I) {

4711

// Scalarize instructions that should remain scalar after vectorization.

4712

if (VF > 1 &&

4713

!(isa<BranchInst>(&I) || isa<PHINode>(&I) || isa<DbgInfoIntrinsic>(&I)) &&

4714

shouldScalarizeInstruction(&I)) {

4715

scalarizeInstruction(&I, Legal->isScalarWithPredication(&I));

4716

return;

4717

}

4718

4719

switch (I.getOpcode()) {

4720

case Instruction::Br:

4721

// Nothing to do for PHIs and BR, since we already took care of the

4722

// loop control flow instructions.

4723

break;

4724

case Instruction::PHI: {

4725

// Vectorize PHINodes.

4726

widenPHIInstruction(&I, UF, VF);

4727

break;

4728

} // End of PHI.

4729

case Instruction::GetElementPtr: {

4730

// Construct a vector GEP by widening the operands of the scalar GEP as

4731

// necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP

4732

// results in a vector of pointers when at least one operand of the GEP

4733

// is vector-typed. Thus, to keep the representation compact, we only use

4734

// vector-typed operands for loop-varying values.

4735

auto *GEP = cast<GetElementPtrInst>(&I);

4736

4737

if (VF > 1 && OrigLoop->hasLoopInvariantOperands(GEP)) {

4738

// If we are vectorizing, but the GEP has only loop-invariant operands,

4739

// the GEP we build (by only using vector-typed operands for

4740

// loop-varying values) would be a scalar pointer. Thus, to ensure we

4741

// produce a vector of pointers, we need to either arbitrarily pick an

4742

// operand to broadcast, or broadcast a clone of the original GEP.

4743

// Here, we broadcast a clone of the original.

4744

4745

// TODO: If at some point we decide to scalarize instructions having

4746

// loop-invariant operands, this special case will no longer be

4747

// required. We would add the scalarization decision to

4748

// collectLoopScalars() and teach getVectorValue() to broadcast

4749

// the lane-zero scalar value.

4750

auto *Clone = Builder.Insert(GEP->clone());

4751

for (unsigned Part = 0; Part < UF; ++Part) {

4752

Value *EntryPart = Builder.CreateVectorSplat(VF, Clone);

4753

VectorLoopValueMap.setVectorValue(&I, Part, EntryPart);

4754

addMetadata(EntryPart, GEP);

4755

}

4756

} else {

4757

// If the GEP has at least one loop-varying operand, we are sure to

4758

// produce a vector of pointers. But if we are only unrolling, we want

4759

// to produce a scalar GEP for each unroll part. Thus, the GEP we

4760

// produce with the code below will be scalar (if VF == 1) or vector

4761

// (otherwise). Note that for the unroll-only case, we still maintain

4762

// values in the vector mapping with initVector, as we do for other

4763

// instructions.

4764

for (unsigned Part = 0; Part < UF; ++Part) {

4765

4766

// The pointer operand of the new GEP. If it's loop-invariant, we

4767

// won't broadcast it.

4768

auto *Ptr =

4769

OrigLoop->isLoopInvariant(GEP->getPointerOperand())

4770

? GEP->getPointerOperand()

4771

: getOrCreateVectorValue(GEP->getPointerOperand(), Part);

4772

4773

// Collect all the indices for the new GEP. If any index is

4774

// loop-invariant, we won't broadcast it.

4775

SmallVector<Value *, 4> Indices;

4776

for (auto &U : make_range(GEP->idx_begin(), GEP->idx_end())) {

4777

if (OrigLoop->isLoopInvariant(U.get()))

4778

Indices.push_back(U.get());

4779

else

4780

Indices.push_back(getOrCreateVectorValue(U.get(), Part));

4781

}

4782

4783

// Create the new GEP. Note that this GEP may be a scalar if VF == 1,

4784

// but it should be a vector, otherwise.

4785

auto *NewGEP = GEP->isInBounds()

4786

? Builder.CreateInBoundsGEP(Ptr, Indices)

4787

: Builder.CreateGEP(Ptr, Indices);

4788

assert((VF == 1 || NewGEP->getType()->isVectorTy()) &&(((VF == 1 || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector") ? static_cast<void> (
0) : __assert_fail ("(VF == 1 || NewGEP->getType()->isVectorTy()) && \"NewGEP is not a pointer vector\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4789, __PRETTY_FUNCTION__))

4789

"NewGEP is not a pointer vector")(((VF == 1 || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector") ? static_cast<void> (
0) : __assert_fail ("(VF == 1 || NewGEP->getType()->isVectorTy()) && \"NewGEP is not a pointer vector\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4789, __PRETTY_FUNCTION__));

4790

VectorLoopValueMap.setVectorValue(&I, Part, NewGEP);

4791

addMetadata(NewGEP, GEP);

4792

}

4793

}

4794

4795

break;

4796

}

4797

case Instruction::UDiv:

4798

case Instruction::SDiv:

4799

case Instruction::SRem:

4800

case Instruction::URem:

4801

// Scalarize with predication if this instruction may divide by zero and

4802

// block execution is conditional, otherwise fallthrough.

4803

if (Legal->isScalarWithPredication(&I)) {

4804

scalarizeInstruction(&I, true);

4805

break;

4806

}

4807

LLVM_FALLTHROUGH[[clang::fallthrough]];

4808

case Instruction::Add:

4809

case Instruction::FAdd:

4810

case Instruction::Sub:

4811

case Instruction::FSub:

4812

case Instruction::Mul:

4813

case Instruction::FMul:

4814

case Instruction::FDiv:

4815

case Instruction::FRem:

4816

case Instruction::Shl:

4817

case Instruction::LShr:

4818

case Instruction::AShr:

4819

case Instruction::And:

4820

case Instruction::Or:

4821

case Instruction::Xor: {

4822

// Just widen binops.

4823

auto *BinOp = cast<BinaryOperator>(&I);

4824

setDebugLocFromInst(Builder, BinOp);

4825

4826

for (unsigned Part = 0; Part < UF; ++Part) {

4827

Value *A = getOrCreateVectorValue(BinOp->getOperand(0), Part);

4828

Value *B = getOrCreateVectorValue(BinOp->getOperand(1), Part);

4829

Value *V = Builder.CreateBinOp(BinOp->getOpcode(), A, B);

4830

4831

if (BinaryOperator *VecOp = dyn_cast<BinaryOperator>(V))

4832

VecOp->copyIRFlags(BinOp);

4833

4834

// Use this vector value for all users of the original instruction.

4835

VectorLoopValueMap.setVectorValue(&I, Part, V);

4836

addMetadata(V, BinOp);

4837

}

4838

4839

break;

4840

}

4841

case Instruction::Select: {

4842

// Widen selects.

4843

// If the selector is loop invariant we can create a select

4844

// instruction with a scalar condition. Otherwise, use vector-select.

4845

auto *SE = PSE.getSE();

4846

bool InvariantCond =

4847

SE->isLoopInvariant(PSE.getSCEV(I.getOperand(0)), OrigLoop);

4848

setDebugLocFromInst(Builder, &I);

4849

4850

// The condition can be loop invariant but still defined inside the

4851

// loop. This means that we can't just use the original 'cond' value.

4852

// We have to take the 'vectorized' value and pick the first lane.

4853

// Instcombine will make this a no-op.

4854

4855

auto *ScalarCond = getOrCreateScalarValue(I.getOperand(0), 0, 0);

4856

4857

for (unsigned Part = 0; Part < UF; ++Part) {

4858

Value *Cond = getOrCreateVectorValue(I.getOperand(0), Part);

4859

Value *Op0 = getOrCreateVectorValue(I.getOperand(1), Part);

4860

Value *Op1 = getOrCreateVectorValue(I.getOperand(2), Part);

4861

Value *Sel =

4862

Builder.CreateSelect(InvariantCond ? ScalarCond : Cond, Op0, Op1);

4863

VectorLoopValueMap.setVectorValue(&I, Part, Sel);

4864

addMetadata(Sel, &I);

4865

}

4866

4867

break;

4868

}

4869

4870

case Instruction::ICmp:

4871

case Instruction::FCmp: {

4872

// Widen compares. Generate vector compares.

4873

bool FCmp = (I.getOpcode() == Instruction::FCmp);

4874

auto *Cmp = dyn_cast<CmpInst>(&I);

4875

setDebugLocFromInst(Builder, Cmp);

4876

for (unsigned Part = 0; Part < UF; ++Part) {

4877

Value *A = getOrCreateVectorValue(Cmp->getOperand(0), Part);

4878

Value *B = getOrCreateVectorValue(Cmp->getOperand(1), Part);

4879

Value *C = nullptr;

4880

if (FCmp) {

4881

C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);

4882

cast<FCmpInst>(C)->copyFastMathFlags(Cmp);

4883

} else {

4884

C = Builder.CreateICmp(Cmp->getPredicate(), A, B);

4885

}

4886

VectorLoopValueMap.setVectorValue(&I, Part, C);

4887

addMetadata(C, &I);

4888

}

4889

4890

break;

4891

}

4892

4893

case Instruction::Store:

4894

case Instruction::Load:

4895

vectorizeMemoryInstruction(&I);

4896

break;

4897

case Instruction::ZExt:

4898

case Instruction::SExt:

4899

case Instruction::FPToUI:

4900

case Instruction::FPToSI:

4901

case Instruction::FPExt:

4902

case Instruction::PtrToInt:

4903

case Instruction::IntToPtr:

4904

case Instruction::SIToFP:

4905

case Instruction::UIToFP:

4906

case Instruction::Trunc:

4907

case Instruction::FPTrunc:

4908

case Instruction::BitCast: {

4909

auto *CI = dyn_cast<CastInst>(&I);

4910

setDebugLocFromInst(Builder, CI);

4911

4912

// Optimize the special case where the source is a constant integer

4913

// induction variable. Notice that we can only optimize the 'trunc' case

4914

// because (a) FP conversions lose precision, (b) sext/zext may wrap, and

4915

// (c) other casts depend on pointer size.

4916

if (Cost->isOptimizableIVTruncate(CI, VF)) {

4917

widenIntOrFpInduction(cast<PHINode>(CI->getOperand(0)),

4918

cast<TruncInst>(CI));

4919

break;

4920

}

4921

4922

/// Vectorize casts.

4923

Type *DestTy =

4924

(VF == 1) ? CI->getType() : VectorType::get(CI->getType(), VF);

4925

4926

for (unsigned Part = 0; Part < UF; ++Part) {

4927

Value *A = getOrCreateVectorValue(CI->getOperand(0), Part);

4928

Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);

4929

VectorLoopValueMap.setVectorValue(&I, Part, Cast);

4930

addMetadata(Cast, &I);

4931

}

4932

break;

4933

}

4934

4935

case Instruction::Call: {

4936

// Ignore dbg intrinsics.

4937

if (isa<DbgInfoIntrinsic>(I))

4938

break;

4939

setDebugLocFromInst(Builder, &I);

4940

4941

Module *M = I.getParent()->getParent()->getParent();

4942

auto *CI = cast<CallInst>(&I);

4943

4944

StringRef FnName = CI->getCalledFunction()->getName();

4945

Function *F = CI->getCalledFunction();

4946

Type *RetTy = ToVectorTy(CI->getType(), VF);

4947

SmallVector<Type *, 4> Tys;

4948

for (Value *ArgOperand : CI->arg_operands())

4949

Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));

4950

4951

Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);

4952

if (ID && (ID == Intrinsic::assume || ID == Intrinsic::lifetime_end ||

4953

ID == Intrinsic::lifetime_start)) {

4954

scalarizeInstruction(&I);

4955

break;

4956

}

4957

// The flag shows whether we use Intrinsic or a usual Call for vectorized

4958

// version of the instruction.

4959

// Is it beneficial to perform intrinsic call compared to lib call?

4960

bool NeedToScalarize;

4961

unsigned CallCost = getVectorCallCost(CI, VF, *TTI, TLI, NeedToScalarize);

4962

bool UseVectorIntrinsic =

4963

ID && getVectorIntrinsicCost(CI, VF, *TTI, TLI) <= CallCost;

4964

if (!UseVectorIntrinsic && NeedToScalarize) {

4965

scalarizeInstruction(&I);

4966

break;

4967

}

4968

4969

for (unsigned Part = 0; Part < UF; ++Part) {

4970

SmallVector<Value *, 4> Args;

4971

for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i) {

4972

Value *Arg = CI->getArgOperand(i);

4973

// Some intrinsics have a scalar argument - don't replace it with a

4974

// vector.

4975

if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, i))

4976

Arg = getOrCreateVectorValue(CI->getArgOperand(i), Part);

4977

Args.push_back(Arg);

4978

}

4979

4980

Function *VectorF;

4981

if (UseVectorIntrinsic) {

4982

// Use vector version of the intrinsic.

4983

Type *TysForDecl[] = {CI->getType()};

4984

if (VF > 1)

4985

TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);

4986

VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);

4987

} else {

4988

// Use vector version of the library call.

4989

StringRef VFnName = TLI->getVectorizedFunction(FnName, VF);

4990

assert(!VFnName.empty() && "Vector function name is empty.")((!VFnName.empty() && "Vector function name is empty."
) ? static_cast<void> (0) : __assert_fail ("!VFnName.empty() && \"Vector function name is empty.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4990, __PRETTY_FUNCTION__));

4991

VectorF = M->getFunction(VFnName);

4992

if (!VectorF) {

4993

// Generate a declaration

4994

FunctionType *FTy = FunctionType::get(RetTy, Tys, false);

4995

VectorF =

4996

Function::Create(FTy, Function::ExternalLinkage, VFnName, M);

4997

VectorF->copyAttributesFrom(F);

4998

}

4999

}

5000

assert(VectorF && "Can't create vector function.")((VectorF && "Can't create vector function.") ? static_cast
<void> (0) : __assert_fail ("VectorF && \"Can't create vector function.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5000, __PRETTY_FUNCTION__));

5001

5002

SmallVector<OperandBundleDef, 1> OpBundles;

5003

CI->getOperandBundlesAsDefs(OpBundles);

5004

CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);

5005

5006

if (isa<FPMathOperator>(V))

5007

V->copyFastMathFlags(CI);

5008

5009

VectorLoopValueMap.setVectorValue(&I, Part, V);

5010

addMetadata(V, &I);

5011

}

5012

5013

break;

5014

}

5015

5016

default:

5017

// All other instructions are unsupported. Scalarize them.

5018

scalarizeInstruction(&I);

5019

break;

5020

} // end of switch.

5021

}

5022

5023

void InnerLoopVectorizer::updateAnalysis() {

5024

// Forget the original basic block.

5025

PSE.getSE()->forgetLoop(OrigLoop);

5026

5027

// Update the dominator tree information.

5028

assert(DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock) &&((DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock
) && "Entry does not dominate exit.") ? static_cast<
void> (0) : __assert_fail ("DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock) && \"Entry does not dominate exit.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5029, __PRETTY_FUNCTION__))

5029

"Entry does not dominate exit.")((DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock
) && "Entry does not dominate exit.") ? static_cast<
void> (0) : __assert_fail ("DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock) && \"Entry does not dominate exit.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5029, __PRETTY_FUNCTION__));

5030

5031

DT->addNewBlock(LI->getLoopFor(LoopVectorBody)->getHeader(),

5032

LoopVectorPreHeader);

5033

DT->addNewBlock(LoopMiddleBlock,

5034

LI->getLoopFor(LoopVectorBody)->getLoopLatch());

5035

DT->addNewBlock(LoopScalarPreHeader, LoopBypassBlocks[0]);

5036

DT->changeImmediateDominator(LoopScalarBody, LoopScalarPreHeader);

5037

DT->changeImmediateDominator(LoopExitBlock, LoopBypassBlocks[0]);

5038

5039

DEBUG(DT->verifyDomTree())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { DT->verifyDomTree(); } } while (false
);

5040

}

5041

5042

/// \brief Check whether it is safe to if-convert this phi node.

5043

///

5044

/// Phi nodes with constant expressions that can trap are not safe to if

5045

/// convert.

5046

static bool canIfConvertPHINodes(BasicBlock *BB) {

5047

for (Instruction &I : *BB) {

5048

auto *Phi = dyn_cast<PHINode>(&I);

5049

if (!Phi)

5050

return true;

5051

for (Value *V : Phi->incoming_values())

5052

if (auto *C = dyn_cast<Constant>(V))

5053

if (C->canTrap())

5054

return false;

5055

}

5056

return true;

5057

}

5058

5059

bool LoopVectorizationLegality::canVectorizeWithIfConvert() {

5060

if (!EnableIfConversion) {

5061

ORE->emit(createMissedAnalysis("IfConversionDisabled")

5062

<< "if-conversion is disabled");

5063

return false;

5064

}

5065

5066

assert(TheLoop->getNumBlocks() > 1 && "Single block loops are vectorizable")((TheLoop->getNumBlocks() > 1 && "Single block loops are vectorizable"
) ? static_cast<void> (0) : __assert_fail ("TheLoop->getNumBlocks() > 1 && \"Single block loops are vectorizable\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5066, __PRETTY_FUNCTION__));

5067

5068

// A list of pointers that we can safely read and write to.

5069

SmallPtrSet<Value *, 8> SafePointes;

5070

5071

// Collect safe addresses.

5072

for (BasicBlock *BB : TheLoop->blocks()) {

5073

if (blockNeedsPredication(BB))

5074

continue;

5075

5076

for (Instruction &I : *BB)

5077

if (auto *Ptr = getPointerOperand(&I))

5078

SafePointes.insert(Ptr);

5079

}

5080

5081

// Collect the blocks that need predication.

5082

BasicBlock *Header = TheLoop->getHeader();

5083

for (BasicBlock *BB : TheLoop->blocks()) {

5084

// We don't support switch statements inside loops.

5085

if (!isa<BranchInst>(BB->getTerminator())) {

5086

ORE->emit(createMissedAnalysis("LoopContainsSwitch", BB->getTerminator())

5087

<< "loop contains a switch statement");

5088

return false;

5089

}

5090

5091

// We must be able to predicate all blocks that need to be predicated.

5092

if (blockNeedsPredication(BB)) {

5093

if (!blockCanBePredicated(BB, SafePointes)) {

5094

ORE->emit(createMissedAnalysis("NoCFGForSelect", BB->getTerminator())

5095

<< "control flow cannot be substituted for a select");

5096

return false;

5097

}

5098

} else if (BB != Header && !canIfConvertPHINodes(BB)) {

5099

ORE->emit(createMissedAnalysis("NoCFGForSelect", BB->getTerminator())

5100

<< "control flow cannot be substituted for a select");

5101

return false;

5102

}

5103

}

5104

5105

// We can if-convert this loop.

5106

return true;

5107

}

5108

5109

bool LoopVectorizationLegality::canVectorize() {

5110

// Store the result and return it at the end instead of exiting early, in case

5111

// allowExtraAnalysis is used to report multiple reasons for not vectorizing.

5112

bool Result = true;

5113

// We must have a loop in canonical form. Loops with indirectbr in them cannot

5114

// be canonicalized.

5115

if (!TheLoop->getLoopPreheader()) {

5116

ORE->emit(createMissedAnalysis("CFGNotUnderstood")

5117

<< "loop control flow is not understood by vectorizer");

5118

if (ORE->allowExtraAnalysis())

5119

Result = false;

5120

else

5121

return false;

5122

}

5123

5124

// FIXME: The code is currently dead, since the loop gets sent to

5125

// LoopVectorizationLegality is already an innermost loop.

5126

5127

// We can only vectorize innermost loops.

5128

if (!TheLoop->empty()) {

5129

ORE->emit(createMissedAnalysis("NotInnermostLoop")

5130

<< "loop is not the innermost loop");

5131

if (ORE->allowExtraAnalysis())

5132

Result = false;

5133

else

5134

return false;

5135

}

5136

5137

// We must have a single backedge.

5138

if (TheLoop->getNumBackEdges() != 1) {

5139

ORE->emit(createMissedAnalysis("CFGNotUnderstood")

5140

<< "loop control flow is not understood by vectorizer");

5141

if (ORE->allowExtraAnalysis())

5142

Result = false;

5143

else

5144

return false;

5145

}

5146

5147

// We must have a single exiting block.

5148

if (!TheLoop->getExitingBlock()) {

5149

ORE->emit(createMissedAnalysis("CFGNotUnderstood")

5150

<< "loop control flow is not understood by vectorizer");

5151

if (ORE->allowExtraAnalysis())

5152

Result = false;

5153

else

5154

return false;

5155

}

5156

5157

// We only handle bottom-tested loops, i.e. loop in which the condition is

5158

// checked at the end of each iteration. With that we can assume that all

5159

// instructions in the loop are executed the same number of times.

5160

if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {

5161

ORE->emit(createMissedAnalysis("CFGNotUnderstood")

5162

<< "loop control flow is not understood by vectorizer");

5163

if (ORE->allowExtraAnalysis())

5164

Result = false;

5165

else

5166

return false;

5167

}

5168

5169

// We need to have a loop header.

5170

DEBUG(dbgs() << "LV: Found a loop: " << TheLoop->getHeader()->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a loop: " <<
TheLoop->getHeader()->getName() << '\n'; } } while
(false)

5171

<< '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a loop: " <<
TheLoop->getHeader()->getName() << '\n'; } } while
(false);

5172

5173

// Check if we can if-convert non-single-bb loops.

5174

unsigned NumBlocks = TheLoop->getNumBlocks();

5175

if (NumBlocks != 1 && !canVectorizeWithIfConvert()) {

5176

DEBUG(dbgs() << "LV: Can't if-convert the loop.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Can't if-convert the loop.\n"
; } } while (false);

5177

if (ORE->allowExtraAnalysis())

5178

Result = false;

5179

else

5180

return false;

5181

}

5182

5183

// Check if we can vectorize the instructions and CFG in this loop.

5184

if (!canVectorizeInstrs()) {

5185

DEBUG(dbgs() << "LV: Can't vectorize the instructions or CFG\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Can't vectorize the instructions or CFG\n"
; } } while (false);

5186

if (ORE->allowExtraAnalysis())

5187

Result = false;

5188

else

5189

return false;

5190

}

5191

5192

// Go over each instruction and look at memory deps.

5193

if (!canVectorizeMemory()) {

5194

DEBUG(dbgs() << "LV: Can't vectorize due to memory conflicts\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Can't vectorize due to memory conflicts\n"
; } } while (false);

5195

if (ORE->allowExtraAnalysis())

5196

Result = false;

5197

else

5198

return false;

5199

}

5200

5201

DEBUG(dbgs() << "LV: We can vectorize this loop"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: We can vectorize this loop"
<< (LAI->getRuntimePointerChecking()->Need ? " (with a runtime bound check)"
: "") << "!\n"; } } while (false)

5202

<< (LAI->getRuntimePointerChecking()->Needdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: We can vectorize this loop"
<< (LAI->getRuntimePointerChecking()->Need ? " (with a runtime bound check)"
: "") << "!\n"; } } while (false)

5203

? " (with a runtime bound check)"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: We can vectorize this loop"
<< (LAI->getRuntimePointerChecking()->Need ? " (with a runtime bound check)"
: "") << "!\n"; } } while (false)

5204

: "")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: We can vectorize this loop"
<< (LAI->getRuntimePointerChecking()->Need ? " (with a runtime bound check)"
: "") << "!\n"; } } while (false)

5205

<< "!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: We can vectorize this loop"
<< (LAI->getRuntimePointerChecking()->Need ? " (with a runtime bound check)"
: "") << "!\n"; } } while (false);

5206

5207

bool UseInterleaved = TTI->enableInterleavedAccessVectorization();

5208

5209

// If an override option has been passed in for interleaved accesses, use it.

5210

if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)

5211

UseInterleaved = EnableInterleavedMemAccesses;

5212

5213

// Analyze interleaved memory accesses.

5214

if (UseInterleaved)

5215

InterleaveInfo.analyzeInterleaving(*getSymbolicStrides());

5216

5217

unsigned SCEVThreshold = VectorizeSCEVCheckThreshold;

5218

if (Hints->getForce() == LoopVectorizeHints::FK_Enabled)

5219

SCEVThreshold = PragmaVectorizeSCEVCheckThreshold;

5220

5221

if (PSE.getUnionPredicate().getComplexity() > SCEVThreshold) {

5222

ORE->emit(createMissedAnalysis("TooManySCEVRunTimeChecks")

5223

<< "Too many SCEV assumptions need to be made and checked "

5224

<< "at runtime");

5225

DEBUG(dbgs() << "LV: Too many SCEV checks needed.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Too many SCEV checks needed.\n"
; } } while (false);

5226

if (ORE->allowExtraAnalysis())

5227

Result = false;

5228

else

5229

return false;

5230

}

5231

5232

// Okay! We've done all the tests. If any have failed, return false. Otherwise

5233

// we can vectorize, and at this point we don't have any other mem analysis

5234

// which may limit our maximum vectorization factor, so just return true with

5235

// no restrictions.

5236

return Result;

5237

}

5238

5239

static Type *convertPointerToIntegerType(const DataLayout &DL, Type *Ty) {

5240

if (Ty->isPointerTy())

5241

return DL.getIntPtrType(Ty);

5242

5243

// It is possible that char's or short's overflow when we ask for the loop's

5244

// trip count, work around this by changing the type size.

5245

if (Ty->getScalarSizeInBits() < 32)

5246

return Type::getInt32Ty(Ty->getContext());

5247

5248

return Ty;

5249

}

5250

5251

static Type *getWiderType(const DataLayout &DL, Type *Ty0, Type *Ty1) {

5252

Ty0 = convertPointerToIntegerType(DL, Ty0);

5253

Ty1 = convertPointerToIntegerType(DL, Ty1);

5254

if (Ty0->getScalarSizeInBits() > Ty1->getScalarSizeInBits())

5255

return Ty0;

5256

return Ty1;

5257

}

5258

5259

/// \brief Check that the instruction has outside loop users and is not an

5260

/// identified reduction variable.

5261

static bool hasOutsideLoopUser(const Loop *TheLoop, Instruction *Inst,

5262

SmallPtrSetImpl<Value *> &AllowedExit) {

5263

// Reduction and Induction instructions are allowed to have exit users. All

5264

// other instructions must not have external users.

5265

if (!AllowedExit.count(Inst))

5266

// Check that all of the users of the loop are inside the BB.

5267

for (User *U : Inst->users()) {

5268

Instruction *UI = cast<Instruction>(U);

5269

// This user may be a reduction exit value.

5270

if (!TheLoop->contains(UI)) {

5271

DEBUG(dbgs() << "LV: Found an outside user for : " << *UI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an outside user for : "
<< *UI << '\n'; } } while (false);

5272

return true;

5273

}

5274

}

5275

return false;

5276

}

5277

5278

void LoopVectorizationLegality::addInductionPhi(

5279

PHINode *Phi, const InductionDescriptor &ID,

5280

SmallPtrSetImpl<Value *> &AllowedExit) {

5281

Inductions[Phi] = ID;

5282

Type *PhiTy = Phi->getType();

5283

const DataLayout &DL = Phi->getModule()->getDataLayout();

5284

5285

// Get the widest type.

5286

if (!PhiTy->isFloatingPointTy()) {

5287

if (!WidestIndTy)

5288

WidestIndTy = convertPointerToIntegerType(DL, PhiTy);

5289

else

5290

WidestIndTy = getWiderType(DL, PhiTy, WidestIndTy);

5291

}

5292

5293

// Int inductions are special because we only allow one IV.

5294

if (ID.getKind() == InductionDescriptor::IK_IntInduction &&

5295

ID.getConstIntStepValue() &&

5296

ID.getConstIntStepValue()->isOne() &&

5297

isa<Constant>(ID.getStartValue()) &&

5298

cast<Constant>(ID.getStartValue())->isNullValue()) {

5299

5300

// Use the phi node with the widest type as induction. Use the last

5301

// one if there are multiple (no good reason for doing this other

5302

// than it is expedient). We've checked that it begins at zero and

5303

// steps by one, so this is a canonical induction variable.

5304

if (!PrimaryInduction || PhiTy == WidestIndTy)

5305

PrimaryInduction = Phi;

5306

}

5307

5308

// Both the PHI node itself, and the "post-increment" value feeding

5309

// back into the PHI node may have external users.

5310

AllowedExit.insert(Phi);

5311

AllowedExit.insert(Phi->getIncomingValueForBlock(TheLoop->getLoopLatch()));

5312

5313

DEBUG(dbgs() << "LV: Found an induction variable.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an induction variable.\n"
; } } while (false);

5314

return;

5315

}

5316

5317

bool LoopVectorizationLegality::canVectorizeInstrs() {

5318

BasicBlock *Header = TheLoop->getHeader();

5319

5320

// Look for the attribute signaling the absence of NaNs.

5321

Function &F = *Header->getParent();

5322

HasFunNoNaNAttr =

5323

F.getFnAttribute("no-nans-fp-math").getValueAsString() == "true";

5324

5325

// For each block in the loop.

5326

for (BasicBlock *BB : TheLoop->blocks()) {

5327

// Scan the instructions in the block and look for hazards.

5328

for (Instruction &I : *BB) {

5329

if (auto *Phi = dyn_cast<PHINode>(&I)) {

5330

Type *PhiTy = Phi->getType();

5331

// Check that this PHI type is allowed.

5332

if (!PhiTy->isIntegerTy() && !PhiTy->isFloatingPointTy() &&

5333

!PhiTy->isPointerTy()) {

5334

ORE->emit(createMissedAnalysis("CFGNotUnderstood", Phi)

5335

<< "loop control flow is not understood by vectorizer");

5336

DEBUG(dbgs() << "LV: Found an non-int non-pointer PHI.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an non-int non-pointer PHI.\n"
; } } while (false);

5337

return false;

5338

}

5339

5340

// If this PHINode is not in the header block, then we know that we

5341

// can convert it to select during if-conversion. No need to check if

5342

// the PHIs in this block are induction or reduction variables.

5343

if (BB != Header) {

5344

// Check that this instruction has no outside users or is an

5345

// identified reduction value with an outside user.

5346

if (!hasOutsideLoopUser(TheLoop, Phi, AllowedExit))

5347

continue;

5348

ORE->emit(createMissedAnalysis("NeitherInductionNorReduction", Phi)

5349

<< "value could not be identified as "

5350

"an induction or reduction variable");

5351

return false;

5352

}

5353

5354

// We only allow if-converted PHIs with exactly two incoming values.

5355

if (Phi->getNumIncomingValues() != 2) {

5356

ORE->emit(createMissedAnalysis("CFGNotUnderstood", Phi)

5357

<< "control flow not understood by vectorizer");

5358

DEBUG(dbgs() << "LV: Found an invalid PHI.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an invalid PHI.\n"
; } } while (false);

5359

return false;

5360

}

5361

5362

RecurrenceDescriptor RedDes;

5363

if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes)) {

5364

if (RedDes.hasUnsafeAlgebra())

5365

Requirements->addUnsafeAlgebraInst(RedDes.getUnsafeAlgebraInst());

5366

AllowedExit.insert(RedDes.getLoopExitInstr());

5367

Reductions[Phi] = RedDes;

5368

continue;

5369

}

5370

5371

InductionDescriptor ID;

5372

if (InductionDescriptor::isInductionPHI(Phi, TheLoop, PSE, ID)) {

5373

addInductionPhi(Phi, ID, AllowedExit);

5374

if (ID.hasUnsafeAlgebra() && !HasFunNoNaNAttr)

5375

Requirements->addUnsafeAlgebraInst(ID.getUnsafeAlgebraInst());

5376

continue;

5377

}

5378

5379

if (RecurrenceDescriptor::isFirstOrderRecurrence(Phi, TheLoop, DT)) {

5380

FirstOrderRecurrences.insert(Phi);

5381

continue;

5382

}

5383

5384

// As a last resort, coerce the PHI to a AddRec expression

5385

// and re-try classifying it a an induction PHI.

5386

if (InductionDescriptor::isInductionPHI(Phi, TheLoop, PSE, ID, true)) {

5387

addInductionPhi(Phi, ID, AllowedExit);

5388

continue;

5389

}

5390

5391

ORE->emit(createMissedAnalysis("NonReductionValueUsedOutsideLoop", Phi)

5392

<< "value that could not be identified as "

5393

"reduction is used outside the loop");

5394

DEBUG(dbgs() << "LV: Found an unidentified PHI." << *Phi << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an unidentified PHI."
<< *Phi << "\n"; } } while (false);

5395

return false;

5396

} // end of PHI handling

5397

5398

// We handle calls that:

5399

// * Are debug info intrinsics.

5400

// * Have a mapping to an IR intrinsic.

5401

// * Have a vector version available.

5402

auto *CI = dyn_cast<CallInst>(&I);

5403

if (CI && !getVectorIntrinsicIDForCall(CI, TLI) &&

5404

!isa<DbgInfoIntrinsic>(CI) &&

5405

!(CI->getCalledFunction() && TLI &&

5406

TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) {

5407

ORE->emit(createMissedAnalysis("CantVectorizeCall", CI)

5408

<< "call instruction cannot be vectorized");

5409

DEBUG(dbgs() << "LV: Found a non-intrinsic, non-libfunc callsite.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a non-intrinsic, non-libfunc callsite.\n"
; } } while (false);

5410

return false;

5411

}

5412

5413

// Intrinsics such as powi,cttz and ctlz are legal to vectorize if the

5414

// second argument is the same (i.e. loop invariant)

5415

if (CI && hasVectorInstrinsicScalarOpd(

5416

getVectorIntrinsicIDForCall(CI, TLI), 1)) {

5417

auto *SE = PSE.getSE();

5418

if (!SE->isLoopInvariant(PSE.getSCEV(CI->getOperand(1)), TheLoop)) {

5419

ORE->emit(createMissedAnalysis("CantVectorizeIntrinsic", CI)

5420

<< "intrinsic instruction cannot be vectorized");

5421

DEBUG(dbgs() << "LV: Found unvectorizable intrinsic " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found unvectorizable intrinsic "
<< *CI << "\n"; } } while (false);

5422

return false;

5423

}

5424

}

5425

5426

// Check that the instruction return type is vectorizable.

5427

// Also, we can't vectorize extractelement instructions.

5428

if ((!VectorType::isValidElementType(I.getType()) &&

5429

!I.getType()->isVoidTy()) ||

5430

isa<ExtractElementInst>(I)) {

5431

ORE->emit(createMissedAnalysis("CantVectorizeInstructionReturnType", &I)

5432

<< "instruction return type cannot be vectorized");

5433

DEBUG(dbgs() << "LV: Found unvectorizable type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found unvectorizable type.\n"
; } } while (false);

5434

return false;

5435

}

5436

5437

// Check that the stored type is vectorizable.

5438

if (auto *ST = dyn_cast<StoreInst>(&I)) {

5439

Type *T = ST->getValueOperand()->getType();

5440

if (!VectorType::isValidElementType(T)) {

5441

ORE->emit(createMissedAnalysis("CantVectorizeStore", ST)

5442

<< "store instruction cannot be vectorized");

5443

return false;

5444

}

5445

5446

// FP instructions can allow unsafe algebra, thus vectorizable by

5447

// non-IEEE-754 compliant SIMD units.

5448

// This applies to floating-point math operations and calls, not memory

5449

// operations, shuffles, or casts, as they don't change precision or

5450

// semantics.

5451

} else if (I.getType()->isFloatingPointTy() && (CI || I.isBinaryOp()) &&

5452

!I.hasUnsafeAlgebra()) {

5453

DEBUG(dbgs() << "LV: Found FP op with unsafe algebra.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found FP op with unsafe algebra.\n"
; } } while (false);

5454

Hints->setPotentiallyUnsafe();

5455

}

5456

5457

// Reduction instructions are allowed to have exit users.

5458

// All other instructions must not have external users.

5459

if (hasOutsideLoopUser(TheLoop, &I, AllowedExit)) {

5460

ORE->emit(createMissedAnalysis("ValueUsedOutsideLoop", &I)

5461

<< "value cannot be used outside the loop");

5462

return false;

5463

}

5464

5465

} // next instr.

5466

}

5467

5468

if (!PrimaryInduction) {

5469

DEBUG(dbgs() << "LV: Did not find one integer induction var.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Did not find one integer induction var.\n"
; } } while (false);

5470

if (Inductions.empty()) {

5471

ORE->emit(createMissedAnalysis("NoInductionVariable")

5472

<< "loop induction variable could not be identified");

5473

return false;

5474

}

5475

}

5476

5477

// Now we know the widest induction type, check if our found induction

5478

// is the same size. If it's not, unset it here and InnerLoopVectorizer

5479

// will create another.

5480

if (PrimaryInduction && WidestIndTy != PrimaryInduction->getType())

5481

PrimaryInduction = nullptr;

5482

5483

return true;

5484

}

5485

5486

void LoopVectorizationCostModel::collectLoopScalars(unsigned VF) {

5487

5488

// We should not collect Scalars more than once per VF. Right now, this

5489

// function is called from collectUniformsAndScalars(), which already does

5490

// this check. Collecting Scalars for VF=1 does not make any sense.

5491

assert(VF >= 2 && !Scalars.count(VF) &&((VF >= 2 && !Scalars.count(VF) && "This function should not be visited twice for the same VF"
) ? static_cast<void> (0) : __assert_fail ("VF >= 2 && !Scalars.count(VF) && \"This function should not be visited twice for the same VF\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5492, __PRETTY_FUNCTION__))

5492

"This function should not be visited twice for the same VF")((VF >= 2 && !Scalars.count(VF) && "This function should not be visited twice for the same VF"
) ? static_cast<void> (0) : __assert_fail ("VF >= 2 && !Scalars.count(VF) && \"This function should not be visited twice for the same VF\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5492, __PRETTY_FUNCTION__));

5493

5494

SmallSetVector<Instruction *, 8> Worklist;

5495

5496

// These sets are used to seed the analysis with pointers used by memory

5497

// accesses that will remain scalar.

5498

SmallSetVector<Instruction *, 8> ScalarPtrs;

5499

SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;

5500

5501

// A helper that returns true if the use of Ptr by MemAccess will be scalar.

5502

// The pointer operands of loads and stores will be scalar as long as the

5503

// memory access is not a gather or scatter operation. The value operand of a

5504

// store will remain scalar if the store is scalarized.

5505

auto isScalarUse = [&](Instruction *MemAccess, Value *Ptr) {

5506

InstWidening WideningDecision = getWideningDecision(MemAccess, VF);

5507

assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5508, __PRETTY_FUNCTION__))

5508

"Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5508, __PRETTY_FUNCTION__));

5509

if (auto *Store = dyn_cast<StoreInst>(MemAccess))

5510

if (Ptr == Store->getValueOperand())

5511

return WideningDecision == CM_Scalarize;

5512

assert(Ptr == getPointerOperand(MemAccess) &&((Ptr == getPointerOperand(MemAccess) && "Ptr is neither a value or pointer operand"
) ? static_cast<void> (0) : __assert_fail ("Ptr == getPointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5513, __PRETTY_FUNCTION__))

5513

"Ptr is neither a value or pointer operand")((Ptr == getPointerOperand(MemAccess) && "Ptr is neither a value or pointer operand"
) ? static_cast<void> (0) : __assert_fail ("Ptr == getPointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5513, __PRETTY_FUNCTION__));

5514

return WideningDecision != CM_GatherScatter;

5515

};

5516

5517

// A helper that returns true if the given value is a bitcast or

5518

// getelementptr instruction contained in the loop.

5519

auto isLoopVaryingBitCastOrGEP = [&](Value *V) {

5520

return ((isa<BitCastInst>(V) && V->getType()->isPointerTy()) ||

5521

isa<GetElementPtrInst>(V)) &&

5522

!TheLoop->isLoopInvariant(V);

5523

};

5524

5525

// A helper that evaluates a memory access's use of a pointer. If the use

5526

// will be a scalar use, and the pointer is only used by memory accesses, we

5527

// place the pointer in ScalarPtrs. Otherwise, the pointer is placed in

5528

// PossibleNonScalarPtrs.

5529

auto evaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {

5530

5531

// We only care about bitcast and getelementptr instructions contained in

5532

// the loop.

5533

if (!isLoopVaryingBitCastOrGEP(Ptr))

5534

return;

5535

5536

// If the pointer has already been identified as scalar (e.g., if it was

5537

// also identified as uniform), there's nothing to do.

5538

auto *I = cast<Instruction>(Ptr);

5539

if (Worklist.count(I))

5540

return;

5541

5542

// If the use of the pointer will be a scalar use, and all users of the

5543

// pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,

5544

// place the pointer in PossibleNonScalarPtrs.

5545

if (isScalarUse(MemAccess, Ptr) && all_of(I->users(), [&](User *U) {

5546

return isa<LoadInst>(U) || isa<StoreInst>(U);

5547

}))

5548

ScalarPtrs.insert(I);

5549

else

5550

PossibleNonScalarPtrs.insert(I);

5551

};

5552

5553

// We seed the scalars analysis with three classes of instructions: (1)

5554

// instructions marked uniform-after-vectorization, (2) bitcast and

5555

// getelementptr instructions used by memory accesses requiring a scalar use,

5556

// and (3) pointer induction variables and their update instructions (we

5557

// currently only scalarize these).

5558

5559

// (1) Add to the worklist all instructions that have been identified as

5560

// uniform-after-vectorization.

5561

Worklist.insert(Uniforms[VF].begin(), Uniforms[VF].end());

5562

5563

// (2) Add to the worklist all bitcast and getelementptr instructions used by

5564

// memory accesses requiring a scalar use. The pointer operands of loads and

5565

// stores will be scalar as long as the memory accesses is not a gather or

5566

// scatter operation. The value operand of a store will remain scalar if the

5567

// store is scalarized.

5568

for (auto *BB : TheLoop->blocks())

5569

for (auto &I : *BB) {

5570

if (auto *Load = dyn_cast<LoadInst>(&I)) {

5571

evaluatePtrUse(Load, Load->getPointerOperand());

5572

} else if (auto *Store = dyn_cast<StoreInst>(&I)) {

5573

evaluatePtrUse(Store, Store->getPointerOperand());

5574

evaluatePtrUse(Store, Store->getValueOperand());

5575

}

5576

}

5577

for (auto *I : ScalarPtrs)

5578

if (!PossibleNonScalarPtrs.count(I)) {

5579

DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *I << "\n"; } } while (false);

5580

Worklist.insert(I);

5581

}

5582

5583

// (3) Add to the worklist all pointer induction variables and their update

5584

// instructions.

5585

5586

// TODO: Once we are able to vectorize pointer induction variables we should

5587

// no longer insert them into the worklist here.

5588

auto *Latch = TheLoop->getLoopLatch();

5589

for (auto &Induction : *Legal->getInductionVars()) {

5590

auto *Ind = Induction.first;

5591

auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));

5592

if (Induction.second.getKind() != InductionDescriptor::IK_PtrInduction)

5593

continue;

5594

Worklist.insert(Ind);

5595

Worklist.insert(IndUpdate);

5596

DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *Ind << "\n"; } } while (false);

5597

DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *IndUpdate << "\n"; } } while (false);

5598

}

5599

5600

// Insert the forced scalars.

5601

// FIXME: Currently widenPHIInstruction() often creates a dead vector

5602

// induction variable when the PHI user is scalarized.

5603

if (ForcedScalars.count(VF))

5604

for (auto *I : ForcedScalars.find(VF)->second)

5605

Worklist.insert(I);

5606

5607

// Expand the worklist by looking through any bitcasts and getelementptr

5608

// instructions we've already identified as scalar. This is similar to the

5609

// expansion step in collectLoopUniforms(); however, here we're only

5610

// expanding to include additional bitcasts and getelementptr instructions.

5611

unsigned Idx = 0;

5612

while (Idx != Worklist.size()) {

5613

Instruction *Dst = Worklist[Idx++];

5614

if (!isLoopVaryingBitCastOrGEP(Dst->getOperand(0)))

5615

continue;

5616

auto *Src = cast<Instruction>(Dst->getOperand(0));

5617

if (all_of(Src->users(), [&](User *U) -> bool {

5618

auto *J = cast<Instruction>(U);

5619

return !TheLoop->contains(J) || Worklist.count(J) ||

5620

((isa<LoadInst>(J) || isa<StoreInst>(J)) &&

5621

isScalarUse(J, Src));

5622

})) {

5623

Worklist.insert(Src);

5624

DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *Src << "\n"; } } while (false);

5625

}

5626

}

5627

5628

// An induction variable will remain scalar if all users of the induction

5629

// variable and induction variable update remain scalar.

5630

for (auto &Induction : *Legal->getInductionVars()) {

5631

auto *Ind = Induction.first;

5632

auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));

5633

5634

// We already considered pointer induction variables, so there's no reason

5635

// to look at their users again.

5636

5637

// TODO: Once we are able to vectorize pointer induction variables we

5638

// should no longer skip over them here.

5639

if (Induction.second.getKind() == InductionDescriptor::IK_PtrInduction)

5640

continue;

5641

5642

// Determine if all users of the induction variable are scalar after

5643

// vectorization.

5644

auto ScalarInd = all_of(Ind->users(), [&](User *U) -> bool {

5645

auto *I = cast<Instruction>(U);

5646

return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I);

5647

});

5648

if (!ScalarInd)

5649

continue;

5650

5651

// Determine if all users of the induction variable update instruction are

5652

// scalar after vectorization.

5653

auto ScalarIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {

5654

auto *I = cast<Instruction>(U);

5655

return I == Ind || !TheLoop->contains(I) || Worklist.count(I);

5656

});

5657

if (!ScalarIndUpdate)

5658

continue;

5659

5660

// The induction variable and its update instruction will remain scalar.

5661

Worklist.insert(Ind);

5662

Worklist.insert(IndUpdate);

5663

5664

5665

}

5666

5667

Scalars[VF].insert(Worklist.begin(), Worklist.end());

5668

}

5669

5670

bool LoopVectorizationLegality::isScalarWithPredication(Instruction *I) {

5671

if (!blockNeedsPredication(I->getParent()))

5672

return false;

5673

switch(I->getOpcode()) {

5674

default:

5675

break;

5676

case Instruction::Store:

5677

return !isMaskRequired(I);

5678

case Instruction::UDiv:

5679

case Instruction::SDiv:

5680

case Instruction::SRem:

5681

case Instruction::URem:

5682

return mayDivideByZero(*I);

5683

}

5684

return false;

5685

}

5686

5687

bool LoopVectorizationLegality::memoryInstructionCanBeWidened(Instruction *I,

5688

unsigned VF) {

5689

// Get and ensure we have a valid memory instruction.

5690

LoadInst *LI = dyn_cast<LoadInst>(I);

5691

StoreInst *SI = dyn_cast<StoreInst>(I);

5692

assert((LI || SI) && "Invalid memory instruction")(((LI || SI) && "Invalid memory instruction") ? static_cast
<void> (0) : __assert_fail ("(LI || SI) && \"Invalid memory instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5692, __PRETTY_FUNCTION__));

5693

5694

auto *Ptr = getPointerOperand(I);

5695

5696

// In order to be widened, the pointer should be consecutive, first of all.

5697

if (!isConsecutivePtr(Ptr))

5698

return false;

5699

5700

// If the instruction is a store located in a predicated block, it will be

5701

// scalarized.

5702

if (isScalarWithPredication(I))

5703

return false;

5704

5705

// If the instruction's allocated size doesn't equal it's type size, it

5706

// requires padding and will be scalarized.

5707

auto &DL = I->getModule()->getDataLayout();

5708

auto *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType();

5709

if (hasIrregularType(ScalarTy, DL, VF))

5710

return false;

5711

5712

return true;

5713

}

5714

5715

void LoopVectorizationCostModel::collectLoopUniforms(unsigned VF) {

5716

5717

// We should not collect Uniforms more than once per VF. Right now,

5718

// this function is called from collectUniformsAndScalars(), which

5719

// already does this check. Collecting Uniforms for VF=1 does not make any

5720

// sense.

5721

5722

assert(VF >= 2 && !Uniforms.count(VF) &&((VF >= 2 && !Uniforms.count(VF) && "This function should not be visited twice for the same VF"
) ? static_cast<void> (0) : __assert_fail ("VF >= 2 && !Uniforms.count(VF) && \"This function should not be visited twice for the same VF\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5723, __PRETTY_FUNCTION__))

5723

"This function should not be visited twice for the same VF")((VF >= 2 && !Uniforms.count(VF) && "This function should not be visited twice for the same VF"
) ? static_cast<void> (0) : __assert_fail ("VF >= 2 && !Uniforms.count(VF) && \"This function should not be visited twice for the same VF\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5723, __PRETTY_FUNCTION__));

5724

5725

// Visit the list of Uniforms. If we'll not find any uniform value, we'll

5726

// not analyze again. Uniforms.count(VF) will return 1.

5727

Uniforms[VF].clear();

5728

5729

// We now know that the loop is vectorizable!

5730

// Collect instructions inside the loop that will remain uniform after

5731

// vectorization.

5732

5733

// Global values, params and instructions outside of current loop are out of

5734

// scope.

5735

auto isOutOfScope = [&](Value *V) -> bool {

5736

Instruction *I = dyn_cast<Instruction>(V);

5737

return (!I || !TheLoop->contains(I));

5738

};

5739

5740

SetVector<Instruction *> Worklist;

5741

BasicBlock *Latch = TheLoop->getLoopLatch();

5742

5743

// Start with the conditional branch. If the branch condition is an

5744

// instruction contained in the loop that is only used by the branch, it is

5745

// uniform.

5746

auto *Cmp = dyn_cast<Instruction>(Latch->getTerminator()->getOperand(0));

5747

if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse()) {

5748

Worklist.insert(Cmp);

5749

DEBUG(dbgs() << "LV: Found uniform instruction: " << *Cmp << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *Cmp << "\n"; } } while (false);

5750

}

5751

5752

// Holds consecutive and consecutive-like pointers. Consecutive-like pointers

5753

// are pointers that are treated like consecutive pointers during

5754

// vectorization. The pointer operands of interleaved accesses are an

5755

// example.

5756

SmallSetVector<Instruction *, 8> ConsecutiveLikePtrs;

5757

5758

// Holds pointer operands of instructions that are possibly non-uniform.

5759

SmallPtrSet<Instruction *, 8> PossibleNonUniformPtrs;

5760

5761

auto isUniformDecision = [&](Instruction *I, unsigned VF) {

5762

InstWidening WideningDecision = getWideningDecision(I, VF);

5763

5764

5765

5766

return (WideningDecision == CM_Widen ||

5767

WideningDecision == CM_Interleave);

5768

};

5769

// Iterate over the instructions in the loop, and collect all

5770

// consecutive-like pointer operands in ConsecutiveLikePtrs. If it's possible

5771

// that a consecutive-like pointer operand will be scalarized, we collect it

5772

// in PossibleNonUniformPtrs instead. We use two sets here because a single

5773

// getelementptr instruction can be used by both vectorized and scalarized

5774

// memory instructions. For example, if a loop loads and stores from the same

5775

// location, but the store is conditional, the store will be scalarized, and

5776

// the getelementptr won't remain uniform.

5777

for (auto *BB : TheLoop->blocks())

5778

for (auto &I : *BB) {

5779

5780

// If there's no pointer operand, there's nothing to do.

5781

auto *Ptr = dyn_cast_or_null<Instruction>(getPointerOperand(&I));

5782

if (!Ptr)

5783

continue;

5784

5785

// True if all users of Ptr are memory accesses that have Ptr as their

5786

// pointer operand.

5787

auto UsersAreMemAccesses = all_of(Ptr->users(), [&](User *U) -> bool {

5788

return getPointerOperand(U) == Ptr;

5789

});

5790

5791

// Ensure the memory instruction will not be scalarized or used by

5792

// gather/scatter, making its pointer operand non-uniform. If the pointer

5793

// operand is used by any instruction other than a memory access, we

5794

// conservatively assume the pointer operand may be non-uniform.

5795

if (!UsersAreMemAccesses || !isUniformDecision(&I, VF))

5796

PossibleNonUniformPtrs.insert(Ptr);

5797

5798

// If the memory instruction will be vectorized and its pointer operand

5799

// is consecutive-like, or interleaving - the pointer operand should

5800

// remain uniform.

5801

else

5802

ConsecutiveLikePtrs.insert(Ptr);

5803

}

5804

5805

// Add to the Worklist all consecutive and consecutive-like pointers that

5806

// aren't also identified as possibly non-uniform.

5807

for (auto *V : ConsecutiveLikePtrs)

5808

if (!PossibleNonUniformPtrs.count(V)) {

5809

DEBUG(dbgs() << "LV: Found uniform instruction: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *V << "\n"; } } while (false);

5810

Worklist.insert(V);

5811

}

5812

5813

// Expand Worklist in topological order: whenever a new instruction

5814

// is added , its users should be either already inside Worklist, or

5815

// out of scope. It ensures a uniform instruction will only be used

5816

// by uniform instructions or out of scope instructions.

5817

unsigned idx = 0;

5818

while (idx != Worklist.size()) {

5819

Instruction *I = Worklist[idx++];

5820

5821

for (auto OV : I->operand_values()) {

5822

if (isOutOfScope(OV))

5823

continue;

5824

auto *OI = cast<Instruction>(OV);

5825

if (all_of(OI->users(), [&](User *U) -> bool {

5826

auto *J = cast<Instruction>(U);

5827

return !TheLoop->contains(J) || Worklist.count(J) ||

5828

(OI == getPointerOperand(J) && isUniformDecision(J, VF));

5829

})) {

5830

Worklist.insert(OI);

5831

DEBUG(dbgs() << "LV: Found uniform instruction: " << *OI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *OI << "\n"; } } while (false);

5832

}

5833

}

5834

}

5835

5836

// Returns true if Ptr is the pointer operand of a memory access instruction

5837

// I, and I is known to not require scalarization.

5838

auto isVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {

5839

return getPointerOperand(I) == Ptr && isUniformDecision(I, VF);

5840

};

5841

5842

// For an instruction to be added into Worklist above, all its users inside

5843

// the loop should also be in Worklist. However, this condition cannot be

5844

// true for phi nodes that form a cyclic dependence. We must process phi

5845

// nodes separately. An induction variable will remain uniform if all users

5846

// of the induction variable and induction variable update remain uniform.

5847

// The code below handles both pointer and non-pointer induction variables.

5848

for (auto &Induction : *Legal->getInductionVars()) {

5849

auto *Ind = Induction.first;

5850

auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));

5851

5852

// Determine if all users of the induction variable are uniform after

5853

// vectorization.

5854

auto UniformInd = all_of(Ind->users(), [&](User *U) -> bool {

5855

auto *I = cast<Instruction>(U);

5856

return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||

5857

isVectorizedMemAccessUse(I, Ind);

5858

});

5859

if (!UniformInd)

5860

continue;

5861

5862

// Determine if all users of the induction variable update instruction are

5863

// uniform after vectorization.

5864

auto UniformIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {

5865

auto *I = cast<Instruction>(U);

5866

return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||

5867

isVectorizedMemAccessUse(I, IndUpdate);

5868

});

5869

if (!UniformIndUpdate)

5870

continue;

5871

5872

// The induction variable and its update instruction will remain uniform.

5873

Worklist.insert(Ind);

5874

Worklist.insert(IndUpdate);

5875

DEBUG(dbgs() << "LV: Found uniform instruction: " << *Ind << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *Ind << "\n"; } } while (false);

5876

DEBUG(dbgs() << "LV: Found uniform instruction: " << *IndUpdate << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *IndUpdate << "\n"; } } while (false);

5877

}

5878

5879

Uniforms[VF].insert(Worklist.begin(), Worklist.end());

5880

}

5881

5882

bool LoopVectorizationLegality::canVectorizeMemory() {

5883

LAI = &(*GetLAA)(*TheLoop);

5884

InterleaveInfo.setLAI(LAI);

5885

const OptimizationRemarkAnalysis *LAR = LAI->getReport();

5886

if (LAR) {

5887

OptimizationRemarkAnalysis VR(Hints->vectorizeAnalysisPassName(),

5888

"loop not vectorized: ", *LAR);

5889

ORE->emit(VR);

5890

}

5891

if (!LAI->canVectorizeMemory())

5892

return false;

5893

5894

if (LAI->hasStoreToLoopInvariantAddress()) {

5895

ORE->emit(createMissedAnalysis("CantVectorizeStoreToLoopInvariantAddress")

5896

<< "write to a loop invariant address could not be vectorized");

5897

DEBUG(dbgs() << "LV: We don't allow storing to uniform addresses\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: We don't allow storing to uniform addresses\n"
; } } while (false);

5898

return false;

5899

}

5900

5901

Requirements->addRuntimePointerChecks(LAI->getNumRuntimePointerChecks());

5902

PSE.addPredicate(LAI->getPSE().getUnionPredicate());

5903

5904

return true;

5905

}

5906

5907

bool LoopVectorizationLegality::isInductionVariable(const Value *V) {

5908

Value *In0 = const_cast<Value *>(V);

5909

PHINode *PN = dyn_cast_or_null<PHINode>(In0);

5910

if (!PN)

5911

return false;

5912

5913

return Inductions.count(PN);

5914

}

5915

5916

bool LoopVectorizationLegality::isFirstOrderRecurrence(const PHINode *Phi) {

5917

return FirstOrderRecurrences.count(Phi);

5918

}

5919

5920

bool LoopVectorizationLegality::blockNeedsPredication(BasicBlock *BB) {

5921

return LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT);

5922

}

5923

5924

bool LoopVectorizationLegality::blockCanBePredicated(

5925

BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs) {

5926

const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel();

5927

5928

for (Instruction &I : *BB) {

5929

// Check that we don't have a constant expression that can trap as operand.

5930

for (Value *Operand : I.operands()) {

5931

if (auto *C = dyn_cast<Constant>(Operand))

5932

if (C->canTrap())

5933

return false;

5934

}

5935

// We might be able to hoist the load.

5936

if (I.mayReadFromMemory()) {

5937

auto *LI = dyn_cast<LoadInst>(&I);

5938

if (!LI)

5939

return false;

5940

if (!SafePtrs.count(LI->getPointerOperand())) {

5941

if (isLegalMaskedLoad(LI->getType(), LI->getPointerOperand()) ||

5942

isLegalMaskedGather(LI->getType())) {

5943

MaskedOp.insert(LI);

5944

continue;

5945

}

5946

// !llvm.mem.parallel_loop_access implies if-conversion safety.

5947

if (IsAnnotatedParallel)

5948

continue;

5949

return false;

5950

}

5951

}

5952

5953

if (I.mayWriteToMemory()) {

5954

auto *SI = dyn_cast<StoreInst>(&I);

5955

// We only support predication of stores in basic blocks with one

5956

// predecessor.

5957

if (!SI)

5958

return false;

5959

5960

// Build a masked store if it is legal for the target.

5961

if (isLegalMaskedStore(SI->getValueOperand()->getType(),

5962

SI->getPointerOperand()) ||

5963

isLegalMaskedScatter(SI->getValueOperand()->getType())) {

5964

MaskedOp.insert(SI);

5965

continue;

5966

}

5967

5968

bool isSafePtr = (SafePtrs.count(SI->getPointerOperand()) != 0);

5969

bool isSinglePredecessor = SI->getParent()->getSinglePredecessor();

5970

5971

if (++NumPredStores > NumberOfStoresToPredicate || !isSafePtr ||

5972

!isSinglePredecessor)

5973

return false;

5974

}

5975

if (I.mayThrow())

5976

return false;

5977

}

5978

5979

return true;

5980

}

5981

5982

void InterleavedAccessInfo::collectConstStrideAccesses(

5983

MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo,

5984

const ValueToValueMap &Strides) {

5985

5986

auto &DL = TheLoop->getHeader()->getModule()->getDataLayout();

5987

5988

// Since it's desired that the load/store instructions be maintained in

5989

// "program order" for the interleaved access analysis, we have to visit the

5990

// blocks in the loop in reverse postorder (i.e., in a topological order).

5991

// Such an ordering will ensure that any load/store that may be executed

5992

// before a second load/store will precede the second load/store in

5993

// AccessStrideInfo.

5994

LoopBlocksDFS DFS(TheLoop);

5995

DFS.perform(LI);

5996

for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO()))

5997

for (auto &I : *BB) {

5998

auto *LI = dyn_cast<LoadInst>(&I);

5999

auto *SI = dyn_cast<StoreInst>(&I);

6000

if (!LI && !SI)

6001

continue;

6002

6003

Value *Ptr = getPointerOperand(&I);

6004

// We don't check wrapping here because we don't know yet if Ptr will be

6005

// part of a full group or a group with gaps. Checking wrapping for all

6006

// pointers (even those that end up in groups with no gaps) will be overly

6007

// conservative. For full groups, wrapping should be ok since if we would

6008

// wrap around the address space we would do a memory access at nullptr

6009

// even without the transformation. The wrapping checks are therefore

6010

// deferred until after we've formed the interleaved groups.

6011

int64_t Stride = getPtrStride(PSE, Ptr, TheLoop, Strides,

6012

/*Assume=*/true, /*ShouldCheckWrap=*/false);

6013

6014

const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);

6015

PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());

6016

uint64_t Size = DL.getTypeAllocSize(PtrTy->getElementType());

6017

6018

// An alignment of 0 means target ABI alignment.

6019

unsigned Align = getMemInstAlignment(&I);

6020

if (!Align)

6021

Align = DL.getABITypeAlignment(PtrTy->getElementType());

6022

6023

AccessStrideInfo[&I] = StrideDescriptor(Stride, Scev, Size, Align);

6024

}

6025

}

6026

6027

// Analyze interleaved accesses and collect them into interleaved load and

6028

// store groups.

6029

6030

// When generating code for an interleaved load group, we effectively hoist all

6031

// loads in the group to the location of the first load in program order. When

6032

// generating code for an interleaved store group, we sink all stores to the

6033

// location of the last store. This code motion can change the order of load

6034

// and store instructions and may break dependences.

6035

6036

// The code generation strategy mentioned above ensures that we won't violate

6037

// any write-after-read (WAR) dependences.

6038

6039

// E.g., for the WAR dependence: a = A[i]; // (1)

6040

// A[i] = b; // (2)

6041

6042

// The store group of (2) is always inserted at or below (2), and the load

6043

// group of (1) is always inserted at or above (1). Thus, the instructions will

6044

// never be reordered. All other dependences are checked to ensure the

6045

// correctness of the instruction reordering.

6046

6047

// The algorithm visits all memory accesses in the loop in bottom-up program

6048

// order. Program order is established by traversing the blocks in the loop in

6049

// reverse postorder when collecting the accesses.

6050

6051

// We visit the memory accesses in bottom-up order because it can simplify the

6052

// construction of store groups in the presence of write-after-write (WAW)

6053

// dependences.

6054

6055

// E.g., for the WAW dependence: A[i] = a; // (1)

6056

// A[i] = b; // (2)

6057

// A[i + 1] = c; // (3)

6058

6059

// We will first create a store group with (3) and (2). (1) can't be added to

6060

// this group because it and (2) are dependent. However, (1) can be grouped

6061

// with other accesses that may precede it in program order. Note that a

6062

// bottom-up order does not imply that WAW dependences should not be checked.

6063

void InterleavedAccessInfo::analyzeInterleaving(

6064

const ValueToValueMap &Strides) {

6065

DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Analyzing interleaved accesses...\n"
; } } while (false);

6066

6067

// Holds all accesses with a constant stride.

6068

MapVector<Instruction *, StrideDescriptor> AccessStrideInfo;

6069

collectConstStrideAccesses(AccessStrideInfo, Strides);

6070

6071

if (AccessStrideInfo.empty())

Assuming the condition is false

→

←

Taking false branch

→

6072

return;

6073

6074

// Collect the dependences in the loop.

6075

collectDependences();

6076

6077

// Holds all interleaved store groups temporarily.

6078

SmallSetVector<InterleaveGroup *, 4> StoreGroups;

6079

// Holds all interleaved load groups temporarily.

6080

SmallSetVector<InterleaveGroup *, 4> LoadGroups;

6081

6082

// Search in bottom-up program order for pairs of accesses (A and B) that can

6083

// form interleaved load or store groups. In the algorithm below, access A

6084

// precedes access B in program order. We initialize a group for B in the

6085

// outer loop of the algorithm, and then in the inner loop, we attempt to

6086

// insert each A into B's group if:

6087

6088

// 1. A and B have the same stride,

6089

// 2. A and B have the same memory object size, and

6090

// 3. A belongs in B's group according to its distance from B.

6091

6092

// Special care is taken to ensure group formation will not break any

6093

// dependences.

6094

for (auto BI = AccessStrideInfo.rbegin(), E = AccessStrideInfo.rend();

←

Loop condition is true. Entering loop body

→

6095

BI != E; ++BI) {

6096

Instruction *B = BI->first;

6097

StrideDescriptor DesB = BI->second;

6098

6099

// Initialize a group for B if it has an allowable stride. Even if we don't

6100

// create a group for B, we continue with the bottom-up algorithm to ensure

6101

// we don't break any of B's dependences.

6102

InterleaveGroup *Group = nullptr;

←

'Group' initialized to a null pointer value

→

6103

if (isStrided(DesB.Stride)) {

←

Taking false branch

→

6104

Group = getInterleaveGroup(B);

6105

if (!Group) {

6106

DEBUG(dbgs() << "LV: Creating an interleave group with:" << *B << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Creating an interleave group with:"
<< *B << '\n'; } } while (false);

6107

Group = createInterleaveGroup(B, DesB.Stride, DesB.Align);

6108

}

6109

if (B->mayWriteToMemory())

6110

StoreGroups.insert(Group);

6111

else

6112

LoadGroups.insert(Group);

6113

}

6114

6115

for (auto AI = std::next(BI); AI != E; ++AI) {

←

Loop condition is true. Entering loop body

→

←

Loop condition is true. Entering loop body

→

←

Loop condition is true. Entering loop body

→

←

Loop condition is true. Entering loop body

→

6116

Instruction *A = AI->first;

6117

StrideDescriptor DesA = AI->second;

6118

6119

// Our code motion strategy implies that we can't have dependences

6120

// between accesses in an interleaved group and other accesses located

6121

// between the first and last member of the group. Note that this also

6122

// means that a group can't have more than one member at a given offset.

6123

// The accesses in a group can have dependences with other accesses, but

6124

// we must ensure we don't extend the boundaries of the group such that

6125

// we encompass those dependent accesses.

6126

6127

// For example, assume we have the sequence of accesses shown below in a

6128

// stride-2 loop:

6129

6130

// (1, 2) is a group | A[i] = a; // (1)

6131

// | A[i-1] = b; // (2) |

6132

// A[i-3] = c; // (3)

6133

// A[i] = d; // (4) | (2, 4) is not a group

6134

6135

// Because accesses (2) and (3) are dependent, we can group (2) with (1)

6136

// but not with (4). If we did, the dependent access (3) would be within

6137

// the boundaries of the (2, 4) group.

6138

if (!canReorderMemAccessesForInterleavedGroups(&*AI, &*BI)) {

Taking false branch

Taking false branch

Taking false branch

Taking false branch

6139

6140

// If a dependence exists and A is already in a group, we know that A

6141

// must be a store since A precedes B and WAR dependences are allowed.

6142

// Thus, A would be sunk below B. We release A's group to prevent this

6143

// illegal code motion. A will then be free to form another group with

6144

// instructions that precede it.

6145

if (isInterleaved(A)) {

6146

InterleaveGroup *StoreGroup = getInterleaveGroup(A);

6147

StoreGroups.remove(StoreGroup);

6148

releaseGroup(StoreGroup);

6149

}

6150

6151

// If a dependence exists and A is not already in a group (or it was

6152

// and we just released it), B might be hoisted above A (if B is a

6153

// load) or another store might be sunk below A (if B is a store). In

6154

// either case, we can't add additional instructions to B's group. B

6155

// will only form a group with instructions that it precedes.

6156

break;

6157

}

6158

6159

// At this point, we've checked for illegal code motion. If either A or B

6160

// isn't strided, there's nothing left to do.

6161

if (!isStrided(DesA.Stride) || !isStrided(DesB.Stride))

←

Taking false branch

→

6162

continue;

←

Execution continues on line 6115

→

←

Execution continues on line 6115

→

←

Execution continues on line 6115

→

6163

6164

// Ignore A if it's already in a group or isn't the same kind of memory

6165

// operation as B.

6166

if (isInterleaved(A) || A->mayReadFromMemory() != B->mayReadFromMemory())

←

Assuming the condition is false

→

←

Taking false branch

→

6167

continue;

6168

6169

// Check rules 1 and 2. Ignore A if its stride or size is different from

6170

// that of B.

6171

if (DesA.Stride != DesB.Stride || DesA.Size != DesB.Size)

←

Taking false branch

→

6172

continue;

6173

6174

// Ignore A if the memory object of A and B don't belong to the same

6175

// address space

6176

if (getMemInstAddressSpace(A) != getMemInstAddressSpace(B))

←

Taking false branch

→

6177

continue;

6178

6179

// Calculate the distance from A to B.

6180

const SCEVConstant *DistToB = dyn_cast<SCEVConstant>(

6181

PSE.getSE()->getMinusSCEV(DesA.Scev, DesB.Scev));

6182

if (!DistToB)

←

Assuming 'DistToB' is non-null

→

←

Taking false branch

→

6183

continue;

6184

int64_t DistanceToB = DistToB->getAPInt().getSExtValue();

6185

6186

// Check rule 3. Ignore A if its distance to B is not a multiple of the

6187

// size.

6188

if (DistanceToB % static_cast<int64_t>(DesB.Size))

←

Taking false branch

→

6189

continue;

6190

6191

// Ignore A if either A or B is in a predicated block. Although we

6192

// currently prevent group formation for predicated accesses, we may be

6193

// able to relax this limitation in the future once we handle more

6194

// complicated blocks.

6195

if (isPredicated(A->getParent()) || isPredicated(B->getParent()))

←

Assuming the condition is false

→

←

Assuming the condition is false

→

←

Taking false branch

→

6196

continue;

6197

6198

// The index of A is the index of B plus A's distance to B in multiples

6199

// of the size.

6200

int IndexA =

6201

Group->getIndex(B) + DistanceToB / static_cast<int64_t>(DesB.Size);

←

Called C++ object pointer is null

6202

6203

// Try to insert A into B's group.

6204

if (Group->insertMember(A, IndexA, DesA.Align)) {

6205

DEBUG(dbgs() << "LV: Inserted:" << *A << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Inserted:" <<
*A << '\n' << " into the interleave group with"
<< *B << '\n'; } } while (false)

6206

<< " into the interleave group with" << *B << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Inserted:" <<
*A << '\n' << " into the interleave group with"
<< *B << '\n'; } } while (false);

6207

InterleaveGroupMap[A] = Group;

6208

6209

// Set the first load in program order as the insert position.

6210

if (A->mayReadFromMemory())

6211

Group->setInsertPos(A);

6212

}

6213

} // Iteration over A accesses.

6214

} // Iteration over B accesses.

6215

6216

// Remove interleaved store groups with gaps.

6217

for (InterleaveGroup *Group : StoreGroups)

6218

if (Group->getNumMembers() != Group->getFactor())

6219

releaseGroup(Group);

6220

6221

// Remove interleaved groups with gaps (currently only loads) whose memory

6222

// accesses may wrap around. We have to revisit the getPtrStride analysis,

6223

// this time with ShouldCheckWrap=true, since collectConstStrideAccesses does

6224

// not check wrapping (see documentation there).

6225

// FORNOW we use Assume=false;

6226

// TODO: Change to Assume=true but making sure we don't exceed the threshold

6227

// of runtime SCEV assumptions checks (thereby potentially failing to

6228

// vectorize altogether).

6229

// Additional optional optimizations:

6230

// TODO: If we are peeling the loop and we know that the first pointer doesn't

6231

// wrap then we can deduce that all pointers in the group don't wrap.

6232

// This means that we can forcefully peel the loop in order to only have to

6233

// check the first pointer for no-wrap. When we'll change to use Assume=true

6234

// we'll only need at most one runtime check per interleaved group.

6235

6236

for (InterleaveGroup *Group : LoadGroups) {

6237

6238

// Case 1: A full group. Can Skip the checks; For full groups, if the wide

6239

// load would wrap around the address space we would do a memory access at

6240

// nullptr even without the transformation.

6241

if (Group->getNumMembers() == Group->getFactor())

6242

continue;

6243

6244

// Case 2: If first and last members of the group don't wrap this implies

6245

// that all the pointers in the group don't wrap.

6246

// So we check only group member 0 (which is always guaranteed to exist),

6247

// and group member Factor - 1; If the latter doesn't exist we rely on

6248

// peeling (if it is a non-reveresed accsess -- see Case 3).

6249

Value *FirstMemberPtr = getPointerOperand(Group->getMember(0));

6250

if (!getPtrStride(PSE, FirstMemberPtr, TheLoop, Strides, /*Assume=*/false,

6251

/*ShouldCheckWrap=*/true)) {

6252

DEBUG(dbgs() << "LV: Invalidate candidate interleaved group due to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Invalidate candidate interleaved group due to "
"first group member potentially pointer-wrapping.\n"; } } while
(false)

6253

"first group member potentially pointer-wrapping.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Invalidate candidate interleaved group due to "
"first group member potentially pointer-wrapping.\n"; } } while
(false);

6254

releaseGroup(Group);

6255

continue;

6256

}

6257

Instruction *LastMember = Group->getMember(Group->getFactor() - 1);

6258

if (LastMember) {

6259

Value *LastMemberPtr = getPointerOperand(LastMember);

6260

if (!getPtrStride(PSE, LastMemberPtr, TheLoop, Strides, /*Assume=*/false,

6261

/*ShouldCheckWrap=*/true)) {

6262

DEBUG(dbgs() << "LV: Invalidate candidate interleaved group due to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Invalidate candidate interleaved group due to "
"last group member potentially pointer-wrapping.\n"; } } while
(false)

6263

"last group member potentially pointer-wrapping.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Invalidate candidate interleaved group due to "
"last group member potentially pointer-wrapping.\n"; } } while
(false);

6264

releaseGroup(Group);

6265

}

6266

} else {

6267

// Case 3: A non-reversed interleaved load group with gaps: We need

6268

// to execute at least one scalar epilogue iteration. This will ensure

6269

// we don't speculatively access memory out-of-bounds. We only need

6270

// to look for a member at index factor - 1, since every group must have

6271

// a member at index zero.

6272

if (Group->isReverse()) {

6273

releaseGroup(Group);

6274

continue;

6275

}

6276

DEBUG(dbgs() << "LV: Interleaved group requires epilogue iteration.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaved group requires epilogue iteration.\n"
; } } while (false);

6277

RequiresScalarEpilogue = true;

6278

}

6279

}

6280

}

6281

6282

Optional<unsigned> LoopVectorizationCostModel::computeMaxVF(bool OptForSize) {

6283

if (!EnableCondStoresVectorization && Legal->getNumPredStores()) {

6284

ORE->emit(createMissedAnalysis("ConditionalStore")

6285

<< "store that is conditionally executed prevents vectorization");

6286

DEBUG(dbgs() << "LV: No vectorization. There are conditional stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: No vectorization. There are conditional stores.\n"
; } } while (false);

6287

return None;

6288

}

6289

6290

if (!OptForSize) // Remaining checks deal with scalar loop when OptForSize.

6291

return computeFeasibleMaxVF(OptForSize);

6292

6293

if (Legal->getRuntimePointerChecking()->Need) {

6294

ORE->emit(createMissedAnalysis("CantVersionLoopWithOptForSize")

6295

<< "runtime pointer checks needed. Enable vectorization of this "

6296

"loop with '#pragma clang loop vectorize(enable)' when "

6297

"compiling with -Os/-Oz");

6298

DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime ptr check is required with -Os/-Oz.\n"
; } } while (false)

6299

<< "LV: Aborting. Runtime ptr check is required with -Os/-Oz.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime ptr check is required with -Os/-Oz.\n"
; } } while (false);

6300

return None;

6301

}

6302

6303

// If we optimize the program for size, avoid creating the tail loop.

6304

unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);

6305

DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found trip count: "
<< TC << '\n'; } } while (false);

6306

6307

// If we don't know the precise trip count, don't try to vectorize.

6308

if (TC < 2) {

6309

ORE->emit(

6310

createMissedAnalysis("UnknownLoopCountComplexCFG")

6311

<< "unable to calculate the loop count due to complex control flow");

6312

DEBUG(dbgs() << "LV: Aborting. A tail loop is required with -Os/-Oz.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. A tail loop is required with -Os/-Oz.\n"
; } } while (false);

6313

return None;

6314

}

6315

6316

unsigned MaxVF = computeFeasibleMaxVF(OptForSize);

6317

6318

if (TC % MaxVF != 0) {

6319

// If the trip count that we found modulo the vectorization factor is not

6320

// zero then we require a tail.

6321

// FIXME: look for a smaller MaxVF that does divide TC rather than give up.

6322

// FIXME: return None if loop requiresScalarEpilog(<MaxVF>), or look for a

6323

// smaller MaxVF that does not require a scalar epilog.

6324

6325

ORE->emit(createMissedAnalysis("NoTailLoopWithOptForSize")

6326

<< "cannot optimize for size and vectorize at the "

6327

"same time. Enable vectorization of this loop "

6328

"with '#pragma clang loop vectorize(enable)' "

6329

"when compiling with -Os/-Oz");

6330

6331

return None;

6332

}

6333

6334

return MaxVF;

6335

}

6336

6337

unsigned LoopVectorizationCostModel::computeFeasibleMaxVF(bool OptForSize) {

6338

MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);

6339

unsigned SmallestType, WidestType;

6340

std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();

6341

unsigned WidestRegister = TTI.getRegisterBitWidth(true);

6342

unsigned MaxSafeDepDist = -1U;

6343

6344

// Get the maximum safe dependence distance in bits computed by LAA. If the

6345

// loop contains any interleaved accesses, we divide the dependence distance

6346

// by the maximum interleave factor of all interleaved groups. Note that

6347

// although the division ensures correctness, this is a fairly conservative

6348

// computation because the maximum distance computed by LAA may not involve

6349

// any of the interleaved accesses.

6350

if (Legal->getMaxSafeDepDistBytes() != -1U)

6351

MaxSafeDepDist =

6352

Legal->getMaxSafeDepDistBytes() * 8 / Legal->getMaxInterleaveFactor();

6353

6354

WidestRegister =

6355

((WidestRegister < MaxSafeDepDist) ? WidestRegister : MaxSafeDepDist);

6356

unsigned MaxVectorSize = WidestRegister / WidestType;

6357

6358

DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestType << " / "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Smallest and Widest types: "
<< SmallestType << " / " << WidestType <<
" bits.\n"; } } while (false)

6359

<< WidestType << " bits.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Smallest and Widest types: "
<< SmallestType << " / " << WidestType <<
" bits.\n"; } } while (false);

6360

DEBUG(dbgs() << "LV: The Widest register is: " << WidestRegisterdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Widest register is: "
<< WidestRegister << " bits.\n"; } } while (false
)

6361

<< " bits.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Widest register is: "
<< WidestRegister << " bits.\n"; } } while (false
);

6362

6363

if (MaxVectorSize == 0) {

6364

DEBUG(dbgs() << "LV: The target has no vector registers.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has no vector registers.\n"
; } } while (false);

6365

MaxVectorSize = 1;

6366

}

6367

6368

assert(MaxVectorSize <= 64 && "Did not expect to pack so many elements"((MaxVectorSize <= 64 && "Did not expect to pack so many elements"
" into one vector!") ? static_cast<void> (0) : __assert_fail
("MaxVectorSize <= 64 && \"Did not expect to pack so many elements\" \" into one vector!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6369, __PRETTY_FUNCTION__))

6369

" into one vector!")((MaxVectorSize <= 64 && "Did not expect to pack so many elements"
" into one vector!") ? static_cast<void> (0) : __assert_fail
("MaxVectorSize <= 64 && \"Did not expect to pack so many elements\" \" into one vector!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6369, __PRETTY_FUNCTION__));

6370

6371

unsigned MaxVF = MaxVectorSize;

6372

if (MaximizeBandwidth && !OptForSize) {

6373

// Collect all viable vectorization factors.

6374

SmallVector<unsigned, 8> VFs;

6375

unsigned NewMaxVectorSize = WidestRegister / SmallestType;

6376

for (unsigned VS = MaxVectorSize; VS <= NewMaxVectorSize; VS *= 2)

6377

VFs.push_back(VS);

6378

6379

// For each VF calculate its register usage.

6380

auto RUs = calculateRegisterUsage(VFs);

6381

6382

// Select the largest VF which doesn't require more registers than existing

6383

// ones.

6384

unsigned TargetNumRegisters = TTI.getNumberOfRegisters(true);

6385

for (int i = RUs.size() - 1; i >= 0; --i) {

6386

if (RUs[i].MaxLocalUsers <= TargetNumRegisters) {

6387

MaxVF = VFs[i];

6388

break;

6389

}

6390

}

6391

}

6392

return MaxVF;

6393

}

6394

6395

LoopVectorizationCostModel::VectorizationFactor

6396

LoopVectorizationCostModel::selectVectorizationFactor(unsigned MaxVF) {

6397

float Cost = expectedCost(1).first;

6398

#ifndef NDEBUG

6399

const float ScalarCost = Cost;

6400

#endif /* NDEBUG */

6401

unsigned Width = 1;

6402

DEBUG(dbgs() << "LV: Scalar loop costs: " << (int)ScalarCost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Scalar loop costs: "
<< (int)ScalarCost << ".\n"; } } while (false);

6403

6404

bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled;

6405

// Ignore scalar width, because the user explicitly wants vectorization.

6406

if (ForceVectorization && MaxVF > 1) {

6407

Width = 2;

6408

Cost = expectedCost(Width).first / (float)Width;

6409

}

6410

6411

for (unsigned i = 2; i <= MaxVF; i *= 2) {

6412

// Notice that the vector loop needs to be executed less times, so

6413

// we need to divide the cost of the vector loops by the width of

6414

// the vector elements.

6415

VectorizationCostTy C = expectedCost(i);

6416

float VectorCost = C.first / (float)i;

6417

DEBUG(dbgs() << "LV: Vector loop of width " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Vector loop of width "
<< i << " costs: " << (int)VectorCost <<
".\n"; } } while (false)

6418

<< " costs: " << (int)VectorCost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Vector loop of width "
<< i << " costs: " << (int)VectorCost <<
".\n"; } } while (false);

6419

if (!C.second && !ForceVectorization) {

6420

DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not considering vector loop of width "
<< i << " because it will not generate any vector instructions.\n"
; } } while (false)

6421

dbgs() << "LV: Not considering vector loop of width " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not considering vector loop of width "
<< i << " because it will not generate any vector instructions.\n"
; } } while (false)

6422

<< " because it will not generate any vector instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not considering vector loop of width "
<< i << " because it will not generate any vector instructions.\n"
; } } while (false);

6423

continue;

6424

}

6425

if (VectorCost < Cost) {

6426

Cost = VectorCost;

6427

Width = i;

6428

}

6429

}

6430

6431

DEBUG(if (ForceVectorization && Width > 1 && Cost >= ScalarCost) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (ForceVectorization && Width
> 1 && Cost >= ScalarCost) dbgs() << "LV: Vectorization seems to be not beneficial, "
<< "but was forced by a user.\n"; } } while (false)

6432

<< "LV: Vectorization seems to be not beneficial, "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (ForceVectorization && Width
> 1 && Cost >= ScalarCost) dbgs() << "LV: Vectorization seems to be not beneficial, "
<< "but was forced by a user.\n"; } } while (false)

6433

<< "but was forced by a user.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (ForceVectorization && Width
> 1 && Cost >= ScalarCost) dbgs() << "LV: Vectorization seems to be not beneficial, "
<< "but was forced by a user.\n"; } } while (false);

6434

DEBUG(dbgs() << "LV: Selecting VF: " << Width << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Selecting VF: " <<
Width << ".\n"; } } while (false);

6435

VectorizationFactor Factor = {Width, (unsigned)(Width * Cost)};

6436

return Factor;

6437

}

6438

6439

std::pair<unsigned, unsigned>

6440

LoopVectorizationCostModel::getSmallestAndWidestTypes() {

6441

unsigned MinWidth = -1U;

6442

unsigned MaxWidth = 8;

6443

const DataLayout &DL = TheFunction->getParent()->getDataLayout();

6444

6445

// For each block.

6446

for (BasicBlock *BB : TheLoop->blocks()) {

6447

// For each instruction in the loop.

6448

for (Instruction &I : *BB) {

6449

Type *T = I.getType();

6450

6451

// Skip ignored values.

6452

if (ValuesToIgnore.count(&I))

6453

continue;

6454

6455

// Only examine Loads, Stores and PHINodes.

6456

if (!isa<LoadInst>(I) && !isa<StoreInst>(I) && !isa<PHINode>(I))

6457

continue;

6458

6459

// Examine PHI nodes that are reduction variables. Update the type to

6460

// account for the recurrence type.

6461

if (auto *PN = dyn_cast<PHINode>(&I)) {

6462

if (!Legal->isReductionVariable(PN))

6463

continue;

6464

RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[PN];

6465

T = RdxDesc.getRecurrenceType();

6466

}

6467

6468

// Examine the stored values.

6469

if (auto *ST = dyn_cast<StoreInst>(&I))

6470

T = ST->getValueOperand()->getType();

6471

6472

// Ignore loaded pointer types and stored pointer types that are not

6473

// vectorizable.

6474

6475

// FIXME: The check here attempts to predict whether a load or store will

6476

// be vectorized. We only know this for certain after a VF has

6477

// been selected. Here, we assume that if an access can be

6478

// vectorized, it will be. We should also look at extending this

6479

// optimization to non-pointer types.

6480

6481

if (T->isPointerTy() && !isConsecutiveLoadOrStore(&I) &&

6482

!Legal->isAccessInterleaved(&I) && !Legal->isLegalGatherOrScatter(&I))

6483

continue;

6484

6485

MinWidth = std::min(MinWidth,

6486

(unsigned)DL.getTypeSizeInBits(T->getScalarType()));

6487

MaxWidth = std::max(MaxWidth,

6488

(unsigned)DL.getTypeSizeInBits(T->getScalarType()));

6489

}

6490

}

6491

6492

return {MinWidth, MaxWidth};

6493

}

6494

6495

unsigned LoopVectorizationCostModel::selectInterleaveCount(bool OptForSize,

6496

unsigned VF,

6497

unsigned LoopCost) {

6498

6499

// -- The interleave heuristics --

6500

// We interleave the loop in order to expose ILP and reduce the loop overhead.

6501

// There are many micro-architectural considerations that we can't predict

6502

// at this level. For example, frontend pressure (on decode or fetch) due to

6503

// code size, or the number and capabilities of the execution ports.

6504

6505

// We use the following heuristics to select the interleave count:

6506

// 1. If the code has reductions, then we interleave to break the cross

6507

// iteration dependency.

6508

// 2. If the loop is really small, then we interleave to reduce the loop

6509

// overhead.

6510

// 3. We don't interleave if we think that we will spill registers to memory

6511

// due to the increased register pressure.

6512

6513

// When we optimize for size, we don't interleave.

6514

if (OptForSize)

6515

return 1;

6516

6517

// We used the distance for the interleave count.

6518

if (Legal->getMaxSafeDepDistBytes() != -1U)

6519

return 1;

6520

6521

// Do not interleave loops with a relatively small trip count.

6522

unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);

6523

if (TC > 1 && TC < TinyTripCountInterleaveThreshold)

6524

return 1;

6525

6526

unsigned TargetNumRegisters = TTI.getNumberOfRegisters(VF > 1);

6527

DEBUG(dbgs() << "LV: The target has " << TargetNumRegistersdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has " <<
TargetNumRegisters << " registers\n"; } } while (false
)

6528

<< " registers\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has " <<
TargetNumRegisters << " registers\n"; } } while (false
);

6529

6530

if (VF == 1) {

6531

if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)

6532

TargetNumRegisters = ForceTargetNumScalarRegs;

6533

} else {

6534

if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)

6535

TargetNumRegisters = ForceTargetNumVectorRegs;

6536

}

6537

6538

RegisterUsage R = calculateRegisterUsage({VF})[0];

6539

// We divide by these constants so assume that we have at least one

6540

// instruction that uses at least one register.

6541

R.MaxLocalUsers = std::max(R.MaxLocalUsers, 1U);

6542

R.NumInstructions = std::max(R.NumInstructions, 1U);

6543

6544

// We calculate the interleave count using the following formula.

6545

// Subtract the number of loop invariants from the number of available

6546

// registers. These registers are used by all of the interleaved instances.

6547

// Next, divide the remaining registers by the number of registers that is

6548

// required by the loop, in order to estimate how many parallel instances

6549

// fit without causing spills. All of this is rounded down if necessary to be

6550

// a power of two. We want power of two interleave count to simplify any

6551

// addressing operations or alignment considerations.

6552

unsigned IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs) /

6553

R.MaxLocalUsers);

6554

6555

// Don't count the induction variable as interleaved.

6556

if (EnableIndVarRegisterHeur)

6557

IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs - 1) /

6558

std::max(1U, (R.MaxLocalUsers - 1)));

6559

6560

// Clamp the interleave ranges to reasonable counts.

6561

unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);

6562

6563

// Check if the user has overridden the max.

6564

if (VF == 1) {

6565

if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)

6566

MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;

6567

} else {

6568

if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)

6569

MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;

6570

}

6571

6572

// If we did not calculate the cost for VF (because the user selected the VF)

6573

// then we calculate the cost of VF here.

6574

if (LoopCost == 0)

6575

LoopCost = expectedCost(VF).first;

6576

6577

// Clamp the calculated IC to be between the 1 and the max interleave count

6578

// that the target allows.

6579

if (IC > MaxInterleaveCount)

6580

IC = MaxInterleaveCount;

6581

else if (IC < 1)

6582

IC = 1;

6583

6584

// Interleave if we vectorized this loop and there is a reduction that could

6585

// benefit from interleaving.

6586

if (VF > 1 && Legal->getReductionVars()->size()) {

6587

DEBUG(dbgs() << "LV: Interleaving because of reductions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving because of reductions.\n"
; } } while (false);

6588

return IC;

6589

}

6590

6591

// Note that if we've already vectorized the loop we will have done the

6592

// runtime check and so interleaving won't require further checks.

6593

bool InterleavingRequiresRuntimePointerCheck =

6594

(VF == 1 && Legal->getRuntimePointerChecking()->Need);

6595

6596

// We want to interleave small loops in order to reduce the loop overhead and

6597

// potentially expose ILP opportunities.

6598

DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop cost is " <<
LoopCost << '\n'; } } while (false);

6599

if (!InterleavingRequiresRuntimePointerCheck && LoopCost < SmallLoopCost) {

6600

// We assume that the cost overhead is 1 and we use the cost model

6601

// to estimate the cost of the loop and interleave until the cost of the

6602

// loop overhead is about 5% of the cost of the loop.

6603

unsigned SmallIC =

6604

std::min(IC, (unsigned)PowerOf2Floor(SmallLoopCost / LoopCost));

6605

6606

// Interleave until store/load ports (estimated by max interleave count) are

6607

// saturated.

6608

unsigned NumStores = Legal->getNumStores();

6609

unsigned NumLoads = Legal->getNumLoads();

6610

unsigned StoresIC = IC / (NumStores ? NumStores : 1);

6611

unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);

6612

6613

// If we have a scalar reduction (vector reductions are already dealt with

6614

// by this point), we can increase the critical path length if the loop

6615

// we're interleaving is inside another loop. Limit, by default to 2, so the

6616

// critical path only gets increased by one reduction operation.

6617

if (Legal->getReductionVars()->size() && TheLoop->getLoopDepth() > 1) {

6618

unsigned F = static_cast<unsigned>(MaxNestedScalarReductionIC);

6619

SmallIC = std::min(SmallIC, F);

6620

StoresIC = std::min(StoresIC, F);

6621

LoadsIC = std::min(LoadsIC, F);

6622

}

6623

6624

if (EnableLoadStoreRuntimeInterleave &&

6625

std::max(StoresIC, LoadsIC) > SmallIC) {

6626

DEBUG(dbgs() << "LV: Interleaving to saturate store or load ports.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to saturate store or load ports.\n"
; } } while (false);

6627

return std::max(StoresIC, LoadsIC);

6628

}

6629

6630

DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to reduce branch cost.\n"
; } } while (false);

6631

return SmallIC;

6632

}

6633

6634

// Interleave if this is a large loop (small loops are already dealt with by

6635

// this point) that could benefit from interleaving.

6636

bool HasReductions = (Legal->getReductionVars()->size() > 0);

6637

if (TTI.enableAggressiveInterleaving(HasReductions)) {

6638

DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to expose ILP.\n"
; } } while (false);

6639

return IC;

6640

}

6641

6642

DEBUG(dbgs() << "LV: Not Interleaving.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not Interleaving.\n"
; } } while (false);

6643

return 1;

6644

}

6645

6646

SmallVector<LoopVectorizationCostModel::RegisterUsage, 8>

6647

LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<unsigned> VFs) {

6648

// This function calculates the register usage by measuring the highest number

6649

// of values that are alive at a single location. Obviously, this is a very

6650

// rough estimation. We scan the loop in a topological order in order and

6651

// assign a number to each instruction. We use RPO to ensure that defs are

6652

// met before their users. We assume that each instruction that has in-loop

6653

// users starts an interval. We record every time that an in-loop value is

6654

// used, so we have a list of the first and last occurrences of each

6655

// instruction. Next, we transpose this data structure into a multi map that

6656

// holds the list of intervals that *end* at a specific location. This multi

6657

// map allows us to perform a linear search. We scan the instructions linearly

6658

// and record each time that a new interval starts, by placing it in a set.

6659

// If we find this value in the multi-map then we remove it from the set.

6660

// The max register usage is the maximum size of the set.

6661

// We also search for instructions that are defined outside the loop, but are

6662

// used inside the loop. We need this number separately from the max-interval

6663

// usage number because when we unroll, loop-invariant values do not take

6664

// more register.

6665

LoopBlocksDFS DFS(TheLoop);

6666

DFS.perform(LI);

6667

6668

RegisterUsage RU;

6669

RU.NumInstructions = 0;

6670

6671

// Each 'key' in the map opens a new interval. The values

6672

// of the map are the index of the 'last seen' usage of the

6673

// instruction that is the key.

6674

typedef DenseMap<Instruction *, unsigned> IntervalMap;

6675

// Maps instruction to its index.

6676

DenseMap<unsigned, Instruction *> IdxToInstr;

6677

// Marks the end of each interval.

6678

IntervalMap EndPoint;

6679

// Saves the list of instruction indices that are used in the loop.

6680

SmallSet<Instruction *, 8> Ends;

6681

// Saves the list of values that are used in the loop but are

6682

// defined outside the loop, such as arguments and constants.

6683

SmallPtrSet<Value *, 8> LoopInvariants;

6684

6685

unsigned Index = 0;

6686

for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) {

6687

RU.NumInstructions += BB->size();

6688

for (Instruction &I : *BB) {

6689

IdxToInstr[Index++] = &I;

6690

6691

// Save the end location of each USE.

6692

for (Value *U : I.operands()) {

6693

auto *Instr = dyn_cast<Instruction>(U);

6694

6695

// Ignore non-instruction values such as arguments, constants, etc.

6696

if (!Instr)

6697

continue;

6698

6699

// If this instruction is outside the loop then record it and continue.

6700

if (!TheLoop->contains(Instr)) {

6701

LoopInvariants.insert(Instr);

6702

continue;

6703

}

6704

6705

// Overwrite previous end points.

6706

EndPoint[Instr] = Index;

6707

Ends.insert(Instr);

6708

}

6709

}

6710

}

6711

6712

// Saves the list of intervals that end with the index in 'key'.

6713

typedef SmallVector<Instruction *, 2> InstrList;

6714

DenseMap<unsigned, InstrList> TransposeEnds;

6715

6716

// Transpose the EndPoints to a list of values that end at each index.

6717

for (auto &Interval : EndPoint)

6718

TransposeEnds[Interval.second].push_back(Interval.first);

6719

6720

SmallSet<Instruction *, 8> OpenIntervals;

6721

6722

// Get the size of the widest register.

6723

unsigned MaxSafeDepDist = -1U;

6724

if (Legal->getMaxSafeDepDistBytes() != -1U)

6725

MaxSafeDepDist = Legal->getMaxSafeDepDistBytes() * 8;

6726

unsigned WidestRegister =

6727

std::min(TTI.getRegisterBitWidth(true), MaxSafeDepDist);

6728

const DataLayout &DL = TheFunction->getParent()->getDataLayout();

6729

6730

SmallVector<RegisterUsage, 8> RUs(VFs.size());

6731

SmallVector<unsigned, 8> MaxUsages(VFs.size(), 0);

6732

6733

DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): Calculating max register usage:\n"
; } } while (false);

6734

6735

// A lambda that gets the register usage for the given type and VF.

6736

auto GetRegUsage = [&DL, WidestRegister](Type *Ty, unsigned VF) {

6737

if (Ty->isTokenTy())

6738

return 0U;

6739

unsigned TypeSize = DL.getTypeSizeInBits(Ty->getScalarType());

6740

return std::max<unsigned>(1, VF * TypeSize / WidestRegister);

6741

};

6742

6743

for (unsigned int i = 0; i < Index; ++i) {

6744

Instruction *I = IdxToInstr[i];

6745

6746

// Remove all of the instructions that end at this location.

6747

InstrList &List = TransposeEnds[i];

6748

for (Instruction *ToRemove : List)

6749

OpenIntervals.erase(ToRemove);

6750

6751

// Ignore instructions that are never used within the loop.

6752

if (!Ends.count(I))

6753

continue;

6754

6755

// Skip ignored values.

6756

if (ValuesToIgnore.count(I))

6757

continue;

6758

6759

// For each VF find the maximum usage of registers.

6760

for (unsigned j = 0, e = VFs.size(); j < e; ++j) {

6761

if (VFs[j] == 1) {

6762

MaxUsages[j] = std::max(MaxUsages[j], OpenIntervals.size());

6763

continue;

6764

}

6765

collectUniformsAndScalars(VFs[j]);

6766

// Count the number of live intervals.

6767

unsigned RegUsage = 0;

6768

for (auto Inst : OpenIntervals) {

6769

// Skip ignored values for VF > 1.

6770

if (VecValuesToIgnore.count(Inst) ||

6771

isScalarAfterVectorization(Inst, VFs[j]))

6772

continue;

6773

RegUsage += GetRegUsage(Inst->getType(), VFs[j]);

6774

}

6775

MaxUsages[j] = std::max(MaxUsages[j], RegUsage);

6776

}

6777

6778

DEBUG(dbgs() << "LV(REG): At #" << i << " Interval # "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): At #" <<
i << " Interval # " << OpenIntervals.size() <<
'\n'; } } while (false)

6779

<< OpenIntervals.size() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): At #" <<
i << " Interval # " << OpenIntervals.size() <<
'\n'; } } while (false);

6780

6781

// Add the current instruction to the list of open intervals.

6782

OpenIntervals.insert(I);

6783

}

6784

6785

for (unsigned i = 0, e = VFs.size(); i < e; ++i) {

6786

unsigned Invariant = 0;

6787

if (VFs[i] == 1)

6788

Invariant = LoopInvariants.size();

6789

else {

6790

for (auto Inst : LoopInvariants)

6791

Invariant += GetRegUsage(Inst->getType(), VFs[i]);

6792

}

6793

6794

DEBUG(dbgs() << "LV(REG): VF = " << VFs[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): VF = " <<
VFs[i] << '\n'; } } while (false);

6795

DEBUG(dbgs() << "LV(REG): Found max usage: " << MaxUsages[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): Found max usage: "
<< MaxUsages[i] << '\n'; } } while (false);

6796

DEBUG(dbgs() << "LV(REG): Found invariant usage: " << Invariant << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): Found invariant usage: "
<< Invariant << '\n'; } } while (false);

6797

DEBUG(dbgs() << "LV(REG): LoopSize: " << RU.NumInstructions << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): LoopSize: " <<
RU.NumInstructions << '\n'; } } while (false);

6798

6799

RU.LoopInvariantRegs = Invariant;

6800

RU.MaxLocalUsers = MaxUsages[i];

6801

RUs[i] = RU;

6802

}

6803

6804

return RUs;

6805

}

6806

6807

void LoopVectorizationCostModel::collectInstsToScalarize(unsigned VF) {

6808

6809

// If we aren't vectorizing the loop, or if we've already collected the

6810

// instructions to scalarize, there's nothing to do. Collection may already

6811

// have occurred if we have a user-selected VF and are now computing the

6812

// expected cost for interleaving.

6813

if (VF < 2 || InstsToScalarize.count(VF))

6814

return;

6815

6816

// Initialize a mapping for VF in InstsToScalalarize. If we find that it's

6817

// not profitable to scalarize any instructions, the presence of VF in the

6818

// map will indicate that we've analyzed it already.

6819

ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];

6820

6821

// Find all the instructions that are scalar with predication in the loop and

6822

// determine if it would be better to not if-convert the blocks they are in.

6823

// If so, we also record the instructions to scalarize.

6824

for (BasicBlock *BB : TheLoop->blocks()) {

6825

if (!Legal->blockNeedsPredication(BB))

6826

continue;

6827

for (Instruction &I : *BB)

6828

if (Legal->isScalarWithPredication(&I)) {

6829

ScalarCostsTy ScalarCosts;

6830

if (computePredInstDiscount(&I, ScalarCosts, VF) >= 0)

6831

ScalarCostsVF.insert(ScalarCosts.begin(), ScalarCosts.end());

6832

6833

// Remember that BB will remain after vectorization.

6834

PredicatedBBsAfterVectorization.insert(BB);

6835

}

6836

}

6837

}

6838

6839

int LoopVectorizationCostModel::computePredInstDiscount(

6840

Instruction *PredInst, DenseMap<Instruction *, unsigned> &ScalarCosts,

6841

unsigned VF) {

6842

6843

assert(!isUniformAfterVectorization(PredInst, VF) &&((!isUniformAfterVectorization(PredInst, VF) && "Instruction marked uniform-after-vectorization will be predicated"
) ? static_cast<void> (0) : __assert_fail ("!isUniformAfterVectorization(PredInst, VF) && \"Instruction marked uniform-after-vectorization will be predicated\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6844, __PRETTY_FUNCTION__))

6844

"Instruction marked uniform-after-vectorization will be predicated")((!isUniformAfterVectorization(PredInst, VF) && "Instruction marked uniform-after-vectorization will be predicated"
) ? static_cast<void> (0) : __assert_fail ("!isUniformAfterVectorization(PredInst, VF) && \"Instruction marked uniform-after-vectorization will be predicated\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6844, __PRETTY_FUNCTION__));

6845

6846

// Initialize the discount to zero, meaning that the scalar version and the

6847

// vector version cost the same.

6848

int Discount = 0;

6849

6850

// Holds instructions to analyze. The instructions we visit are mapped in

6851

// ScalarCosts. Those instructions are the ones that would be scalarized if

6852

// we find that the scalar version costs less.

6853

SmallVector<Instruction *, 8> Worklist;

6854

6855

// Returns true if the given instruction can be scalarized.

6856

auto canBeScalarized = [&](Instruction *I) -> bool {

6857

6858

// We only attempt to scalarize instructions forming a single-use chain

6859

// from the original predicated block that would otherwise be vectorized.

6860

// Although not strictly necessary, we give up on instructions we know will

6861

// already be scalar to avoid traversing chains that are unlikely to be

6862

// beneficial.

6863

if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||

6864

isScalarAfterVectorization(I, VF))

6865

return false;

6866

6867

// If the instruction is scalar with predication, it will be analyzed

6868

// separately. We ignore it within the context of PredInst.

6869

if (Legal->isScalarWithPredication(I))

6870

return false;

6871

6872

// If any of the instruction's operands are uniform after vectorization,

6873

// the instruction cannot be scalarized. This prevents, for example, a

6874

// masked load from being scalarized.

6875

6876

// We assume we will only emit a value for lane zero of an instruction

6877

// marked uniform after vectorization, rather than VF identical values.

6878

// Thus, if we scalarize an instruction that uses a uniform, we would

6879

// create uses of values corresponding to the lanes we aren't emitting code

6880

// for. This behavior can be changed by allowing getScalarValue to clone

6881

// the lane zero values for uniforms rather than asserting.

6882

for (Use &U : I->operands())

6883

if (auto *J = dyn_cast<Instruction>(U.get()))

6884

if (isUniformAfterVectorization(J, VF))

6885

return false;

6886

6887

// Otherwise, we can scalarize the instruction.

6888

return true;

6889

};

6890

6891

// Returns true if an operand that cannot be scalarized must be extracted

6892

// from a vector. We will account for this scalarization overhead below. Note

6893

// that the non-void predicated instructions are placed in their own blocks,

6894

// and their return values are inserted into vectors. Thus, an extract would

6895

// still be required.

6896

auto needsExtract = [&](Instruction *I) -> bool {

6897

return TheLoop->contains(I) && !isScalarAfterVectorization(I, VF);

6898

};

6899

6900

// Compute the expected cost discount from scalarizing the entire expression

6901

// feeding the predicated instruction. We currently only consider expressions

6902

// that are single-use instruction chains.

6903

Worklist.push_back(PredInst);

6904

while (!Worklist.empty()) {

6905

Instruction *I = Worklist.pop_back_val();

6906

6907

// If we've already analyzed the instruction, there's nothing to do.

6908

if (ScalarCosts.count(I))

6909

continue;

6910

6911

// Compute the cost of the vector instruction. Note that this cost already

6912

// includes the scalarization overhead of the predicated instruction.

6913

unsigned VectorCost = getInstructionCost(I, VF).first;

6914

6915

// Compute the cost of the scalarized instruction. This cost is the cost of

6916

// the instruction as if it wasn't if-converted and instead remained in the

6917

// predicated block. We will scale this cost by block probability after

6918

// computing the scalarization overhead.

6919

unsigned ScalarCost = VF * getInstructionCost(I, 1).first;

6920

6921

// Compute the scalarization overhead of needed insertelement instructions

6922

// and phi nodes.

6923

if (Legal->isScalarWithPredication(I) && !I->getType()->isVoidTy()) {

6924

ScalarCost += TTI.getScalarizationOverhead(ToVectorTy(I->getType(), VF),

6925

true, false);

6926

ScalarCost += VF * TTI.getCFInstrCost(Instruction::PHI);

6927

}

6928

6929

// Compute the scalarization overhead of needed extractelement

6930

// instructions. For each of the instruction's operands, if the operand can

6931

// be scalarized, add it to the worklist; otherwise, account for the

6932

// overhead.

6933

for (Use &U : I->operands())

6934

if (auto *J = dyn_cast<Instruction>(U.get())) {

6935

assert(VectorType::isValidElementType(J->getType()) &&((VectorType::isValidElementType(J->getType()) && "Instruction has non-scalar type"
) ? static_cast<void> (0) : __assert_fail ("VectorType::isValidElementType(J->getType()) && \"Instruction has non-scalar type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6936, __PRETTY_FUNCTION__))

6936

"Instruction has non-scalar type")((VectorType::isValidElementType(J->getType()) && "Instruction has non-scalar type"
) ? static_cast<void> (0) : __assert_fail ("VectorType::isValidElementType(J->getType()) && \"Instruction has non-scalar type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6936, __PRETTY_FUNCTION__));

6937

if (canBeScalarized(J))

6938

Worklist.push_back(J);

6939

else if (needsExtract(J))

6940

ScalarCost += TTI.getScalarizationOverhead(

6941

ToVectorTy(J->getType(),VF), false, true);

6942

}

6943

6944

// Scale the total scalar cost by block probability.

6945

ScalarCost /= getReciprocalPredBlockProb();

6946

6947

// Compute the discount. A non-negative discount means the vector version

6948

// of the instruction costs more, and scalarizing would be beneficial.

6949

Discount += VectorCost - ScalarCost;

6950

ScalarCosts[I] = ScalarCost;

6951

}

6952

6953

return Discount;

6954

}

6955

6956

LoopVectorizationCostModel::VectorizationCostTy

6957

LoopVectorizationCostModel::expectedCost(unsigned VF) {

6958

VectorizationCostTy Cost;

6959

6960

// Collect Uniform and Scalar instructions after vectorization with VF.

6961

collectUniformsAndScalars(VF);

6962

6963

// Collect the instructions (and their associated costs) that will be more

6964

// profitable to scalarize.

6965

collectInstsToScalarize(VF);

6966

6967

// For each block.

6968

for (BasicBlock *BB : TheLoop->blocks()) {

6969

VectorizationCostTy BlockCost;

6970

6971

// For each instruction in the old loop.

6972

for (Instruction &I : *BB) {

6973

// Skip dbg intrinsics.

6974

if (isa<DbgInfoIntrinsic>(I))

6975

continue;

6976

6977

// Skip ignored values.

6978

if (ValuesToIgnore.count(&I))

6979

continue;

6980

6981

VectorizationCostTy C = getInstructionCost(&I, VF);

6982

6983

// Check if we should override the cost.

6984

if (ForceTargetInstructionCost.getNumOccurrences() > 0)

6985

C.first = ForceTargetInstructionCost;

6986

6987

BlockCost.first += C.first;

6988

BlockCost.second |= C.second;

6989

DEBUG(dbgs() << "LV: Found an estimated cost of " << C.first << " for VF "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an estimated cost of "
<< C.first << " for VF " << VF << " For instruction: "
<< I << '\n'; } } while (false)

6990

<< VF << " For instruction: " << I << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an estimated cost of "
<< C.first << " for VF " << VF << " For instruction: "
<< I << '\n'; } } while (false);

6991

}

6992

6993

// If we are vectorizing a predicated block, it will have been

6994

// if-converted. This means that the block's instructions (aside from

6995

// stores and instructions that may divide by zero) will now be

6996

// unconditionally executed. For the scalar case, we may not always execute

6997

// the predicated block. Thus, scale the block's cost by the probability of

6998

// executing it.

6999

if (VF == 1 && Legal->blockNeedsPredication(BB))

7000

BlockCost.first /= getReciprocalPredBlockProb();

7001

7002

Cost.first += BlockCost.first;

7003

Cost.second |= BlockCost.second;

7004

}

7005

7006

return Cost;

7007

}

7008

7009

/// \brief Gets Address Access SCEV after verifying that the access pattern

7010

/// is loop invariant except the induction variable dependence.

7011

///

7012

/// This SCEV can be sent to the Target in order to estimate the address

7013

/// calculation cost.

7014

static const SCEV *getAddressAccessSCEV(

7015

Value *Ptr,

7016

LoopVectorizationLegality *Legal,

7017

ScalarEvolution *SE,

7018

const Loop *TheLoop) {

7019

auto *Gep = dyn_cast<GetElementPtrInst>(Ptr);

7020

if (!Gep)

7021

return nullptr;

7022

7023

// We are looking for a gep with all loop invariant indices except for one

7024

// which should be an induction variable.

7025

unsigned NumOperands = Gep->getNumOperands();

7026

for (unsigned i = 1; i < NumOperands; ++i) {

7027

Value *Opd = Gep->getOperand(i);

7028

if (!SE->isLoopInvariant(SE->getSCEV(Opd), TheLoop) &&

7029

!Legal->isInductionVariable(Opd))

7030

return nullptr;

7031

}

7032

7033

// Now we know we have a GEP ptr, %inv, %ind, %inv. return the Ptr SCEV.

7034

return SE->getSCEV(Ptr);

7035

}

7036

7037

static bool isStrideMul(Instruction *I, LoopVectorizationLegality *Legal) {

7038

return Legal->hasStride(I->getOperand(0)) ||

7039

Legal->hasStride(I->getOperand(1));

7040

}

7041

7042

unsigned LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,

7043

unsigned VF) {

7044

Type *ValTy = getMemInstValueType(I);

7045

auto SE = PSE.getSE();

7046

7047

unsigned Alignment = getMemInstAlignment(I);

7048

unsigned AS = getMemInstAddressSpace(I);

7049

Value *Ptr = getPointerOperand(I);

7050

Type *PtrTy = ToVectorTy(Ptr->getType(), VF);

7051

7052

// Figure out whether the access is strided and get the stride value

7053

// if it's known in compile time

7054

const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, SE, TheLoop);

7055

7056

// Get the cost of the scalar memory instruction and address computation.

7057

unsigned Cost = VF * TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV);

7058

7059

Cost += VF *

7060

TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment,

7061

AS, I);

7062

7063

// Get the overhead of the extractelement and insertelement instructions

7064

// we might create due to scalarization.

7065

Cost += getScalarizationOverhead(I, VF, TTI);

7066

7067

// If we have a predicated store, it may not be executed for each vector

7068

// lane. Scale the cost by the probability of executing the predicated

7069

// block.

7070

if (Legal->isScalarWithPredication(I))

7071

Cost /= getReciprocalPredBlockProb();

7072

7073

return Cost;

7074

}

7075

7076

unsigned LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,

7077

unsigned VF) {

7078

Type *ValTy = getMemInstValueType(I);

7079

Type *VectorTy = ToVectorTy(ValTy, VF);

7080

unsigned Alignment = getMemInstAlignment(I);

7081

Value *Ptr = getPointerOperand(I);

7082

unsigned AS = getMemInstAddressSpace(I);

7083

int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);

7084

7085

assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&(((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
"Stride should be 1 or -1 for consecutive memory access") ? static_cast
<void> (0) : __assert_fail ("(ConsecutiveStride == 1 || ConsecutiveStride == -1) && \"Stride should be 1 or -1 for consecutive memory access\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7086, __PRETTY_FUNCTION__))

7086

"Stride should be 1 or -1 for consecutive memory access")(((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
"Stride should be 1 or -1 for consecutive memory access") ? static_cast
<void> (0) : __assert_fail ("(ConsecutiveStride == 1 || ConsecutiveStride == -1) && \"Stride should be 1 or -1 for consecutive memory access\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7086, __PRETTY_FUNCTION__));

7087

unsigned Cost = 0;

7088

if (Legal->isMaskRequired(I))

7089

Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS);

7090

else

7091

Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS, I);

7092

7093

bool Reverse = ConsecutiveStride < 0;

7094

if (Reverse)

7095

Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0);

7096

return Cost;

7097

}

7098

7099

unsigned LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,

7100

unsigned VF) {

7101

LoadInst *LI = cast<LoadInst>(I);

7102

Type *ValTy = LI->getType();

7103

Type *VectorTy = ToVectorTy(ValTy, VF);

7104

unsigned Alignment = LI->getAlignment();

7105

unsigned AS = LI->getPointerAddressSpace();

7106

7107

return TTI.getAddressComputationCost(ValTy) +

7108

TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS) +

7109

TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy);

7110

}

7111

7112

unsigned LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,

7113

unsigned VF) {

7114

Type *ValTy = getMemInstValueType(I);

7115

Type *VectorTy = ToVectorTy(ValTy, VF);

7116

unsigned Alignment = getMemInstAlignment(I);

7117

Value *Ptr = getPointerOperand(I);

7118

7119

return TTI.getAddressComputationCost(VectorTy) +

7120

TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,

7121

Legal->isMaskRequired(I), Alignment);

7122

}

7123

7124

unsigned LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,

7125

unsigned VF) {

7126

Type *ValTy = getMemInstValueType(I);

7127

Type *VectorTy = ToVectorTy(ValTy, VF);

7128

unsigned AS = getMemInstAddressSpace(I);

7129

7130

auto Group = Legal->getInterleavedAccessGroup(I);

7131

7132

7133

unsigned InterleaveFactor = Group->getFactor();

7134

Type *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);

7135

7136

// Holds the indices of existing members in an interleaved load group.

7137

// An interleaved store group doesn't need this as it doesn't allow gaps.

7138

SmallVector<unsigned, 4> Indices;

7139

if (isa<LoadInst>(I)) {

7140

for (unsigned i = 0; i < InterleaveFactor; i++)

7141

if (Group->getMember(i))

7142

Indices.push_back(i);

7143

}

7144

7145

// Calculate the cost of the whole interleaved group.

7146

unsigned Cost = TTI.getInterleavedMemoryOpCost(I->getOpcode(), WideVecTy,

7147

Group->getFactor(), Indices,

7148

Group->getAlignment(), AS);

7149

7150

if (Group->isReverse())

7151

Cost += Group->getNumMembers() *

7152

TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0);

7153

return Cost;

7154

}

7155

7156

unsigned LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,

7157

unsigned VF) {

7158

7159

// Calculate scalar cost only. Vectorization cost should be ready at this

7160

// moment.

7161

if (VF == 1) {

7162

Type *ValTy = getMemInstValueType(I);

7163

unsigned Alignment = getMemInstAlignment(I);

7164

unsigned AS = getMemInstAddressSpace(I);

7165

7166

return TTI.getAddressComputationCost(ValTy) +

7167

TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS, I);

7168

}

7169

return getWideningCost(I, VF);

7170

}

7171

7172

LoopVectorizationCostModel::VectorizationCostTy

7173

LoopVectorizationCostModel::getInstructionCost(Instruction *I, unsigned VF) {

7174

// If we know that this instruction will remain uniform, check the cost of

7175

// the scalar version.

7176

if (isUniformAfterVectorization(I, VF))

7177

VF = 1;

7178

7179

if (VF > 1 && isProfitableToScalarize(I, VF))

7180

return VectorizationCostTy(InstsToScalarize[VF][I], false);

7181

7182

// Forced scalars do not have any scalarization overhead.

7183

if (VF > 1 && ForcedScalars.count(VF) &&

7184

ForcedScalars.find(VF)->second.count(I))

7185

return VectorizationCostTy((getInstructionCost(I, 1).first * VF), false);

7186

7187

Type *VectorTy;

7188

unsigned C = getInstructionCost(I, VF, VectorTy);

7189

7190

bool TypeNotScalarized =

7191

VF > 1 && VectorTy->isVectorTy() && TTI.getNumberOfParts(VectorTy) < VF;

7192

return VectorizationCostTy(C, TypeNotScalarized);

7193

}

7194

7195

void LoopVectorizationCostModel::setCostBasedWideningDecision(unsigned VF) {

7196

if (VF == 1)

7197

return;

7198

for (BasicBlock *BB : TheLoop->blocks()) {

7199

// For each instruction in the old loop.

7200

for (Instruction &I : *BB) {

7201

Value *Ptr = getPointerOperand(&I);

7202

if (!Ptr)

7203

continue;

7204

7205

if (isa<LoadInst>(&I) && Legal->isUniform(Ptr)) {

7206

// Scalar load + broadcast

7207

unsigned Cost = getUniformMemOpCost(&I, VF);

7208

setWideningDecision(&I, VF, CM_Scalarize, Cost);

7209

continue;

7210

}

7211

7212

// We assume that widening is the best solution when possible.

7213

if (Legal->memoryInstructionCanBeWidened(&I, VF)) {

7214

unsigned Cost = getConsecutiveMemOpCost(&I, VF);

7215

setWideningDecision(&I, VF, CM_Widen, Cost);

7216

continue;

7217

}

7218

7219

// Choose between Interleaving, Gather/Scatter or Scalarization.

7220

unsigned InterleaveCost = UINT_MAX(2147483647 *2U +1U);

7221

unsigned NumAccesses = 1;

7222

if (Legal->isAccessInterleaved(&I)) {

7223

auto Group = Legal->getInterleavedAccessGroup(&I);

7224

7225

7226

// Make one decision for the whole group.

7227

if (getWideningDecision(&I, VF) != CM_Unknown)

7228

continue;

7229

7230

NumAccesses = Group->getNumMembers();

7231

InterleaveCost = getInterleaveGroupCost(&I, VF);

7232

}

7233

7234

unsigned GatherScatterCost =

7235

Legal->isLegalGatherOrScatter(&I)

7236

? getGatherScatterCost(&I, VF) * NumAccesses

7237

: UINT_MAX(2147483647 *2U +1U);

7238

7239

unsigned ScalarizationCost =

7240

getMemInstScalarizationCost(&I, VF) * NumAccesses;

7241

7242

// Choose better solution for the current VF,

7243

// write down this decision and use it during vectorization.

7244

unsigned Cost;

7245

InstWidening Decision;

7246

if (InterleaveCost <= GatherScatterCost &&

7247

InterleaveCost < ScalarizationCost) {

7248

Decision = CM_Interleave;

7249

Cost = InterleaveCost;

7250

} else if (GatherScatterCost < ScalarizationCost) {

7251

Decision = CM_GatherScatter;

7252

Cost = GatherScatterCost;

7253

} else {

7254

Decision = CM_Scalarize;

7255

Cost = ScalarizationCost;

7256

}

7257

// If the instructions belongs to an interleave group, the whole group

7258

// receives the same decision. The whole group receives the cost, but

7259

// the cost will actually be assigned to one instruction.

7260

if (auto Group = Legal->getInterleavedAccessGroup(&I))

7261

setWideningDecision(Group, VF, Decision, Cost);

7262

else

7263

setWideningDecision(&I, VF, Decision, Cost);

7264

}

7265

}

7266

7267

// Make sure that any load of address and any other address computation

7268

// remains scalar unless there is gather/scatter support. This avoids

7269

// inevitable extracts into address registers, and also has the benefit of

7270

// activating LSR more, since that pass can't optimize vectorized

7271

// addresses.

7272

if (TTI.prefersVectorizedAddressing())

7273

return;

7274

7275

// Start with all scalar pointer uses.

7276

SmallPtrSet<Instruction *, 8> AddrDefs;

7277

for (BasicBlock *BB : TheLoop->blocks())

7278

for (Instruction &I : *BB) {

7279

Instruction *PtrDef =

7280

dyn_cast_or_null<Instruction>(getPointerOperand(&I));

7281

if (PtrDef && TheLoop->contains(PtrDef) &&

7282

getWideningDecision(&I, VF) != CM_GatherScatter)

7283

AddrDefs.insert(PtrDef);

7284

}

7285

7286

// Add all instructions used to generate the addresses.

7287

SmallVector<Instruction *, 4> Worklist;

7288

for (auto *I : AddrDefs)

7289

Worklist.push_back(I);

7290

while (!Worklist.empty()) {

7291

Instruction *I = Worklist.pop_back_val();

7292

for (auto &Op : I->operands())

7293

if (auto *InstOp = dyn_cast<Instruction>(Op))

7294

if ((InstOp->getParent() == I->getParent()) && !isa<PHINode>(InstOp) &&

7295

AddrDefs.insert(InstOp).second == true)

7296

Worklist.push_back(InstOp);

7297

}

7298

7299

for (auto *I : AddrDefs) {

7300

if (isa<LoadInst>(I)) {

7301

// Setting the desired widening decision should ideally be handled in

7302

// by cost functions, but since this involves the task of finding out

7303

// if the loaded register is involved in an address computation, it is

7304

// instead changed here when we know this is the case.

7305

if (getWideningDecision(I, VF) == CM_Widen)

7306

// Scalarize a widened load of address.

7307

setWideningDecision(I, VF, CM_Scalarize,

7308

(VF * getMemoryInstructionCost(I, 1)));

7309

else if (auto Group = Legal->getInterleavedAccessGroup(I)) {

7310

// Scalarize an interleave group of address loads.

7311

for (unsigned I = 0; I < Group->getFactor(); ++I) {

7312

if (Instruction *Member = Group->getMember(I))

7313

setWideningDecision(Member, VF, CM_Scalarize,

7314

(VF * getMemoryInstructionCost(Member, 1)));

7315

}

7316

}

7317

} else

7318

// Make sure I gets scalarized and a cost estimate without

7319

// scalarization overhead.

7320

ForcedScalars[VF].insert(I);

7321

}

7322

}

7323

7324

unsigned LoopVectorizationCostModel::getInstructionCost(Instruction *I,

7325

unsigned VF,

7326

Type *&VectorTy) {

7327

Type *RetTy = I->getType();

7328

if (canTruncateToMinimalBitwidth(I, VF))

7329

RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]);

7330

VectorTy = isScalarAfterVectorization(I, VF) ? RetTy : ToVectorTy(RetTy, VF);

7331

auto SE = PSE.getSE();

7332

7333

// TODO: We need to estimate the cost of intrinsic calls.

7334

switch (I->getOpcode()) {

7335

case Instruction::GetElementPtr:

7336

// We mark this instruction as zero-cost because the cost of GEPs in

7337

// vectorized code depends on whether the corresponding memory instruction

7338

// is scalarized or not. Therefore, we handle GEPs with the memory

7339

// instruction cost.

7340

return 0;

7341

case Instruction::Br: {

7342

// In cases of scalarized and predicated instructions, there will be VF

7343

// predicated blocks in the vectorized loop. Each branch around these

7344

// blocks requires also an extract of its vector compare i1 element.

7345

bool ScalarPredicatedBB = false;

7346

BranchInst *BI = cast<BranchInst>(I);

7347

if (VF > 1 && BI->isConditional() &&

7348

(PredicatedBBsAfterVectorization.count(BI->getSuccessor(0)) ||

7349

PredicatedBBsAfterVectorization.count(BI->getSuccessor(1))))

7350

ScalarPredicatedBB = true;

7351

7352

if (ScalarPredicatedBB) {

7353

// Return cost for branches around scalarized and predicated blocks.

7354

Type *Vec_i1Ty =

7355

VectorType::get(IntegerType::getInt1Ty(RetTy->getContext()), VF);

7356

return (TTI.getScalarizationOverhead(Vec_i1Ty, false, true) +

7357

(TTI.getCFInstrCost(Instruction::Br) * VF));

7358

} else if (I->getParent() == TheLoop->getLoopLatch() || VF == 1)

7359

// The back-edge branch will remain, as will all scalar branches.

7360

return TTI.getCFInstrCost(Instruction::Br);

7361

else

7362

// This branch will be eliminated by if-conversion.

7363

return 0;

7364

// Note: We currently assume zero cost for an unconditional branch inside

7365

// a predicated block since it will become a fall-through, although we

7366

// may decide in the future to call TTI for all branches.

7367

}

7368

case Instruction::PHI: {

7369

auto *Phi = cast<PHINode>(I);

7370

7371

// First-order recurrences are replaced by vector shuffles inside the loop.

7372

if (VF > 1 && Legal->isFirstOrderRecurrence(Phi))

7373

return TTI.getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,

7374

VectorTy, VF - 1, VectorTy);

7375

7376

// Phi nodes in non-header blocks (not inductions, reductions, etc.) are

7377

// converted into select instructions. We require N - 1 selects per phi

7378

// node, where N is the number of incoming values.

7379

if (VF > 1 && Phi->getParent() != TheLoop->getHeader())

7380

return (Phi->getNumIncomingValues() - 1) *

7381

TTI.getCmpSelInstrCost(

7382

Instruction::Select, ToVectorTy(Phi->getType(), VF),

7383

ToVectorTy(Type::getInt1Ty(Phi->getContext()), VF));

7384

7385

return TTI.getCFInstrCost(Instruction::PHI);

7386

}

7387

case Instruction::UDiv:

7388

case Instruction::SDiv:

7389

case Instruction::URem:

7390

case Instruction::SRem:

7391

// If we have a predicated instruction, it may not be executed for each

7392

// vector lane. Get the scalarization cost and scale this amount by the

7393

// probability of executing the predicated block. If the instruction is not

7394

// predicated, we fall through to the next case.

7395

if (VF > 1 && Legal->isScalarWithPredication(I)) {

7396

unsigned Cost = 0;

7397

7398

// These instructions have a non-void type, so account for the phi nodes

7399

// that we will create. This cost is likely to be zero. The phi node

7400

// cost, if any, should be scaled by the block probability because it

7401

// models a copy at the end of each predicated block.

7402

Cost += VF * TTI.getCFInstrCost(Instruction::PHI);

7403

7404

// The cost of the non-predicated instruction.

7405

Cost += VF * TTI.getArithmeticInstrCost(I->getOpcode(), RetTy);

7406

7407

// The cost of insertelement and extractelement instructions needed for

7408

// scalarization.

7409

Cost += getScalarizationOverhead(I, VF, TTI);

7410

7411

// Scale the cost by the probability of executing the predicated blocks.

7412

// This assumes the predicated block for each vector lane is equally

7413

// likely.

7414

return Cost / getReciprocalPredBlockProb();

7415

}

7416

LLVM_FALLTHROUGH[[clang::fallthrough]];

7417

case Instruction::Add:

7418

case Instruction::FAdd:

7419

case Instruction::Sub:

7420

case Instruction::FSub:

7421

case Instruction::Mul:

7422

case Instruction::FMul:

7423

case Instruction::FDiv:

7424

case Instruction::FRem:

7425

case Instruction::Shl:

7426

case Instruction::LShr:

7427

case Instruction::AShr:

7428

case Instruction::And:

7429

case Instruction::Or:

7430

case Instruction::Xor: {

7431

// Since we will replace the stride by 1 the multiplication should go away.

7432

if (I->getOpcode() == Instruction::Mul && isStrideMul(I, Legal))

7433

return 0;

7434

// Certain instructions can be cheaper to vectorize if they have a constant

7435

// second vector operand. One example of this are shifts on x86.

7436

TargetTransformInfo::OperandValueKind Op1VK =

7437

TargetTransformInfo::OK_AnyValue;

7438

TargetTransformInfo::OperandValueKind Op2VK =

7439

TargetTransformInfo::OK_AnyValue;

7440

TargetTransformInfo::OperandValueProperties Op1VP =

7441

TargetTransformInfo::OP_None;

7442

TargetTransformInfo::OperandValueProperties Op2VP =

7443

TargetTransformInfo::OP_None;

7444

Value *Op2 = I->getOperand(1);

7445

7446

// Check for a splat or for a non uniform vector of constants.

7447

if (isa<ConstantInt>(Op2)) {

7448

ConstantInt *CInt = cast<ConstantInt>(Op2);

7449

if (CInt && CInt->getValue().isPowerOf2())

7450

Op2VP = TargetTransformInfo::OP_PowerOf2;

7451

Op2VK = TargetTransformInfo::OK_UniformConstantValue;

7452

} else if (isa<ConstantVector>(Op2) || isa<ConstantDataVector>(Op2)) {

7453

Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;

7454

Constant *SplatValue = cast<Constant>(Op2)->getSplatValue();

7455

if (SplatValue) {

7456

ConstantInt *CInt = dyn_cast<ConstantInt>(SplatValue);

7457

if (CInt && CInt->getValue().isPowerOf2())

7458

Op2VP = TargetTransformInfo::OP_PowerOf2;

7459

Op2VK = TargetTransformInfo::OK_UniformConstantValue;

7460

}

7461

} else if (Legal->isUniform(Op2)) {

7462

Op2VK = TargetTransformInfo::OK_UniformValue;

7463

}

7464

SmallVector<const Value *, 4> Operands(I->operand_values());

7465

unsigned N = isScalarAfterVectorization(I, VF) ? VF : 1;

7466

return N * TTI.getArithmeticInstrCost(I->getOpcode(), VectorTy, Op1VK,

7467

Op2VK, Op1VP, Op2VP, Operands);

7468

}

7469

case Instruction::Select: {

7470

SelectInst *SI = cast<SelectInst>(I);

7471

const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());

7472

bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));

7473

Type *CondTy = SI->getCondition()->getType();

7474

if (!ScalarCond)

7475

CondTy = VectorType::get(CondTy, VF);

7476

7477

return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy, I);

7478

}

7479

case Instruction::ICmp:

7480

case Instruction::FCmp: {

7481

Type *ValTy = I->getOperand(0)->getType();

7482

Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));

7483

if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF))

7484

ValTy = IntegerType::get(ValTy->getContext(), MinBWs[Op0AsInstruction]);

7485

VectorTy = ToVectorTy(ValTy, VF);

7486

return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, nullptr, I);

7487

}

7488

case Instruction::Store:

7489

case Instruction::Load: {

7490

unsigned Width = VF;

7491

if (Width > 1) {

7492

InstWidening Decision = getWideningDecision(I, Width);

7493

assert(Decision != CM_Unknown &&((Decision != CM_Unknown && "CM decision should be taken at this point"
) ? static_cast<void> (0) : __assert_fail ("Decision != CM_Unknown && \"CM decision should be taken at this point\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7494, __PRETTY_FUNCTION__))

7494

"CM decision should be taken at this point")((Decision != CM_Unknown && "CM decision should be taken at this point"
) ? static_cast<void> (0) : __assert_fail ("Decision != CM_Unknown && \"CM decision should be taken at this point\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7494, __PRETTY_FUNCTION__));

7495

if (Decision == CM_Scalarize)

7496

Width = 1;

7497

}

7498

VectorTy = ToVectorTy(getMemInstValueType(I), Width);

7499

return getMemoryInstructionCost(I, VF);

7500

}

7501

case Instruction::ZExt:

7502

case Instruction::SExt:

7503

case Instruction::FPToUI:

7504

case Instruction::FPToSI:

7505

case Instruction::FPExt:

7506

case Instruction::PtrToInt:

7507

case Instruction::IntToPtr:

7508

case Instruction::SIToFP:

7509

case Instruction::UIToFP:

7510

case Instruction::Trunc:

7511

case Instruction::FPTrunc:

7512

case Instruction::BitCast: {

7513

// We optimize the truncation of induction variables having constant

7514

// integer steps. The cost of these truncations is the same as the scalar

7515

// operation.

7516

if (isOptimizableIVTruncate(I, VF)) {

7517

auto *Trunc = cast<TruncInst>(I);

7518

return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),

7519

Trunc->getSrcTy(), Trunc);

7520

}

7521

7522

Type *SrcScalarTy = I->getOperand(0)->getType();

7523

Type *SrcVecTy =

7524

VectorTy->isVectorTy() ? ToVectorTy(SrcScalarTy, VF) : SrcScalarTy;

7525

if (canTruncateToMinimalBitwidth(I, VF)) {

7526

// This cast is going to be shrunk. This may remove the cast or it might

7527

// turn it into slightly different cast. For example, if MinBW == 16,

7528

// "zext i8 %1 to i32" becomes "zext i8 %1 to i16".

7529

7530

// Calculate the modified src and dest types.

7531

Type *MinVecTy = VectorTy;

7532

if (I->getOpcode() == Instruction::Trunc) {

7533

SrcVecTy = smallestIntegerVectorType(SrcVecTy, MinVecTy);

7534

VectorTy =

7535

largestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy);

7536

} else if (I->getOpcode() == Instruction::ZExt ||

7537

I->getOpcode() == Instruction::SExt) {

7538

SrcVecTy = largestIntegerVectorType(SrcVecTy, MinVecTy);

7539

VectorTy =

7540

smallestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy);

7541

}

7542

}

7543

7544

unsigned N = isScalarAfterVectorization(I, VF) ? VF : 1;

7545

return N * TTI.getCastInstrCost(I->getOpcode(), VectorTy, SrcVecTy, I);

7546

}

7547

case Instruction::Call: {

7548

bool NeedToScalarize;

7549

CallInst *CI = cast<CallInst>(I);

7550

unsigned CallCost = getVectorCallCost(CI, VF, TTI, TLI, NeedToScalarize);

7551

if (getVectorIntrinsicIDForCall(CI, TLI))

7552

return std::min(CallCost, getVectorIntrinsicCost(CI, VF, TTI, TLI));

7553

return CallCost;

7554

}

7555

default:

7556

// The cost of executing VF copies of the scalar instruction. This opcode

7557

// is unknown. Assume that it is the same as 'mul'.

7558

return VF * TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy) +

7559

getScalarizationOverhead(I, VF, TTI);

7560

} // end of switch.

7561

}

7562

7563

char LoopVectorize::ID = 0;

7564

static const char lv_name[] = "Loop Vectorization";

7565

INITIALIZE_PASS_BEGIN(LoopVectorize, LV_NAME, lv_name, false, false)static void *initializeLoopVectorizePassOnce(PassRegistry &
Registry) {

7566

INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);

7567

INITIALIZE_PASS_DEPENDENCY(BasicAAWrapperPass)initializeBasicAAWrapperPassPass(Registry);

7568

INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);

7569

INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)initializeGlobalsAAWrapperPassPass(Registry);

7570

INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);

7571

INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)initializeBlockFrequencyInfoWrapperPassPass(Registry);

7572

INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry);

7573

INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)initializeScalarEvolutionWrapperPassPass(Registry);

7574

INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)initializeLoopInfoWrapperPassPass(Registry);

7575

INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis)initializeLoopAccessLegacyAnalysisPass(Registry);

7576

INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)initializeDemandedBitsWrapperPassPass(Registry);

7577

INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry);

7578

INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false)PassInfo *PI = new PassInfo( lv_name, "loop-vectorize", &
LoopVectorize::ID, PassInfo::NormalCtor_t(callDefaultCtor<
LoopVectorize>), false, false); Registry.registerPass(*PI,
true); return PI; } static llvm::once_flag InitializeLoopVectorizePassFlag
; void llvm::initializeLoopVectorizePass(PassRegistry &Registry
) { llvm::call_once(InitializeLoopVectorizePassFlag, initializeLoopVectorizePassOnce
, std::ref(Registry)); }

7579

7580

namespace llvm {

7581

Pass *createLoopVectorizePass(bool NoUnrolling, bool AlwaysVectorize) {

7582

return new LoopVectorize(NoUnrolling, AlwaysVectorize);

7583

}

7584

}

7585

7586

bool LoopVectorizationCostModel::isConsecutiveLoadOrStore(Instruction *Inst) {

7587

7588

// Check if the pointer operand of a load or store instruction is

7589

// consecutive.

7590

if (auto *Ptr = getPointerOperand(Inst))

7591

return Legal->isConsecutivePtr(Ptr);

7592

return false;

7593

}

7594

7595

void LoopVectorizationCostModel::collectValuesToIgnore() {

7596

// Ignore ephemeral values.

7597

CodeMetrics::collectEphemeralValues(TheLoop, AC, ValuesToIgnore);

7598

7599

// Ignore type-promoting instructions we identified during reduction

7600

// detection.

7601

for (auto &Reduction : *Legal->getReductionVars()) {

7602

RecurrenceDescriptor &RedDes = Reduction.second;

7603

SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();

7604

VecValuesToIgnore.insert(Casts.begin(), Casts.end());

7605

}

7606

}

7607

7608

LoopVectorizationCostModel::VectorizationFactor

7609

LoopVectorizationPlanner::plan(bool OptForSize, unsigned UserVF) {

7610

7611

// Width 1 means no vectorize, cost 0 means uncomputed cost.

7612

const LoopVectorizationCostModel::VectorizationFactor NoVectorization = {1U,

7613

0U};

7614

Optional<unsigned> MaybeMaxVF = CM.computeMaxVF(OptForSize);

7615

if (!MaybeMaxVF.hasValue()) // Cases considered too costly to vectorize.

7616

return NoVectorization;

7617

7618

if (UserVF) {

7619

DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Using user VF " <<
UserVF << ".\n"; } } while (false);

7620

assert(isPowerOf2_32(UserVF) && "VF needs to be a power of two")((isPowerOf2_32(UserVF) && "VF needs to be a power of two"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(UserVF) && \"VF needs to be a power of two\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7620, __PRETTY_FUNCTION__));

7621

// Collect the instructions (and their associated costs) that will be more

7622

// profitable to scalarize.

7623

CM.selectUserVectorizationFactor(UserVF);

7624

return {UserVF, 0};

7625

}

7626

7627

unsigned MaxVF = MaybeMaxVF.getValue();

7628

assert(MaxVF != 0 && "MaxVF is zero.")((MaxVF != 0 && "MaxVF is zero.") ? static_cast<void
> (0) : __assert_fail ("MaxVF != 0 && \"MaxVF is zero.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7628, __PRETTY_FUNCTION__));

7629

if (MaxVF == 1)

7630

return NoVectorization;

7631

7632

// Select the optimal vectorization factor.

7633

return CM.selectVectorizationFactor(MaxVF);

7634

}

7635

7636

void LoopVectorizationPlanner::executePlan(InnerLoopVectorizer &ILV) {

7637

// Perform the actual loop transformation.

7638

7639

// 1. Create a new empty loop. Unlink the old loop and connect the new one.

7640

ILV.createVectorizedLoopSkeleton();

7641

7642

//===------------------------------------------------===//

7643

7644

// Notice: any optimization or new instruction that go

7645

// into the code below should also be implemented in

7646

// the cost-model.

7647

7648

//===------------------------------------------------===//

7649

7650

// 2. Copy and widen instructions from the old loop into the new loop.

7651

7652

// Collect instructions from the original loop that will become trivially dead

7653

// in the vectorized loop. We don't need to vectorize these instructions. For

7654

// example, original induction update instructions can become dead because we

7655

// separately emit induction "steps" when generating code for the new loop.

7656

// Similarly, we create a new latch condition when setting up the structure

7657

// of the new loop, so the old one can become dead.

7658

SmallPtrSet<Instruction *, 4> DeadInstructions;

7659

collectTriviallyDeadInstructions(DeadInstructions);

7660

7661

// Scan the loop in a topological order to ensure that defs are vectorized

7662

// before users.

7663

LoopBlocksDFS DFS(OrigLoop);

7664

DFS.perform(LI);

7665

7666

// Vectorize all instructions in the original loop that will not become

7667

// trivially dead when vectorized.

7668

for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO()))

7669

for (Instruction &I : *BB)

7670

if (!DeadInstructions.count(&I))

7671

ILV.vectorizeInstruction(I);

7672

7673

// 3. Fix the vectorized code: take care of header phi's, live-outs,

7674

// predication, updating analyses.

7675

ILV.fixVectorizedLoop();

7676

}

7677

7678

void LoopVectorizationPlanner::collectTriviallyDeadInstructions(

7679

SmallPtrSetImpl<Instruction *> &DeadInstructions) {

7680

BasicBlock *Latch = OrigLoop->getLoopLatch();

7681

7682

// We create new control-flow for the vectorized loop, so the original

7683

// condition will be dead after vectorization if it's only used by the

7684

// branch.

7685

auto *Cmp = dyn_cast<Instruction>(Latch->getTerminator()->getOperand(0));

7686

if (Cmp && Cmp->hasOneUse())

7687

DeadInstructions.insert(Cmp);

7688

7689

// We create new "steps" for induction variable updates to which the original

7690

// induction variables map. An original update instruction will be dead if

7691

// all its users except the induction variable are dead.

7692

for (auto &Induction : *Legal->getInductionVars()) {

7693

PHINode *Ind = Induction.first;

7694

auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));

7695

if (all_of(IndUpdate->users(), [&](User *U) -> bool {

7696

return U == Ind || DeadInstructions.count(cast<Instruction>(U));

7697

}))

7698

DeadInstructions.insert(IndUpdate);

7699

}

7700

}

7701

7702

void InnerLoopUnroller::vectorizeMemoryInstruction(Instruction *Instr) {

7703

auto *SI = dyn_cast<StoreInst>(Instr);

7704

bool IfPredicateInstr = (SI && Legal->blockNeedsPredication(SI->getParent()));

7705

7706

return scalarizeInstruction(Instr, IfPredicateInstr);

7707

}

7708

7709

Value *InnerLoopUnroller::reverseVector(Value *Vec) { return Vec; }

7710

7711

Value *InnerLoopUnroller::getBroadcastInstrs(Value *V) { return V; }

7712

7713

Value *InnerLoopUnroller::getStepVector(Value *Val, int StartIdx, Value *Step,

7714

Instruction::BinaryOps BinOp) {

7715

// When unrolling and the VF is 1, we only need to add a simple scalar.

7716

Type *Ty = Val->getType();

7717

assert(!Ty->isVectorTy() && "Val must be a scalar")((!Ty->isVectorTy() && "Val must be a scalar") ? static_cast
<void> (0) : __assert_fail ("!Ty->isVectorTy() && \"Val must be a scalar\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7717, __PRETTY_FUNCTION__));

7718

7719

if (Ty->isFloatingPointTy()) {

7720

Constant *C = ConstantFP::get(Ty, (double)StartIdx);

7721

7722

// Floating point operations had to be 'fast' to enable the unrolling.

7723

Value *MulOp = addFastMathFlag(Builder.CreateFMul(C, Step));

7724

return addFastMathFlag(Builder.CreateBinOp(BinOp, Val, MulOp));

7725

}

7726

Constant *C = ConstantInt::get(Ty, StartIdx);

7727

return Builder.CreateAdd(Val, Builder.CreateMul(C, Step), "induction");

7728

}

7729

7730

static void AddRuntimeUnrollDisableMetaData(Loop *L) {

7731

SmallVector<Metadata *, 4> MDs;

7732

// Reserve first location for self reference to the LoopID metadata node.

7733

MDs.push_back(nullptr);

7734

bool IsUnrollMetadata = false;

7735

MDNode *LoopID = L->getLoopID();

7736

if (LoopID) {

7737

// First find existing loop unrolling disable metadata.

7738

for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {

7739

auto *MD = dyn_cast<MDNode>(LoopID->getOperand(i));

7740

if (MD) {

7741

const auto *S = dyn_cast<MDString>(MD->getOperand(0));

7742

IsUnrollMetadata =

7743

S && S->getString().startswith("llvm.loop.unroll.disable");

7744

}

7745

MDs.push_back(LoopID->getOperand(i));

7746

}

7747

}

7748

7749

if (!IsUnrollMetadata) {

7750

// Add runtime unroll disable metadata.

7751

LLVMContext &Context = L->getHeader()->getContext();

7752

SmallVector<Metadata *, 1> DisableOperands;

7753

DisableOperands.push_back(

7754

MDString::get(Context, "llvm.loop.unroll.runtime.disable"));

7755

MDNode *DisableNode = MDNode::get(Context, DisableOperands);

7756

MDs.push_back(DisableNode);

7757

MDNode *NewLoopID = MDNode::get(Context, MDs);

7758

// Set operand 0 to refer to the loop id itself.

7759

NewLoopID->replaceOperandWith(0, NewLoopID);

7760

L->setLoopID(NewLoopID);

7761

}

7762

}

7763

7764

bool LoopVectorizePass::processLoop(Loop *L) {

7765

assert(L->empty() && "Only process inner loops.")((L->empty() && "Only process inner loops.") ? static_cast
<void> (0) : __assert_fail ("L->empty() && \"Only process inner loops.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7765, __PRETTY_FUNCTION__));

7766

7767

#ifndef NDEBUG

7768

const std::string DebugLocStr = getDebugLocString(L);

7769

#endif /* NDEBUG */

7770

7771

DEBUG(dbgs() << "\nLV: Checking a loop in \""do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "\nLV: Checking a loop in \""
<< L->getHeader()->getParent()->getName() <<
"\" from " << DebugLocStr << "\n"; } } while (false
)

7772

<< L->getHeader()->getParent()->getName() << "\" from "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "\nLV: Checking a loop in \""
<< L->getHeader()->getParent()->getName() <<
"\" from " << DebugLocStr << "\n"; } } while (false
)

7773

<< DebugLocStr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "\nLV: Checking a loop in \""
<< L->getHeader()->getParent()->getName() <<
"\" from " << DebugLocStr << "\n"; } } while (false
);

7774

7775

LoopVectorizeHints Hints(L, DisableUnrolling, *ORE);

7776

7777

DEBUG(dbgs() << "LV: Loop hints:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
" force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)

7778

<< " force="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
" force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)

7779

<< (Hints.getForce() == LoopVectorizeHints::FK_Disableddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
" force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)

7780

? "disabled"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
" force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)

7781

: (Hints.getForce() == LoopVectorizeHints::FK_Enableddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
" force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)

7782

? "enabled"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
" force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)

7783

: "?"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
" force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)

7784

<< " width=" << Hints.getWidth()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
" force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)

7785

<< " unroll=" << Hints.getInterleave() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
" force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false);

7786

7787

// Function containing loop

7788

Function *F = L->getHeader()->getParent();

7789

7790

// Looking at the diagnostic output is the only way to determine if a loop

7791

// was vectorized (other than looking at the IR or machine code), so it

7792

// is important to generate an optimization remark for each loop. Most of

7793

// these messages are generated as OptimizationRemarkAnalysis. Remarks

7794

// generated as OptimizationRemark and OptimizationRemarkMissed are

7795

// less verbose reporting vectorized loops and unvectorized loops that may

7796

// benefit from vectorization, respectively.

7797

7798

if (!Hints.allowVectorization(F, L, AlwaysVectorize)) {

7799

DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints prevent vectorization.\n"
; } } while (false);

7800

return false;

7801

}

7802

7803

// Check the loop for a trip count threshold:

7804

// do not vectorize loops with a tiny trip count.

7805

unsigned ExpectedTC = SE->getSmallConstantMaxTripCount(L);

7806

bool HasExpectedTC = (ExpectedTC > 0);

7807

7808

if (!HasExpectedTC && LoopVectorizeWithBlockFrequency) {

7809

auto EstimatedTC = getLoopEstimatedTripCount(L);

7810

if (EstimatedTC) {

7811

ExpectedTC = *EstimatedTC;

7812

HasExpectedTC = true;

7813

}

7814

}

7815

7816

if (HasExpectedTC && ExpectedTC < TinyTripCountVectorThreshold) {

7817

DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a loop with a very small trip count. "
<< "This loop is not worth vectorizing."; } } while (false
)

7818

<< "This loop is not worth vectorizing.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a loop with a very small trip count. "
<< "This loop is not worth vectorizing."; } } while (false
);

7819

if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)

7820

DEBUG(dbgs() << " But vectorizing was explicitly forced.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << " But vectorizing was explicitly forced.\n"
; } } while (false);

7821

else {

7822

DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "\n"; } } while (false);

7823

ORE->emit(createMissedAnalysis(Hints.vectorizeAnalysisPassName(),

7824

"NotBeneficial", L)

7825

<< "vectorization is not beneficial "

7826

"and is not explicitly forced");

7827

return false;

7828

}

7829

}

7830

7831

PredicatedScalarEvolution PSE(*SE, *L);

7832

7833

// Check if it is legal to vectorize the loop.

7834

LoopVectorizationRequirements Requirements(*ORE);

7835

LoopVectorizationLegality LVL(L, PSE, DT, TLI, AA, F, TTI, GetLAA, LI, ORE,

7836

&Requirements, &Hints);

7837

if (!LVL.canVectorize()) {

7838

DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Cannot prove legality.\n"
; } } while (false);

7839

emitMissedWarning(F, L, Hints, ORE);

7840

return false;

7841

}

7842

7843

// Check the function attributes to find out if this function should be

7844

// optimized for size.

7845

bool OptForSize =

7846

Hints.getForce() != LoopVectorizeHints::FK_Enabled && F->optForSize();

7847

7848

// Check the function attributes to see if implicit floats are allowed.

7849

// FIXME: This check doesn't seem possibly correct -- what if the loop is

7850

// an integer loop and the vector instructions selected are purely integer

7851

// vector instructions?

7852

if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {

7853

DEBUG(dbgs() << "LV: Can't vectorize when the NoImplicitFloat"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Can't vectorize when the NoImplicitFloat"
"attribute is used.\n"; } } while (false)

7854

"attribute is used.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Can't vectorize when the NoImplicitFloat"
"attribute is used.\n"; } } while (false);

7855

ORE->emit(createMissedAnalysis(Hints.vectorizeAnalysisPassName(),

7856

"NoImplicitFloat", L)

7857

<< "loop not vectorized due to NoImplicitFloat attribute");

7858

emitMissedWarning(F, L, Hints, ORE);

7859

return false;

7860

}

7861

7862

// Check if the target supports potentially unsafe FP vectorization.

7863

// FIXME: Add a check for the type of safety issue (denormal, signaling)

7864

// for the target we're vectorizing for, to make sure none of the

7865

// additional fp-math flags can help.

7866

if (Hints.isPotentiallyUnsafe() &&

7867

TTI->isFPVectorizationPotentiallyUnsafe()) {

7868

DEBUG(dbgs() << "LV: Potentially unsafe FP op prevents vectorization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Potentially unsafe FP op prevents vectorization.\n"
; } } while (false);

7869

ORE->emit(

7870

createMissedAnalysis(Hints.vectorizeAnalysisPassName(), "UnsafeFP", L)

7871

<< "loop not vectorized due to unsafe FP support.");

7872

emitMissedWarning(F, L, Hints, ORE);

7873

return false;

7874

}

7875

7876

// Use the cost model.

7877

LoopVectorizationCostModel CM(L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE, F,

7878

&Hints);

7879

CM.collectValuesToIgnore();

7880

7881

// Use the planner for vectorization.

7882

LoopVectorizationPlanner LVP(L, LI, &LVL, CM);

7883

7884

// Get user vectorization factor.

7885

unsigned UserVF = Hints.getWidth();

7886

7887

// Plan how to best vectorize, return the best VF and its cost.

7888

LoopVectorizationCostModel::VectorizationFactor VF =

7889

LVP.plan(OptForSize, UserVF);

7890

7891

// Select the interleave count.

7892

unsigned IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost);

7893

7894

// Get user interleave count.

7895

unsigned UserIC = Hints.getInterleave();

7896

7897

// Identify the diagnostic messages that should be produced.

7898

std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;

7899

bool VectorizeLoop = true, InterleaveLoop = true;

7900

if (Requirements.doesNotMeet(F, L, Hints)) {

7901

DEBUG(dbgs() << "LV: Not vectorizing: loop did not meet vectorization "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: loop did not meet vectorization "
"requirements.\n"; } } while (false)

7902

"requirements.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: loop did not meet vectorization "
"requirements.\n"; } } while (false);

7903

emitMissedWarning(F, L, Hints, ORE);

7904

return false;

7905

}

7906

7907

if (VF.Width == 1) {

7908

DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Vectorization is possible but not beneficial.\n"
; } } while (false);

7909

VecDiagMsg = std::make_pair(

7910

"VectorizationNotBeneficial",

7911

"the cost-model indicates that vectorization is not beneficial");

7912

VectorizeLoop = false;

7913

}

7914

7915

if (IC == 1 && UserIC <= 1) {

7916

// Tell the user interleaving is not beneficial.

7917

DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving is not beneficial.\n"
; } } while (false);

7918

IntDiagMsg = std::make_pair(

7919

"InterleavingNotBeneficial",

7920

"the cost-model indicates that interleaving is not beneficial");

7921

InterleaveLoop = false;

7922

if (UserIC == 1) {

7923

IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";

7924

IntDiagMsg.second +=

7925

" and is explicitly disabled or interleave count is set to 1";

7926

}

7927

} else if (IC > 1 && UserIC == 1) {

7928

// Tell the user interleaving is beneficial, but it explicitly disabled.

7929

DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving is beneficial but is explicitly disabled."
; } } while (false)

7930

<< "LV: Interleaving is beneficial but is explicitly disabled.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving is beneficial but is explicitly disabled."
; } } while (false);

7931

IntDiagMsg = std::make_pair(

7932

"InterleavingBeneficialButDisabled",

7933

"the cost-model indicates that interleaving is beneficial "

7934

"but is explicitly disabled or interleave count is set to 1");

7935

InterleaveLoop = false;

7936

}

7937

7938

// Override IC if user provided an interleave count.

7939

IC = UserIC > 0 ? UserIC : IC;

7940

7941

// Emit diagnostic messages, if any.

7942

const char *VAPassName = Hints.vectorizeAnalysisPassName();

7943

if (!VectorizeLoop && !InterleaveLoop) {

7944

// Do not vectorize or interleaving the loop.

7945

ORE->emit(OptimizationRemarkMissed(VAPassName, VecDiagMsg.first,

7946

L->getStartLoc(), L->getHeader())

7947

<< VecDiagMsg.second);

7948

ORE->emit(OptimizationRemarkMissed(LV_NAME"loop-vectorize", IntDiagMsg.first,

7949

L->getStartLoc(), L->getHeader())

7950

<< IntDiagMsg.second);

7951

return false;

7952

} else if (!VectorizeLoop && InterleaveLoop) {

7953

DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleave Count is "
<< IC << '\n'; } } while (false);

7954

ORE->emit(OptimizationRemarkAnalysis(VAPassName, VecDiagMsg.first,

7955

L->getStartLoc(), L->getHeader())

7956

<< VecDiagMsg.second);

7957

} else if (VectorizeLoop && !InterleaveLoop) {

7958

DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width << ") in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a vectorizable loop ("
<< VF.Width << ") in " << DebugLocStr <<
'\n'; } } while (false)

7959

<< DebugLocStr << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a vectorizable loop ("
<< VF.Width << ") in " << DebugLocStr <<
'\n'; } } while (false);

7960

ORE->emit(OptimizationRemarkAnalysis(LV_NAME"loop-vectorize", IntDiagMsg.first,

7961

L->getStartLoc(), L->getHeader())

7962

<< IntDiagMsg.second);

7963

} else if (VectorizeLoop && InterleaveLoop) {

7964

7965

7966

7967

}

7968

7969

using namespace ore;

7970

if (!VectorizeLoop) {

7971

assert(IC > 1 && "interleave count should not be 1 or 0")((IC > 1 && "interleave count should not be 1 or 0"
) ? static_cast<void> (0) : __assert_fail ("IC > 1 && \"interleave count should not be 1 or 0\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7971, __PRETTY_FUNCTION__));

7972

// If we decided that it is not legal to vectorize the loop, then

7973

// interleave it.

7974

InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, AC, ORE, IC, &LVL,

7975

&CM);

7976

LVP.executePlan(Unroller);

7977

7978

ORE->emit(OptimizationRemark(LV_NAME"loop-vectorize", "Interleaved", L->getStartLoc(),

7979

L->getHeader())

7980

<< "interleaved loop (interleaved count: "

7981

<< NV("InterleaveCount", IC) << ")");

7982

} else {

7983

// If we decided that it is *legal* to vectorize the loop, then do it.

7984

InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, IC,

7985

&LVL, &CM);

7986

LVP.executePlan(LB);

7987

++LoopsVectorized;

7988

7989

// Add metadata to disable runtime unrolling a scalar loop when there are

7990

// no runtime checks about strides and memory. A scalar loop that is

7991

// rarely used is not worth unrolling.

7992

if (!LB.areSafetyChecksAdded())

7993

AddRuntimeUnrollDisableMetaData(L);

7994

7995

// Report the vectorization decision.

7996

ORE->emit(OptimizationRemark(LV_NAME"loop-vectorize", "Vectorized", L->getStartLoc(),

7997

L->getHeader())

7998

<< "vectorized loop (vectorization width: "

7999

<< NV("VectorizationFactor", VF.Width)

8000

<< ", interleaved count: " << NV("InterleaveCount", IC) << ")");

8001

}

8002

8003

// Mark the loop as already vectorized to avoid vectorizing again.

8004

Hints.setAlreadyVectorized();

8005

8006

DEBUG(verifyFunction(*L->getHeader()->getParent()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { verifyFunction(*L->getHeader()->getParent
()); } } while (false);

8007

return true;

8008

}

8009

8010

bool LoopVectorizePass::runImpl(

8011

Function &F, ScalarEvolution &SE_, LoopInfo &LI_, TargetTransformInfo &TTI_,

8012

DominatorTree &DT_, BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_,

8013

DemandedBits &DB_, AliasAnalysis &AA_, AssumptionCache &AC_,

8014

std::function<const LoopAccessInfo &(Loop &)> &GetLAA_,

8015

OptimizationRemarkEmitter &ORE_) {

8016

8017

SE = &SE_;

8018

LI = &LI_;

8019

TTI = &TTI_;

8020

DT = &DT_;

8021

BFI = &BFI_;

8022

TLI = TLI_;

8023

AA = &AA_;

8024

AC = &AC_;

8025

GetLAA = &GetLAA_;

8026

DB = &DB_;

8027

ORE = &ORE_;

8028

8029

// Don't attempt if

8030

// 1. the target claims to have no vector registers, and

8031

// 2. interleaving won't help ILP.

8032

8033

// The second condition is necessary because, even if the target has no

8034

// vector registers, loop vectorization may still enable scalar

8035

// interleaving.

8036

if (!TTI->getNumberOfRegisters(true) && TTI->getMaxInterleaveFactor(1) < 2)

8037

return false;

8038

8039

bool Changed = false;

8040

8041

// The vectorizer requires loops to be in simplified form.

8042

// Since simplification may add new inner loops, it has to run before the

8043

// legality and profitability checks. This means running the loop vectorizer

8044

// will simplify all loops, regardless of whether anything end up being

8045

// vectorized.

8046

for (auto &L : *LI)

8047

Changed |= simplifyLoop(L, DT, LI, SE, AC, false /* PreserveLCSSA */);

8048

8049

// Build up a worklist of inner-loops to vectorize. This is necessary as

8050

// the act of vectorizing or partially unrolling a loop creates new loops

8051

// and can invalidate iterators across the loops.

8052

SmallVector<Loop *, 8> Worklist;

8053

8054

for (Loop *L : *LI)

8055

addAcyclicInnerLoop(*L, Worklist);

8056

8057

LoopsAnalyzed += Worklist.size();

8058

8059

// Now walk the identified inner loops.

8060

while (!Worklist.empty()) {

8061

Loop *L = Worklist.pop_back_val();

8062

8063

// For the inner loops we actually process, form LCSSA to simplify the

8064

// transform.

8065

Changed |= formLCSSARecursively(*L, *DT, LI, SE);

8066

8067

Changed |= processLoop(L);

8068

}

8069

8070

// Process each loop nest in the function.

8071

return Changed;

8072

8073

}

8074

8075

8076

PreservedAnalyses LoopVectorizePass::run(Function &F,

8077

FunctionAnalysisManager &AM) {

8078

auto &SE = AM.getResult<ScalarEvolutionAnalysis>(F);

8079

auto &LI = AM.getResult<LoopAnalysis>(F);

8080

auto &TTI = AM.getResult<TargetIRAnalysis>(F);

8081

auto &DT = AM.getResult<DominatorTreeAnalysis>(F);

8082

auto &BFI = AM.getResult<BlockFrequencyAnalysis>(F);

8083

auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);

8084

auto &AA = AM.getResult<AAManager>(F);

8085

auto &AC = AM.getResult<AssumptionAnalysis>(F);

8086

auto &DB = AM.getResult<DemandedBitsAnalysis>(F);

8087

auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);

8088

8089

auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();

8090

std::function<const LoopAccessInfo &(Loop &)> GetLAA =

8091

[&](Loop &L) -> const LoopAccessInfo & {

8092

LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI};

8093

return LAM.getResult<LoopAccessAnalysis>(L, AR);

8094

};

8095

bool Changed =

8096

runImpl(F, SE, LI, TTI, DT, BFI, &TLI, DB, AA, AC, GetLAA, ORE);

8097

if (!Changed)

8098

return PreservedAnalyses::all();

8099

PreservedAnalyses PA;

8100

PA.preserve<LoopAnalysis>();

8101

PA.preserve<DominatorTreeAnalysis>();

8102

PA.preserve<BasicAA>();

8103

PA.preserve<GlobalsAA>();

8104

return PA;

8105

}