LLVM  3.7.0
StraightLineStrengthReduce.cpp
Go to the documentation of this file.
1 //===-- StraightLineStrengthReduce.cpp - ------------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements straight-line strength reduction (SLSR). Unlike loop
11 // strength reduction, this algorithm is designed to reduce arithmetic
12 // redundancy in straight-line code instead of loops. It has proven to be
13 // effective in simplifying arithmetic statements derived from an unrolled loop.
14 // It can also simplify the logic of SeparateConstOffsetFromGEP.
15 //
16 // There are many optimizations we can perform in the domain of SLSR. This file
17 // for now contains only an initial step. Specifically, we look for strength
18 // reduction candidates in the following forms:
19 //
20 // Form 1: B + i * S
21 // Form 2: (B + i) * S
22 // Form 3: &B[i * S]
23 //
24 // where S is an integer variable, and i is a constant integer. If we found two
25 // candidates S1 and S2 in the same form and S1 dominates S2, we may rewrite S2
26 // in a simpler way with respect to S1. For example,
27 //
28 // S1: X = B + i * S
29 // S2: Y = B + i' * S => X + (i' - i) * S
30 //
31 // S1: X = (B + i) * S
32 // S2: Y = (B + i') * S => X + (i' - i) * S
33 //
34 // S1: X = &B[i * S]
35 // S2: Y = &B[i' * S] => &X[(i' - i) * S]
36 //
37 // Note: (i' - i) * S is folded to the extent possible.
38 //
39 // This rewriting is in general a good idea. The code patterns we focus on
40 // usually come from loop unrolling, so (i' - i) * S is likely the same
41 // across iterations and can be reused. When that happens, the optimized form
42 // takes only one add starting from the second iteration.
43 //
44 // When such rewriting is possible, we call S1 a "basis" of S2. When S2 has
45 // multiple bases, we choose to rewrite S2 with respect to its "immediate"
46 // basis, the basis that is the closest ancestor in the dominator tree.
47 //
48 // TODO:
49 //
50 // - Floating point arithmetics when fast math is enabled.
51 //
52 // - SLSR may decrease ILP at the architecture level. Targets that are very
53 // sensitive to ILP may want to disable it. Having SLSR to consider ILP is
54 // left as future work.
55 //
56 // - When (i' - i) is constant but i and i' are not, we could still perform
57 // SLSR.
58 #include <vector>
59 
60 #include "llvm/ADT/DenseSet.h"
61 #include "llvm/ADT/FoldingSet.h"
65 #include "llvm/IR/DataLayout.h"
66 #include "llvm/IR/Dominators.h"
67 #include "llvm/IR/IRBuilder.h"
68 #include "llvm/IR/Module.h"
69 #include "llvm/IR/PatternMatch.h"
71 #include "llvm/Transforms/Scalar.h"
73 
74 using namespace llvm;
75 using namespace PatternMatch;
76 
77 namespace {
78 
79 class StraightLineStrengthReduce : public FunctionPass {
80 public:
81  // SLSR candidate. Such a candidate must be in one of the forms described in
82  // the header comments.
83  struct Candidate : public ilist_node<Candidate> {
84  enum Kind {
85  Invalid, // reserved for the default constructor
86  Add, // B + i * S
87  Mul, // (B + i) * S
88  GEP, // &B[..][i * S][..]
89  };
90 
91  Candidate()
92  : CandidateKind(Invalid), Base(nullptr), Index(nullptr),
93  Stride(nullptr), Ins(nullptr), Basis(nullptr) {}
94  Candidate(Kind CT, const SCEV *B, ConstantInt *Idx, Value *S,
95  Instruction *I)
96  : CandidateKind(CT), Base(B), Index(Idx), Stride(S), Ins(I),
97  Basis(nullptr) {}
98  Kind CandidateKind;
99  const SCEV *Base;
100  // Note that Index and Stride of a GEP candidate do not necessarily have the
101  // same integer type. In that case, during rewriting, Stride will be
102  // sign-extended or truncated to Index's type.
103  ConstantInt *Index;
104  Value *Stride;
105  // The instruction this candidate corresponds to. It helps us to rewrite a
106  // candidate with respect to its immediate basis. Note that one instruction
107  // can correspond to multiple candidates depending on how you associate the
108  // expression. For instance,
109  //
110  // (a + 1) * (b + 2)
111  //
112  // can be treated as
113  //
114  // <Base: a, Index: 1, Stride: b + 2>
115  //
116  // or
117  //
118  // <Base: b, Index: 2, Stride: a + 1>
119  Instruction *Ins;
120  // Points to the immediate basis of this candidate, or nullptr if we cannot
121  // find any basis for this candidate.
122  Candidate *Basis;
123  };
124 
125  static char ID;
126 
127  StraightLineStrengthReduce()
128  : FunctionPass(ID), DL(nullptr), DT(nullptr), TTI(nullptr) {
130  }
131 
132  void getAnalysisUsage(AnalysisUsage &AU) const override {
136  // We do not modify the shape of the CFG.
137  AU.setPreservesCFG();
138  }
139 
140  bool doInitialization(Module &M) override {
141  DL = &M.getDataLayout();
142  return false;
143  }
144 
145  bool runOnFunction(Function &F) override;
146 
147 private:
148  // Returns true if Basis is a basis for C, i.e., Basis dominates C and they
149  // share the same base and stride.
150  bool isBasisFor(const Candidate &Basis, const Candidate &C);
151  // Returns whether the candidate can be folded into an addressing mode.
152  bool isFoldable(const Candidate &C, TargetTransformInfo *TTI,
153  const DataLayout *DL);
154  // Returns true if C is already in a simplest form and not worth being
155  // rewritten.
156  bool isSimplestForm(const Candidate &C);
157  // Checks whether I is in a candidate form. If so, adds all the matching forms
158  // to Candidates, and tries to find the immediate basis for each of them.
159  void allocateCandidatesAndFindBasis(Instruction *I);
160  // Allocate candidates and find bases for Add instructions.
161  void allocateCandidatesAndFindBasisForAdd(Instruction *I);
162  // Given I = LHS + RHS, factors RHS into i * S and makes (LHS + i * S) a
163  // candidate.
164  void allocateCandidatesAndFindBasisForAdd(Value *LHS, Value *RHS,
165  Instruction *I);
166  // Allocate candidates and find bases for Mul instructions.
167  void allocateCandidatesAndFindBasisForMul(Instruction *I);
168  // Splits LHS into Base + Index and, if succeeds, calls
169  // allocateCandidatesAndFindBasis.
170  void allocateCandidatesAndFindBasisForMul(Value *LHS, Value *RHS,
171  Instruction *I);
172  // Allocate candidates and find bases for GetElementPtr instructions.
173  void allocateCandidatesAndFindBasisForGEP(GetElementPtrInst *GEP);
174  // A helper function that scales Idx with ElementSize before invoking
175  // allocateCandidatesAndFindBasis.
176  void allocateCandidatesAndFindBasisForGEP(const SCEV *B, ConstantInt *Idx,
177  Value *S, uint64_t ElementSize,
178  Instruction *I);
179  // Adds the given form <CT, B, Idx, S> to Candidates, and finds its immediate
180  // basis.
181  void allocateCandidatesAndFindBasis(Candidate::Kind CT, const SCEV *B,
182  ConstantInt *Idx, Value *S,
183  Instruction *I);
184  // Rewrites candidate C with respect to Basis.
185  void rewriteCandidateWithBasis(const Candidate &C, const Candidate &Basis);
186  // A helper function that factors ArrayIdx to a product of a stride and a
187  // constant index, and invokes allocateCandidatesAndFindBasis with the
188  // factorings.
189  void factorArrayIndex(Value *ArrayIdx, const SCEV *Base, uint64_t ElementSize,
191  // Emit code that computes the "bump" from Basis to C. If the candidate is a
192  // GEP and the bump is not divisible by the element size of the GEP, this
193  // function sets the BumpWithUglyGEP flag to notify its caller to bump the
194  // basis using an ugly GEP.
195  static Value *emitBump(const Candidate &Basis, const Candidate &C,
196  IRBuilder<> &Builder, const DataLayout *DL,
197  bool &BumpWithUglyGEP);
198 
199  const DataLayout *DL;
200  DominatorTree *DT;
201  ScalarEvolution *SE;
202  TargetTransformInfo *TTI;
203  ilist<Candidate> Candidates;
204  // Temporarily holds all instructions that are unlinked (but not deleted) by
205  // rewriteCandidateWithBasis. These instructions will be actually removed
206  // after all rewriting finishes.
207  std::vector<Instruction *> UnlinkedInstructions;
208 };
209 } // anonymous namespace
210 
212 INITIALIZE_PASS_BEGIN(StraightLineStrengthReduce, "slsr",
213  "Straight line strength reduction", false, false)
217 INITIALIZE_PASS_END(StraightLineStrengthReduce, "slsr",
218  "Straight line strength reduction", false, false)
219 
221  return new StraightLineStrengthReduce();
222 }
223 
224 bool StraightLineStrengthReduce::isBasisFor(const Candidate &Basis,
225  const Candidate &C) {
226  return (Basis.Ins != C.Ins && // skip the same instruction
227  // They must have the same type too. Basis.Base == C.Base doesn't
228  // guarantee their types are the same (PR23975).
229  Basis.Ins->getType() == C.Ins->getType() &&
230  // Basis must dominate C in order to rewrite C with respect to Basis.
231  DT->dominates(Basis.Ins->getParent(), C.Ins->getParent()) &&
232  // They share the same base, stride, and candidate kind.
233  Basis.Base == C.Base && Basis.Stride == C.Stride &&
234  Basis.CandidateKind == C.CandidateKind);
235 }
236 
238  const TargetTransformInfo *TTI,
239  const DataLayout *DL) {
240  GlobalVariable *BaseGV = nullptr;
241  int64_t BaseOffset = 0;
242  bool HasBaseReg = false;
243  int64_t Scale = 0;
244 
245  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getPointerOperand()))
246  BaseGV = GV;
247  else
248  HasBaseReg = true;
249 
251  for (auto I = GEP->idx_begin(); I != GEP->idx_end(); ++I, ++GTI) {
252  if (isa<SequentialType>(*GTI)) {
253  int64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
254  if (ConstantInt *ConstIdx = dyn_cast<ConstantInt>(*I)) {
255  BaseOffset += ConstIdx->getSExtValue() * ElementSize;
256  } else {
257  // Needs scale register.
258  if (Scale != 0) {
259  // No addressing mode takes two scale registers.
260  return false;
261  }
262  Scale = ElementSize;
263  }
264  } else {
265  StructType *STy = cast<StructType>(*GTI);
266  uint64_t Field = cast<ConstantInt>(*I)->getZExtValue();
267  BaseOffset += DL->getStructLayout(STy)->getElementOffset(Field);
268  }
269  }
270 
271  unsigned AddrSpace = GEP->getPointerAddressSpace();
272  return TTI->isLegalAddressingMode(GEP->getType()->getElementType(), BaseGV,
273  BaseOffset, HasBaseReg, Scale, AddrSpace);
274 }
275 
276 // Returns whether (Base + Index * Stride) can be folded to an addressing mode.
277 static bool isAddFoldable(const SCEV *Base, ConstantInt *Index, Value *Stride,
278  TargetTransformInfo *TTI) {
279  return TTI->isLegalAddressingMode(Base->getType(), nullptr, 0, true,
280  Index->getSExtValue());
281 }
282 
283 bool StraightLineStrengthReduce::isFoldable(const Candidate &C,
284  TargetTransformInfo *TTI,
285  const DataLayout *DL) {
286  if (C.CandidateKind == Candidate::Add)
287  return isAddFoldable(C.Base, C.Index, C.Stride, TTI);
288  if (C.CandidateKind == Candidate::GEP)
289  return isGEPFoldable(cast<GetElementPtrInst>(C.Ins), TTI, DL);
290  return false;
291 }
292 
293 // Returns true if GEP has zero or one non-zero index.
295  unsigned NumNonZeroIndices = 0;
296  for (auto I = GEP->idx_begin(); I != GEP->idx_end(); ++I) {
297  ConstantInt *ConstIdx = dyn_cast<ConstantInt>(*I);
298  if (ConstIdx == nullptr || !ConstIdx->isZero())
299  ++NumNonZeroIndices;
300  }
301  return NumNonZeroIndices <= 1;
302 }
303 
304 bool StraightLineStrengthReduce::isSimplestForm(const Candidate &C) {
305  if (C.CandidateKind == Candidate::Add) {
306  // B + 1 * S or B + (-1) * S
307  return C.Index->isOne() || C.Index->isMinusOne();
308  }
309  if (C.CandidateKind == Candidate::Mul) {
310  // (B + 0) * S
311  return C.Index->isZero();
312  }
313  if (C.CandidateKind == Candidate::GEP) {
314  // (char*)B + S or (char*)B - S
315  return ((C.Index->isOne() || C.Index->isMinusOne()) &&
316  hasOnlyOneNonZeroIndex(cast<GetElementPtrInst>(C.Ins)));
317  }
318  return false;
319 }
320 
321 // TODO: We currently implement an algorithm whose time complexity is linear in
322 // the number of existing candidates. However, we could do better by using
323 // ScopedHashTable. Specifically, while traversing the dominator tree, we could
324 // maintain all the candidates that dominate the basic block being traversed in
325 // a ScopedHashTable. This hash table is indexed by the base and the stride of
326 // a candidate. Therefore, finding the immediate basis of a candidate boils down
327 // to one hash-table look up.
328 void StraightLineStrengthReduce::allocateCandidatesAndFindBasis(
329  Candidate::Kind CT, const SCEV *B, ConstantInt *Idx, Value *S,
330  Instruction *I) {
331  Candidate C(CT, B, Idx, S, I);
332  // SLSR can complicate an instruction in two cases:
333  //
334  // 1. If we can fold I into an addressing mode, computing I is likely free or
335  // takes only one instruction.
336  //
337  // 2. I is already in a simplest form. For example, when
338  // X = B + 8 * S
339  // Y = B + S,
340  // rewriting Y to X - 7 * S is probably a bad idea.
341  //
342  // In the above cases, we still add I to the candidate list so that I can be
343  // the basis of other candidates, but we leave I's basis blank so that I
344  // won't be rewritten.
345  if (!isFoldable(C, TTI, DL) && !isSimplestForm(C)) {
346  // Try to compute the immediate basis of C.
347  unsigned NumIterations = 0;
348  // Limit the scan radius to avoid running in quadratice time.
349  static const unsigned MaxNumIterations = 50;
350  for (auto Basis = Candidates.rbegin();
351  Basis != Candidates.rend() && NumIterations < MaxNumIterations;
352  ++Basis, ++NumIterations) {
353  if (isBasisFor(*Basis, C)) {
354  C.Basis = &(*Basis);
355  break;
356  }
357  }
358  }
359  // Regardless of whether we find a basis for C, we need to push C to the
360  // candidate list so that it can be the basis of other candidates.
361  Candidates.push_back(C);
362 }
363 
364 void StraightLineStrengthReduce::allocateCandidatesAndFindBasis(
365  Instruction *I) {
366  switch (I->getOpcode()) {
367  case Instruction::Add:
368  allocateCandidatesAndFindBasisForAdd(I);
369  break;
370  case Instruction::Mul:
371  allocateCandidatesAndFindBasisForMul(I);
372  break;
373  case Instruction::GetElementPtr:
374  allocateCandidatesAndFindBasisForGEP(cast<GetElementPtrInst>(I));
375  break;
376  }
377 }
378 
379 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForAdd(
380  Instruction *I) {
381  // Try matching B + i * S.
382  if (!isa<IntegerType>(I->getType()))
383  return;
384 
385  assert(I->getNumOperands() == 2 && "isn't I an add?");
386  Value *LHS = I->getOperand(0), *RHS = I->getOperand(1);
387  allocateCandidatesAndFindBasisForAdd(LHS, RHS, I);
388  if (LHS != RHS)
389  allocateCandidatesAndFindBasisForAdd(RHS, LHS, I);
390 }
391 
392 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForAdd(
393  Value *LHS, Value *RHS, Instruction *I) {
394  Value *S = nullptr;
395  ConstantInt *Idx = nullptr;
396  if (match(RHS, m_Mul(m_Value(S), m_ConstantInt(Idx)))) {
397  // I = LHS + RHS = LHS + Idx * S
398  allocateCandidatesAndFindBasis(Candidate::Add, SE->getSCEV(LHS), Idx, S, I);
399  } else if (match(RHS, m_Shl(m_Value(S), m_ConstantInt(Idx)))) {
400  // I = LHS + RHS = LHS + (S << Idx) = LHS + S * (1 << Idx)
401  APInt One(Idx->getBitWidth(), 1);
402  Idx = ConstantInt::get(Idx->getContext(), One << Idx->getValue());
403  allocateCandidatesAndFindBasis(Candidate::Add, SE->getSCEV(LHS), Idx, S, I);
404  } else {
405  // At least, I = LHS + 1 * RHS
406  ConstantInt *One = ConstantInt::get(cast<IntegerType>(I->getType()), 1);
407  allocateCandidatesAndFindBasis(Candidate::Add, SE->getSCEV(LHS), One, RHS,
408  I);
409  }
410 }
411 
412 // Returns true if A matches B + C where C is constant.
413 static bool matchesAdd(Value *A, Value *&B, ConstantInt *&C) {
414  return (match(A, m_Add(m_Value(B), m_ConstantInt(C))) ||
415  match(A, m_Add(m_ConstantInt(C), m_Value(B))));
416 }
417 
418 // Returns true if A matches B | C where C is constant.
419 static bool matchesOr(Value *A, Value *&B, ConstantInt *&C) {
420  return (match(A, m_Or(m_Value(B), m_ConstantInt(C))) ||
421  match(A, m_Or(m_ConstantInt(C), m_Value(B))));
422 }
423 
424 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForMul(
425  Value *LHS, Value *RHS, Instruction *I) {
426  Value *B = nullptr;
427  ConstantInt *Idx = nullptr;
428  if (matchesAdd(LHS, B, Idx)) {
429  // If LHS is in the form of "Base + Index", then I is in the form of
430  // "(Base + Index) * RHS".
431  allocateCandidatesAndFindBasis(Candidate::Mul, SE->getSCEV(B), Idx, RHS, I);
432  } else if (matchesOr(LHS, B, Idx) && haveNoCommonBitsSet(B, Idx, *DL)) {
433  // If LHS is in the form of "Base | Index" and Base and Index have no common
434  // bits set, then
435  // Base | Index = Base + Index
436  // and I is thus in the form of "(Base + Index) * RHS".
437  allocateCandidatesAndFindBasis(Candidate::Mul, SE->getSCEV(B), Idx, RHS, I);
438  } else {
439  // Otherwise, at least try the form (LHS + 0) * RHS.
440  ConstantInt *Zero = ConstantInt::get(cast<IntegerType>(I->getType()), 0);
441  allocateCandidatesAndFindBasis(Candidate::Mul, SE->getSCEV(LHS), Zero, RHS,
442  I);
443  }
444 }
445 
446 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForMul(
447  Instruction *I) {
448  // Try matching (B + i) * S.
449  // TODO: we could extend SLSR to float and vector types.
450  if (!isa<IntegerType>(I->getType()))
451  return;
452 
453  assert(I->getNumOperands() == 2 && "isn't I a mul?");
454  Value *LHS = I->getOperand(0), *RHS = I->getOperand(1);
455  allocateCandidatesAndFindBasisForMul(LHS, RHS, I);
456  if (LHS != RHS) {
457  // Symmetrically, try to split RHS to Base + Index.
458  allocateCandidatesAndFindBasisForMul(RHS, LHS, I);
459  }
460 }
461 
462 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
463  const SCEV *B, ConstantInt *Idx, Value *S, uint64_t ElementSize,
464  Instruction *I) {
465  // I = B + sext(Idx *nsw S) * ElementSize
466  // = B + (sext(Idx) * sext(S)) * ElementSize
467  // = B + (sext(Idx) * ElementSize) * sext(S)
468  // Casting to IntegerType is safe because we skipped vector GEPs.
469  IntegerType *IntPtrTy = cast<IntegerType>(DL->getIntPtrType(I->getType()));
470  ConstantInt *ScaledIdx = ConstantInt::get(
471  IntPtrTy, Idx->getSExtValue() * (int64_t)ElementSize, true);
472  allocateCandidatesAndFindBasis(Candidate::GEP, B, ScaledIdx, S, I);
473 }
474 
475 void StraightLineStrengthReduce::factorArrayIndex(Value *ArrayIdx,
476  const SCEV *Base,
477  uint64_t ElementSize,
479  // At least, ArrayIdx = ArrayIdx *nsw 1.
480  allocateCandidatesAndFindBasisForGEP(
481  Base, ConstantInt::get(cast<IntegerType>(ArrayIdx->getType()), 1),
482  ArrayIdx, ElementSize, GEP);
483  Value *LHS = nullptr;
484  ConstantInt *RHS = nullptr;
485  // One alternative is matching the SCEV of ArrayIdx instead of ArrayIdx
486  // itself. This would allow us to handle the shl case for free. However,
487  // matching SCEVs has two issues:
488  //
489  // 1. this would complicate rewriting because the rewriting procedure
490  // would have to translate SCEVs back to IR instructions. This translation
491  // is difficult when LHS is further evaluated to a composite SCEV.
492  //
493  // 2. ScalarEvolution is designed to be control-flow oblivious. It tends
494  // to strip nsw/nuw flags which are critical for SLSR to trace into
495  // sext'ed multiplication.
496  if (match(ArrayIdx, m_NSWMul(m_Value(LHS), m_ConstantInt(RHS)))) {
497  // SLSR is currently unsafe if i * S may overflow.
498  // GEP = Base + sext(LHS *nsw RHS) * ElementSize
499  allocateCandidatesAndFindBasisForGEP(Base, RHS, LHS, ElementSize, GEP);
500  } else if (match(ArrayIdx, m_NSWShl(m_Value(LHS), m_ConstantInt(RHS)))) {
501  // GEP = Base + sext(LHS <<nsw RHS) * ElementSize
502  // = Base + sext(LHS *nsw (1 << RHS)) * ElementSize
503  APInt One(RHS->getBitWidth(), 1);
504  ConstantInt *PowerOf2 =
505  ConstantInt::get(RHS->getContext(), One << RHS->getValue());
506  allocateCandidatesAndFindBasisForGEP(Base, PowerOf2, LHS, ElementSize, GEP);
507  }
508 }
509 
510 void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
511  GetElementPtrInst *GEP) {
512  // TODO: handle vector GEPs
513  if (GEP->getType()->isVectorTy())
514  return;
515 
516  SmallVector<const SCEV *, 4> IndexExprs;
517  for (auto I = GEP->idx_begin(); I != GEP->idx_end(); ++I)
518  IndexExprs.push_back(SE->getSCEV(*I));
519 
521  for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I) {
522  if (!isa<SequentialType>(*GTI++))
523  continue;
524 
525  const SCEV *OrigIndexExpr = IndexExprs[I - 1];
526  IndexExprs[I - 1] = SE->getConstant(OrigIndexExpr->getType(), 0);
527 
528  // The base of this candidate is GEP's base plus the offsets of all
529  // indices except this current one.
530  const SCEV *BaseExpr = SE->getGEPExpr(GEP->getSourceElementType(),
531  SE->getSCEV(GEP->getPointerOperand()),
532  IndexExprs, GEP->isInBounds());
533  Value *ArrayIdx = GEP->getOperand(I);
534  uint64_t ElementSize = DL->getTypeAllocSize(*GTI);
535  factorArrayIndex(ArrayIdx, BaseExpr, ElementSize, GEP);
536  // When ArrayIdx is the sext of a value, we try to factor that value as
537  // well. Handling this case is important because array indices are
538  // typically sign-extended to the pointer size.
539  Value *TruncatedArrayIdx = nullptr;
540  if (match(ArrayIdx, m_SExt(m_Value(TruncatedArrayIdx))))
541  factorArrayIndex(TruncatedArrayIdx, BaseExpr, ElementSize, GEP);
542 
543  IndexExprs[I - 1] = OrigIndexExpr;
544  }
545 }
546 
547 // A helper function that unifies the bitwidth of A and B.
548 static void unifyBitWidth(APInt &A, APInt &B) {
549  if (A.getBitWidth() < B.getBitWidth())
550  A = A.sext(B.getBitWidth());
551  else if (A.getBitWidth() > B.getBitWidth())
552  B = B.sext(A.getBitWidth());
553 }
554 
555 Value *StraightLineStrengthReduce::emitBump(const Candidate &Basis,
556  const Candidate &C,
557  IRBuilder<> &Builder,
558  const DataLayout *DL,
559  bool &BumpWithUglyGEP) {
560  APInt Idx = C.Index->getValue(), BasisIdx = Basis.Index->getValue();
561  unifyBitWidth(Idx, BasisIdx);
562  APInt IndexOffset = Idx - BasisIdx;
563 
564  BumpWithUglyGEP = false;
565  if (Basis.CandidateKind == Candidate::GEP) {
566  APInt ElementSize(
567  IndexOffset.getBitWidth(),
568  DL->getTypeAllocSize(
569  cast<GetElementPtrInst>(Basis.Ins)->getType()->getElementType()));
570  APInt Q, R;
571  APInt::sdivrem(IndexOffset, ElementSize, Q, R);
572  if (R.getSExtValue() == 0)
573  IndexOffset = Q;
574  else
575  BumpWithUglyGEP = true;
576  }
577 
578  // Compute Bump = C - Basis = (i' - i) * S.
579  // Common case 1: if (i' - i) is 1, Bump = S.
580  if (IndexOffset.getSExtValue() == 1)
581  return C.Stride;
582  // Common case 2: if (i' - i) is -1, Bump = -S.
583  if (IndexOffset.getSExtValue() == -1)
584  return Builder.CreateNeg(C.Stride);
585 
586  // Otherwise, Bump = (i' - i) * sext/trunc(S). Note that (i' - i) and S may
587  // have different bit widths.
588  IntegerType *DeltaType =
589  IntegerType::get(Basis.Ins->getContext(), IndexOffset.getBitWidth());
590  Value *ExtendedStride = Builder.CreateSExtOrTrunc(C.Stride, DeltaType);
591  if (IndexOffset.isPowerOf2()) {
592  // If (i' - i) is a power of 2, Bump = sext/trunc(S) << log(i' - i).
593  ConstantInt *Exponent = ConstantInt::get(DeltaType, IndexOffset.logBase2());
594  return Builder.CreateShl(ExtendedStride, Exponent);
595  }
596  if ((-IndexOffset).isPowerOf2()) {
597  // If (i - i') is a power of 2, Bump = -sext/trunc(S) << log(i' - i).
598  ConstantInt *Exponent =
599  ConstantInt::get(DeltaType, (-IndexOffset).logBase2());
600  return Builder.CreateNeg(Builder.CreateShl(ExtendedStride, Exponent));
601  }
602  Constant *Delta = ConstantInt::get(DeltaType, IndexOffset);
603  return Builder.CreateMul(ExtendedStride, Delta);
604 }
605 
606 void StraightLineStrengthReduce::rewriteCandidateWithBasis(
607  const Candidate &C, const Candidate &Basis) {
608  assert(C.CandidateKind == Basis.CandidateKind && C.Base == Basis.Base &&
609  C.Stride == Basis.Stride);
610  // We run rewriteCandidateWithBasis on all candidates in a post-order, so the
611  // basis of a candidate cannot be unlinked before the candidate.
612  assert(Basis.Ins->getParent() != nullptr && "the basis is unlinked");
613 
614  // An instruction can correspond to multiple candidates. Therefore, instead of
615  // simply deleting an instruction when we rewrite it, we mark its parent as
616  // nullptr (i.e. unlink it) so that we can skip the candidates whose
617  // instruction is already rewritten.
618  if (!C.Ins->getParent())
619  return;
620 
621  IRBuilder<> Builder(C.Ins);
622  bool BumpWithUglyGEP;
623  Value *Bump = emitBump(Basis, C, Builder, DL, BumpWithUglyGEP);
624  Value *Reduced = nullptr; // equivalent to but weaker than C.Ins
625  switch (C.CandidateKind) {
626  case Candidate::Add:
627  case Candidate::Mul:
628  // C = Basis + Bump
629  if (BinaryOperator::isNeg(Bump)) {
630  // If Bump is a neg instruction, emit C = Basis - (-Bump).
631  Reduced =
632  Builder.CreateSub(Basis.Ins, BinaryOperator::getNegArgument(Bump));
633  // We only use the negative argument of Bump, and Bump itself may be
634  // trivially dead.
636  } else {
637  // It's tempting to preserve nsw on Bump and/or Reduced. However, it's
638  // usually unsound, e.g.,
639  //
640  // X = (-2 +nsw 1) *nsw INT_MAX
641  // Y = (-2 +nsw 3) *nsw INT_MAX
642  // =>
643  // Y = X + 2 * INT_MAX
644  //
645  // Neither + and * in the resultant expression are nsw.
646  Reduced = Builder.CreateAdd(Basis.Ins, Bump);
647  }
648  break;
649  case Candidate::GEP:
650  {
651  Type *IntPtrTy = DL->getIntPtrType(C.Ins->getType());
652  bool InBounds = cast<GetElementPtrInst>(C.Ins)->isInBounds();
653  if (BumpWithUglyGEP) {
654  // C = (char *)Basis + Bump
655  unsigned AS = Basis.Ins->getType()->getPointerAddressSpace();
656  Type *CharTy = Type::getInt8PtrTy(Basis.Ins->getContext(), AS);
657  Reduced = Builder.CreateBitCast(Basis.Ins, CharTy);
658  if (InBounds)
659  Reduced =
660  Builder.CreateInBoundsGEP(Builder.getInt8Ty(), Reduced, Bump);
661  else
662  Reduced = Builder.CreateGEP(Builder.getInt8Ty(), Reduced, Bump);
663  Reduced = Builder.CreateBitCast(Reduced, C.Ins->getType());
664  } else {
665  // C = gep Basis, Bump
666  // Canonicalize bump to pointer size.
667  Bump = Builder.CreateSExtOrTrunc(Bump, IntPtrTy);
668  if (InBounds)
669  Reduced = Builder.CreateInBoundsGEP(nullptr, Basis.Ins, Bump);
670  else
671  Reduced = Builder.CreateGEP(nullptr, Basis.Ins, Bump);
672  }
673  }
674  break;
675  default:
676  llvm_unreachable("C.CandidateKind is invalid");
677  };
678  Reduced->takeName(C.Ins);
679  C.Ins->replaceAllUsesWith(Reduced);
680  // Unlink C.Ins so that we can skip other candidates also corresponding to
681  // C.Ins. The actual deletion is postponed to the end of runOnFunction.
682  C.Ins->removeFromParent();
683  UnlinkedInstructions.push_back(C.Ins);
684 }
685 
686 bool StraightLineStrengthReduce::runOnFunction(Function &F) {
687  if (skipOptnoneFunction(F))
688  return false;
689 
690  TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
691  DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
692  SE = &getAnalysis<ScalarEvolution>();
693  // Traverse the dominator tree in the depth-first order. This order makes sure
694  // all bases of a candidate are in Candidates when we process it.
695  for (auto node = GraphTraits<DominatorTree *>::nodes_begin(DT);
696  node != GraphTraits<DominatorTree *>::nodes_end(DT); ++node) {
697  for (auto &I : *node->getBlock())
698  allocateCandidatesAndFindBasis(&I);
699  }
700 
701  // Rewrite candidates in the reverse depth-first order. This order makes sure
702  // a candidate being rewritten is not a basis for any other candidate.
703  while (!Candidates.empty()) {
704  const Candidate &C = Candidates.back();
705  if (C.Basis != nullptr) {
706  rewriteCandidateWithBasis(C, *C.Basis);
707  }
708  Candidates.pop_back();
709  }
710 
711  // Delete all unlink instructions.
712  for (auto *UnlinkedInst : UnlinkedInstructions) {
713  for (unsigned I = 0, E = UnlinkedInst->getNumOperands(); I != E; ++I) {
714  Value *Op = UnlinkedInst->getOperand(I);
715  UnlinkedInst->setOperand(I, nullptr);
717  }
718  delete UnlinkedInst;
719  }
720  bool Ret = !UnlinkedInstructions.empty();
721  UnlinkedInstructions.clear();
722  return Ret;
723 }
FunctionPass * createStraightLineStrengthReducePass()
Value * CreateGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Definition: IRBuilder.h:1032
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:64
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Value * CreateSExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a SExt or Trunc from the integer value V to DestTy.
Definition: IRBuilder.h:1214
Type * getSourceElementType() const
Definition: Instructions.h:926
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:114
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: GraphTraits.h:27
unsigned getBitWidth() const
getBitWidth - Return the bitwidth of this constant.
Definition: Constants.h:111
unsigned getNumOperands() const
Definition: User.h:138
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Definition: Instructions.h:976
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
Definition: PatternMatch.h:458
ScalarEvolution - This class is the main scalar evolution driver.
static void unifyBitWidth(APInt &A, APInt &B)
bool haveNoCommonBitsSet(Value *LHS, Value *RHS, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Returns true if LHS and RHS have no common bits set.
static void sdivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Definition: APInt.cpp:1987
static bool matchesAdd(Value *A, Value *&B, ConstantInt *&C)
F(f)
Hexagon Common GEP
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:726
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:41
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:70
static const Value * getNegArgument(const Value *BinOp)
Helper functions to extract the unary argument of a NEG, FNEG or NOT operation implemented via Sub...
static bool matchesOr(Value *A, Value *&B, ConstantInt *&C)
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:551
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:106
StructType - Class to represent struct types.
Definition: DerivedTypes.h:191
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:75
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:517
unsigned logBase2(const APInt &APIVal)
Returns the floor log base 2 of the specified APInt value.
Definition: APInt.h:1782
static bool hasOnlyOneNonZeroIndex(GetElementPtrInst *GEP)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
Definition: PatternMatch.h:434
void initializeStraightLineStrengthReducePass(PassRegistry &)
op_iterator idx_begin()
Definition: Instructions.h:954
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
Definition: PatternMatch.h:75
INITIALIZE_PASS_BEGIN(StraightLineStrengthReduce,"slsr","Straight line strength reduction", false, false) INITIALIZE_PASS_END(StraightLineStrengthReduce
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:67
Type * getElementType() const
Definition: DerivedTypes.h:323
bool isInBounds() const
isInBounds - Determine whether the GEP has the inbounds flag.
uint64_t getElementOffset(unsigned Idx) const
Definition: DataLayout.h:491
GetElementPtrInst - an instruction for type-safe pointer arithmetic to access elements of arrays and ...
Definition: Instructions.h:830
bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace=0) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
Wrapper pass for TargetTransformInfo.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
Definition: PatternMatch.h:512
static volatile int One
Definition: InfiniteTest.cpp:9
Type * getType() const
getType - Return the LLVM type of this SCEV expression.
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
Definition: Type.h:226
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:704
This is an important base class in LLVM.
Definition: Constant.h:41
Straight line strength reduction
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1339
APInt LLVM_ATTRIBUTE_UNUSED_RESULT sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:955
Represent the analysis usage information of a pass.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
Definition: PatternMatch.h:524
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1273
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:294
Value * getOperand(unsigned i) const
Definition: User.h:118
Class to represent integer types.
Definition: DerivedTypes.h:37
Value * CreateInBoundsGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Definition: IRBuilder.h:1049
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr)
RecursivelyDeleteTriviallyDeadInstructions - If the specified value is a trivially dead instruction...
Definition: Local.cpp:340
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:519
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:283
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:386
CastClass_match< OpTy, Instruction::SExt > m_SExt(const OpTy &Op)
Matches SExt.
Definition: PatternMatch.h:807
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
Definition: DataLayout.cpp:694
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:304
static bool isGEPFoldable(GetElementPtrInst *GEP, const TargetTransformInfo *TTI, const DataLayout *DL)
This is the shared class of boolean and integer constants.
Definition: Constants.h:47
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:388
unsigned logBase2() const
Definition: APInt.h:1521
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1253
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
SequentialType * getType() const
Definition: Instructions.h:922
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:748
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:582
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:161
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:263
Straight line strength false
static bool isNeg(const Value *V)
Check if the given Value is a NEG, FNeg, or NOT instruction.
Class for arbitrary precision integers.
Definition: APInt.h:73
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:296
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:823
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:372
SCEV - This class represents an analyzed expression in the program.
ilist_node - Base class that provides next/prev services for nodes that use ilist_nextprev_traits or ...
Definition: ilist_node.h:43
#define I(x, y, z)
Definition: MD5.cpp:54
OverflowingBinaryOp_match< LHS, RHS, Instruction::Shl, OverflowingBinaryOperator::NoSignedWrap > m_NSWShl(const LHS &L, const RHS &R)
Definition: PatternMatch.h:593
const ARM::ArchExtKind Kind
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:935
LLVM Value Representation.
Definition: Value.h:69
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:112
OverflowingBinaryOp_match< LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoSignedWrap > m_NSWMul(const LHS &L, const RHS &R)
Definition: PatternMatch.h:585
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:203
This pass exposes codegen information to IR-level passes.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:125
static bool isAddFoldable(const SCEV *Base, ConstantInt *Index, Value *Stride, TargetTransformInfo *TTI)
gep_type_iterator gep_type_begin(const User *GEP)