LLVM  3.7.0
Reassociate.cpp
Go to the documentation of this file.
1 //===- Reassociate.cpp - Reassociate binary expressions -------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass reassociates commutative expressions in an order that is designed
11 // to promote better constant propagation, GCSE, LICM, PRE, etc.
12 //
13 // For example: 4 + (x + 5) -> x + (4 + 5)
14 //
15 // In the implementation of this algorithm, constants are assigned rank = 0,
16 // function arguments are rank = 1, and other values are assigned ranks
17 // corresponding to the reverse post order traversal of current function
18 // (starting at 2), which effectively gives values in deep loops higher rank
19 // than values not in loops.
20 //
21 //===----------------------------------------------------------------------===//
22 
23 #include "llvm/Transforms/Scalar.h"
24 #include "llvm/ADT/DenseMap.h"
26 #include "llvm/ADT/STLExtras.h"
27 #include "llvm/ADT/SetVector.h"
28 #include "llvm/ADT/Statistic.h"
29 #include "llvm/IR/CFG.h"
30 #include "llvm/IR/Constants.h"
31 #include "llvm/IR/DerivedTypes.h"
32 #include "llvm/IR/Function.h"
33 #include "llvm/IR/IRBuilder.h"
34 #include "llvm/IR/Instructions.h"
35 #include "llvm/IR/IntrinsicInst.h"
36 #include "llvm/IR/ValueHandle.h"
37 #include "llvm/Pass.h"
38 #include "llvm/Support/Debug.h"
41 #include <algorithm>
42 using namespace llvm;
43 
44 #define DEBUG_TYPE "reassociate"
45 
46 STATISTIC(NumChanged, "Number of insts reassociated");
47 STATISTIC(NumAnnihil, "Number of expr tree annihilated");
48 STATISTIC(NumFactor , "Number of multiplies factored");
49 
50 namespace {
51  struct ValueEntry {
52  unsigned Rank;
53  Value *Op;
54  ValueEntry(unsigned R, Value *O) : Rank(R), Op(O) {}
55  };
56  inline bool operator<(const ValueEntry &LHS, const ValueEntry &RHS) {
57  return LHS.Rank > RHS.Rank; // Sort so that highest rank goes to start.
58  }
59 }
60 
61 #ifndef NDEBUG
62 /// Print out the expression identified in the Ops list.
63 ///
65  Module *M = I->getParent()->getParent()->getParent();
66  dbgs() << Instruction::getOpcodeName(I->getOpcode()) << " "
67  << *Ops[0].Op->getType() << '\t';
68  for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
69  dbgs() << "[ ";
70  Ops[i].Op->printAsOperand(dbgs(), false, M);
71  dbgs() << ", #" << Ops[i].Rank << "] ";
72  }
73 }
74 #endif
75 
76 namespace {
77  /// \brief Utility class representing a base and exponent pair which form one
78  /// factor of some product.
79  struct Factor {
80  Value *Base;
81  unsigned Power;
82 
83  Factor(Value *Base, unsigned Power) : Base(Base), Power(Power) {}
84 
85  /// \brief Sort factors by their Base.
86  struct BaseSorter {
87  bool operator()(const Factor &LHS, const Factor &RHS) {
88  return LHS.Base < RHS.Base;
89  }
90  };
91 
92  /// \brief Compare factors for equal bases.
93  struct BaseEqual {
94  bool operator()(const Factor &LHS, const Factor &RHS) {
95  return LHS.Base == RHS.Base;
96  }
97  };
98 
99  /// \brief Sort factors in descending order by their power.
100  struct PowerDescendingSorter {
101  bool operator()(const Factor &LHS, const Factor &RHS) {
102  return LHS.Power > RHS.Power;
103  }
104  };
105 
106  /// \brief Compare factors for equal powers.
107  struct PowerEqual {
108  bool operator()(const Factor &LHS, const Factor &RHS) {
109  return LHS.Power == RHS.Power;
110  }
111  };
112  };
113 
114  /// Utility class representing a non-constant Xor-operand. We classify
115  /// non-constant Xor-Operands into two categories:
116  /// C1) The operand is in the form "X & C", where C is a constant and C != ~0
117  /// C2)
118  /// C2.1) The operand is in the form of "X | C", where C is a non-zero
119  /// constant.
120  /// C2.2) Any operand E which doesn't fall into C1 and C2.1, we view this
121  /// operand as "E | 0"
122  class XorOpnd {
123  public:
124  XorOpnd(Value *V);
125 
126  bool isInvalid() const { return SymbolicPart == nullptr; }
127  bool isOrExpr() const { return isOr; }
128  Value *getValue() const { return OrigVal; }
129  Value *getSymbolicPart() const { return SymbolicPart; }
130  unsigned getSymbolicRank() const { return SymbolicRank; }
131  const APInt &getConstPart() const { return ConstPart; }
132 
133  void Invalidate() { SymbolicPart = OrigVal = nullptr; }
134  void setSymbolicRank(unsigned R) { SymbolicRank = R; }
135 
136  // Sort the XorOpnd-Pointer in ascending order of symbolic-value-rank.
137  // The purpose is twofold:
138  // 1) Cluster together the operands sharing the same symbolic-value.
139  // 2) Operand having smaller symbolic-value-rank is permuted earlier, which
140  // could potentially shorten crital path, and expose more loop-invariants.
141  // Note that values' rank are basically defined in RPO order (FIXME).
142  // So, if Rank(X) < Rank(Y) < Rank(Z), it means X is defined earlier
143  // than Y which is defined earlier than Z. Permute "x | 1", "Y & 2",
144  // "z" in the order of X-Y-Z is better than any other orders.
145  struct PtrSortFunctor {
146  bool operator()(XorOpnd * const &LHS, XorOpnd * const &RHS) {
147  return LHS->getSymbolicRank() < RHS->getSymbolicRank();
148  }
149  };
150  private:
151  Value *OrigVal;
152  Value *SymbolicPart;
153  APInt ConstPart;
154  unsigned SymbolicRank;
155  bool isOr;
156  };
157 }
158 
159 namespace {
160  class Reassociate : public FunctionPass {
162  DenseMap<AssertingVH<Value>, unsigned> ValueRankMap;
164  bool MadeChange;
165  public:
166  static char ID; // Pass identification, replacement for typeid
167  Reassociate() : FunctionPass(ID) {
169  }
170 
171  bool runOnFunction(Function &F) override;
172 
173  void getAnalysisUsage(AnalysisUsage &AU) const override {
174  AU.setPreservesCFG();
175  }
176  private:
177  void BuildRankMap(Function &F);
178  unsigned getRank(Value *V);
179  void canonicalizeOperands(Instruction *I);
180  void ReassociateExpression(BinaryOperator *I);
181  void RewriteExprTree(BinaryOperator *I, SmallVectorImpl<ValueEntry> &Ops);
182  Value *OptimizeExpression(BinaryOperator *I,
184  Value *OptimizeAdd(Instruction *I, SmallVectorImpl<ValueEntry> &Ops);
185  Value *OptimizeXor(Instruction *I, SmallVectorImpl<ValueEntry> &Ops);
186  bool CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, APInt &ConstOpnd,
187  Value *&Res);
188  bool CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, XorOpnd *Opnd2,
189  APInt &ConstOpnd, Value *&Res);
190  bool collectMultiplyFactors(SmallVectorImpl<ValueEntry> &Ops,
191  SmallVectorImpl<Factor> &Factors);
192  Value *buildMinimalMultiplyDAG(IRBuilder<> &Builder,
193  SmallVectorImpl<Factor> &Factors);
194  Value *OptimizeMul(BinaryOperator *I, SmallVectorImpl<ValueEntry> &Ops);
195  Value *RemoveFactorFromExpression(Value *V, Value *Factor);
196  void EraseInst(Instruction *I);
197  void OptimizeInst(Instruction *I);
198  Instruction *canonicalizeNegConstExpr(Instruction *I);
199  };
200 }
201 
202 XorOpnd::XorOpnd(Value *V) {
203  assert(!isa<ConstantInt>(V) && "No ConstantInt");
204  OrigVal = V;
206  SymbolicRank = 0;
207 
208  if (I && (I->getOpcode() == Instruction::Or ||
209  I->getOpcode() == Instruction::And)) {
210  Value *V0 = I->getOperand(0);
211  Value *V1 = I->getOperand(1);
212  if (isa<ConstantInt>(V0))
213  std::swap(V0, V1);
214 
215  if (ConstantInt *C = dyn_cast<ConstantInt>(V1)) {
216  ConstPart = C->getValue();
217  SymbolicPart = V0;
218  isOr = (I->getOpcode() == Instruction::Or);
219  return;
220  }
221  }
222 
223  // view the operand as "V | 0"
224  SymbolicPart = V;
225  ConstPart = APInt::getNullValue(V->getType()->getIntegerBitWidth());
226  isOr = true;
227 }
228 
229 char Reassociate::ID = 0;
230 INITIALIZE_PASS(Reassociate, "reassociate",
231  "Reassociate expressions", false, false)
232 
233 // Public interface to the Reassociate pass
234 FunctionPass *llvm::createReassociatePass() { return new Reassociate(); }
235 
236 /// Return true if V is an instruction of the specified opcode and if it
237 /// only has one use.
238 static BinaryOperator *isReassociableOp(Value *V, unsigned Opcode) {
239  if (V->hasOneUse() && isa<Instruction>(V) &&
240  cast<Instruction>(V)->getOpcode() == Opcode &&
241  (!isa<FPMathOperator>(V) ||
242  cast<Instruction>(V)->hasUnsafeAlgebra()))
243  return cast<BinaryOperator>(V);
244  return nullptr;
245 }
246 
247 static BinaryOperator *isReassociableOp(Value *V, unsigned Opcode1,
248  unsigned Opcode2) {
249  if (V->hasOneUse() && isa<Instruction>(V) &&
250  (cast<Instruction>(V)->getOpcode() == Opcode1 ||
251  cast<Instruction>(V)->getOpcode() == Opcode2) &&
252  (!isa<FPMathOperator>(V) ||
253  cast<Instruction>(V)->hasUnsafeAlgebra()))
254  return cast<BinaryOperator>(V);
255  return nullptr;
256 }
257 
259  switch (I->getOpcode()) {
260  case Instruction::PHI:
261  case Instruction::LandingPad:
262  case Instruction::Alloca:
263  case Instruction::Load:
264  case Instruction::Invoke:
265  case Instruction::UDiv:
266  case Instruction::SDiv:
267  case Instruction::FDiv:
268  case Instruction::URem:
269  case Instruction::SRem:
270  case Instruction::FRem:
271  return true;
272  case Instruction::Call:
273  return !isa<DbgInfoIntrinsic>(I);
274  default:
275  return false;
276  }
277 }
278 
279 void Reassociate::BuildRankMap(Function &F) {
280  unsigned i = 2;
281 
282  // Assign distinct ranks to function arguments.
283  for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
284  ValueRankMap[&*I] = ++i;
285  DEBUG(dbgs() << "Calculated Rank[" << I->getName() << "] = " << i << "\n");
286  }
287 
290  E = RPOT.end(); I != E; ++I) {
291  BasicBlock *BB = *I;
292  unsigned BBRank = RankMap[BB] = ++i << 16;
293 
294  // Walk the basic block, adding precomputed ranks for any instructions that
295  // we cannot move. This ensures that the ranks for these instructions are
296  // all different in the block.
297  for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
298  if (isUnmovableInstruction(I))
299  ValueRankMap[&*I] = ++BBRank;
300  }
301 }
302 
303 unsigned Reassociate::getRank(Value *V) {
305  if (!I) {
306  if (isa<Argument>(V)) return ValueRankMap[V]; // Function argument.
307  return 0; // Otherwise it's a global or constant, rank 0.
308  }
309 
310  if (unsigned Rank = ValueRankMap[I])
311  return Rank; // Rank already known?
312 
313  // If this is an expression, return the 1+MAX(rank(LHS), rank(RHS)) so that
314  // we can reassociate expressions for code motion! Since we do not recurse
315  // for PHI nodes, we cannot have infinite recursion here, because there
316  // cannot be loops in the value graph that do not go through PHI nodes.
317  unsigned Rank = 0, MaxRank = RankMap[I->getParent()];
318  for (unsigned i = 0, e = I->getNumOperands();
319  i != e && Rank != MaxRank; ++i)
320  Rank = std::max(Rank, getRank(I->getOperand(i)));
321 
322  // If this is a not or neg instruction, do not count it for rank. This
323  // assures us that X and ~X will have the same rank.
326  ++Rank;
327 
328  DEBUG(dbgs() << "Calculated Rank[" << V->getName() << "] = " << Rank << "\n");
329 
330  return ValueRankMap[I] = Rank;
331 }
332 
333 // Canonicalize constants to RHS. Otherwise, sort the operands by rank.
334 void Reassociate::canonicalizeOperands(Instruction *I) {
335  assert(isa<BinaryOperator>(I) && "Expected binary operator.");
336  assert(I->isCommutative() && "Expected commutative operator.");
337 
338  Value *LHS = I->getOperand(0);
339  Value *RHS = I->getOperand(1);
340  unsigned LHSRank = getRank(LHS);
341  unsigned RHSRank = getRank(RHS);
342 
343  if (isa<Constant>(RHS))
344  return;
345 
346  if (isa<Constant>(LHS) || RHSRank < LHSRank)
347  cast<BinaryOperator>(I)->swapOperands();
348 }
349 
350 static BinaryOperator *CreateAdd(Value *S1, Value *S2, const Twine &Name,
351  Instruction *InsertBefore, Value *FlagsOp) {
352  if (S1->getType()->isIntOrIntVectorTy())
353  return BinaryOperator::CreateAdd(S1, S2, Name, InsertBefore);
354  else {
355  BinaryOperator *Res =
356  BinaryOperator::CreateFAdd(S1, S2, Name, InsertBefore);
357  Res->setFastMathFlags(cast<FPMathOperator>(FlagsOp)->getFastMathFlags());
358  return Res;
359  }
360 }
361 
362 static BinaryOperator *CreateMul(Value *S1, Value *S2, const Twine &Name,
363  Instruction *InsertBefore, Value *FlagsOp) {
364  if (S1->getType()->isIntOrIntVectorTy())
365  return BinaryOperator::CreateMul(S1, S2, Name, InsertBefore);
366  else {
367  BinaryOperator *Res =
368  BinaryOperator::CreateFMul(S1, S2, Name, InsertBefore);
369  Res->setFastMathFlags(cast<FPMathOperator>(FlagsOp)->getFastMathFlags());
370  return Res;
371  }
372 }
373 
375  Instruction *InsertBefore, Value *FlagsOp) {
376  if (S1->getType()->isIntOrIntVectorTy())
377  return BinaryOperator::CreateNeg(S1, Name, InsertBefore);
378  else {
379  BinaryOperator *Res = BinaryOperator::CreateFNeg(S1, Name, InsertBefore);
380  Res->setFastMathFlags(cast<FPMathOperator>(FlagsOp)->getFastMathFlags());
381  return Res;
382  }
383 }
384 
385 /// Replace 0-X with X*-1.
387  Type *Ty = Neg->getType();
388  Constant *NegOne = Ty->isIntOrIntVectorTy() ?
390 
391  BinaryOperator *Res = CreateMul(Neg->getOperand(1), NegOne, "", Neg, Neg);
392  Neg->setOperand(1, Constant::getNullValue(Ty)); // Drop use of op.
393  Res->takeName(Neg);
394  Neg->replaceAllUsesWith(Res);
395  Res->setDebugLoc(Neg->getDebugLoc());
396  return Res;
397 }
398 
399 /// Returns k such that lambda(2^Bitwidth) = 2^k, where lambda is the Carmichael
400 /// function. This means that x^(2^k) === 1 mod 2^Bitwidth for
401 /// every odd x, i.e. x^(2^k) = 1 for every odd x in Bitwidth-bit arithmetic.
402 /// Note that 0 <= k < Bitwidth, and if Bitwidth > 3 then x^(2^k) = 0 for every
403 /// even x in Bitwidth-bit arithmetic.
404 static unsigned CarmichaelShift(unsigned Bitwidth) {
405  if (Bitwidth < 3)
406  return Bitwidth - 1;
407  return Bitwidth - 2;
408 }
409 
410 /// Add the extra weight 'RHS' to the existing weight 'LHS',
411 /// reducing the combined weight using any special properties of the operation.
412 /// The existing weight LHS represents the computation X op X op ... op X where
413 /// X occurs LHS times. The combined weight represents X op X op ... op X with
414 /// X occurring LHS + RHS times. If op is "Xor" for example then the combined
415 /// operation is equivalent to X if LHS + RHS is odd, or 0 if LHS + RHS is even;
416 /// the routine returns 1 in LHS in the first case, and 0 in LHS in the second.
417 static void IncorporateWeight(APInt &LHS, const APInt &RHS, unsigned Opcode) {
418  // If we were working with infinite precision arithmetic then the combined
419  // weight would be LHS + RHS. But we are using finite precision arithmetic,
420  // and the APInt sum LHS + RHS may not be correct if it wraps (it is correct
421  // for nilpotent operations and addition, but not for idempotent operations
422  // and multiplication), so it is important to correctly reduce the combined
423  // weight back into range if wrapping would be wrong.
424 
425  // If RHS is zero then the weight didn't change.
426  if (RHS.isMinValue())
427  return;
428  // If LHS is zero then the combined weight is RHS.
429  if (LHS.isMinValue()) {
430  LHS = RHS;
431  return;
432  }
433  // From this point on we know that neither LHS nor RHS is zero.
434 
435  if (Instruction::isIdempotent(Opcode)) {
436  // Idempotent means X op X === X, so any non-zero weight is equivalent to a
437  // weight of 1. Keeping weights at zero or one also means that wrapping is
438  // not a problem.
439  assert(LHS == 1 && RHS == 1 && "Weights not reduced!");
440  return; // Return a weight of 1.
441  }
442  if (Instruction::isNilpotent(Opcode)) {
443  // Nilpotent means X op X === 0, so reduce weights modulo 2.
444  assert(LHS == 1 && RHS == 1 && "Weights not reduced!");
445  LHS = 0; // 1 + 1 === 0 modulo 2.
446  return;
447  }
448  if (Opcode == Instruction::Add || Opcode == Instruction::FAdd) {
449  // TODO: Reduce the weight by exploiting nsw/nuw?
450  LHS += RHS;
451  return;
452  }
453 
454  assert((Opcode == Instruction::Mul || Opcode == Instruction::FMul) &&
455  "Unknown associative operation!");
456  unsigned Bitwidth = LHS.getBitWidth();
457  // If CM is the Carmichael number then a weight W satisfying W >= CM+Bitwidth
458  // can be replaced with W-CM. That's because x^W=x^(W-CM) for every Bitwidth
459  // bit number x, since either x is odd in which case x^CM = 1, or x is even in
460  // which case both x^W and x^(W - CM) are zero. By subtracting off multiples
461  // of CM like this weights can always be reduced to the range [0, CM+Bitwidth)
462  // which by a happy accident means that they can always be represented using
463  // Bitwidth bits.
464  // TODO: Reduce the weight by exploiting nsw/nuw? (Could do much better than
465  // the Carmichael number).
466  if (Bitwidth > 3) {
467  /// CM - The value of Carmichael's lambda function.
468  APInt CM = APInt::getOneBitSet(Bitwidth, CarmichaelShift(Bitwidth));
469  // Any weight W >= Threshold can be replaced with W - CM.
470  APInt Threshold = CM + Bitwidth;
471  assert(LHS.ult(Threshold) && RHS.ult(Threshold) && "Weights not reduced!");
472  // For Bitwidth 4 or more the following sum does not overflow.
473  LHS += RHS;
474  while (LHS.uge(Threshold))
475  LHS -= CM;
476  } else {
477  // To avoid problems with overflow do everything the same as above but using
478  // a larger type.
479  unsigned CM = 1U << CarmichaelShift(Bitwidth);
480  unsigned Threshold = CM + Bitwidth;
481  assert(LHS.getZExtValue() < Threshold && RHS.getZExtValue() < Threshold &&
482  "Weights not reduced!");
483  unsigned Total = LHS.getZExtValue() + RHS.getZExtValue();
484  while (Total >= Threshold)
485  Total -= CM;
486  LHS = Total;
487  }
488 }
489 
490 typedef std::pair<Value*, APInt> RepeatedValue;
491 
492 /// Given an associative binary expression, return the leaf
493 /// nodes in Ops along with their weights (how many times the leaf occurs). The
494 /// original expression is the same as
495 /// (Ops[0].first op Ops[0].first op ... Ops[0].first) <- Ops[0].second times
496 /// op
497 /// (Ops[1].first op Ops[1].first op ... Ops[1].first) <- Ops[1].second times
498 /// op
499 /// ...
500 /// op
501 /// (Ops[N].first op Ops[N].first op ... Ops[N].first) <- Ops[N].second times
502 ///
503 /// Note that the values Ops[0].first, ..., Ops[N].first are all distinct.
504 ///
505 /// This routine may modify the function, in which case it returns 'true'. The
506 /// changes it makes may well be destructive, changing the value computed by 'I'
507 /// to something completely different. Thus if the routine returns 'true' then
508 /// you MUST either replace I with a new expression computed from the Ops array,
509 /// or use RewriteExprTree to put the values back in.
510 ///
511 /// A leaf node is either not a binary operation of the same kind as the root
512 /// node 'I' (i.e. is not a binary operator at all, or is, but with a different
513 /// opcode), or is the same kind of binary operator but has a use which either
514 /// does not belong to the expression, or does belong to the expression but is
515 /// a leaf node. Every leaf node has at least one use that is a non-leaf node
516 /// of the expression, while for non-leaf nodes (except for the root 'I') every
517 /// use is a non-leaf node of the expression.
518 ///
519 /// For example:
520 /// expression graph node names
521 ///
522 /// + | I
523 /// / \ |
524 /// + + | A, B
525 /// / \ / \ |
526 /// * + * | C, D, E
527 /// / \ / \ / \ |
528 /// + * | F, G
529 ///
530 /// The leaf nodes are C, E, F and G. The Ops array will contain (maybe not in
531 /// that order) (C, 1), (E, 1), (F, 2), (G, 2).
532 ///
533 /// The expression is maximal: if some instruction is a binary operator of the
534 /// same kind as 'I', and all of its uses are non-leaf nodes of the expression,
535 /// then the instruction also belongs to the expression, is not a leaf node of
536 /// it, and its operands also belong to the expression (but may be leaf nodes).
537 ///
538 /// NOTE: This routine will set operands of non-leaf non-root nodes to undef in
539 /// order to ensure that every non-root node in the expression has *exactly one*
540 /// use by a non-leaf node of the expression. This destruction means that the
541 /// caller MUST either replace 'I' with a new expression or use something like
542 /// RewriteExprTree to put the values back in if the routine indicates that it
543 /// made a change by returning 'true'.
544 ///
545 /// In the above example either the right operand of A or the left operand of B
546 /// will be replaced by undef. If it is B's operand then this gives:
547 ///
548 /// + | I
549 /// / \ |
550 /// + + | A, B - operand of B replaced with undef
551 /// / \ \ |
552 /// * + * | C, D, E
553 /// / \ / \ / \ |
554 /// + * | F, G
555 ///
556 /// Note that such undef operands can only be reached by passing through 'I'.
557 /// For example, if you visit operands recursively starting from a leaf node
558 /// then you will never see such an undef operand unless you get back to 'I',
559 /// which requires passing through a phi node.
560 ///
561 /// Note that this routine may also mutate binary operators of the wrong type
562 /// that have all uses inside the expression (i.e. only used by non-leaf nodes
563 /// of the expression) if it can turn them into binary operators of the right
564 /// type and thus make the expression bigger.
565 
568  DEBUG(dbgs() << "LINEARIZE: " << *I << '\n');
569  unsigned Bitwidth = I->getType()->getScalarType()->getPrimitiveSizeInBits();
570  unsigned Opcode = I->getOpcode();
571  assert(I->isAssociative() && I->isCommutative() &&
572  "Expected an associative and commutative operation!");
573 
574  // Visit all operands of the expression, keeping track of their weight (the
575  // number of paths from the expression root to the operand, or if you like
576  // the number of times that operand occurs in the linearized expression).
577  // For example, if I = X + A, where X = A + B, then I, X and B have weight 1
578  // while A has weight two.
579 
580  // Worklist of non-leaf nodes (their operands are in the expression too) along
581  // with their weights, representing a certain number of paths to the operator.
582  // If an operator occurs in the worklist multiple times then we found multiple
583  // ways to get to it.
584  SmallVector<std::pair<BinaryOperator*, APInt>, 8> Worklist; // (Op, Weight)
585  Worklist.push_back(std::make_pair(I, APInt(Bitwidth, 1)));
586  bool Changed = false;
587 
588  // Leaves of the expression are values that either aren't the right kind of
589  // operation (eg: a constant, or a multiply in an add tree), or are, but have
590  // some uses that are not inside the expression. For example, in I = X + X,
591  // X = A + B, the value X has two uses (by I) that are in the expression. If
592  // X has any other uses, for example in a return instruction, then we consider
593  // X to be a leaf, and won't analyze it further. When we first visit a value,
594  // if it has more than one use then at first we conservatively consider it to
595  // be a leaf. Later, as the expression is explored, we may discover some more
596  // uses of the value from inside the expression. If all uses turn out to be
597  // from within the expression (and the value is a binary operator of the right
598  // kind) then the value is no longer considered to be a leaf, and its operands
599  // are explored.
600 
601  // Leaves - Keeps track of the set of putative leaves as well as the number of
602  // paths to each leaf seen so far.
603  typedef DenseMap<Value*, APInt> LeafMap;
604  LeafMap Leaves; // Leaf -> Total weight so far.
605  SmallVector<Value*, 8> LeafOrder; // Ensure deterministic leaf output order.
606 
607 #ifndef NDEBUG
608  SmallPtrSet<Value*, 8> Visited; // For sanity checking the iteration scheme.
609 #endif
610  while (!Worklist.empty()) {
611  std::pair<BinaryOperator*, APInt> P = Worklist.pop_back_val();
612  I = P.first; // We examine the operands of this binary operator.
613 
614  for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx) { // Visit operands.
615  Value *Op = I->getOperand(OpIdx);
616  APInt Weight = P.second; // Number of paths to this operand.
617  DEBUG(dbgs() << "OPERAND: " << *Op << " (" << Weight << ")\n");
618  assert(!Op->use_empty() && "No uses, so how did we get to it?!");
619 
620  // If this is a binary operation of the right kind with only one use then
621  // add its operands to the expression.
622  if (BinaryOperator *BO = isReassociableOp(Op, Opcode)) {
623  assert(Visited.insert(Op).second && "Not first visit!");
624  DEBUG(dbgs() << "DIRECT ADD: " << *Op << " (" << Weight << ")\n");
625  Worklist.push_back(std::make_pair(BO, Weight));
626  continue;
627  }
628 
629  // Appears to be a leaf. Is the operand already in the set of leaves?
630  LeafMap::iterator It = Leaves.find(Op);
631  if (It == Leaves.end()) {
632  // Not in the leaf map. Must be the first time we saw this operand.
633  assert(Visited.insert(Op).second && "Not first visit!");
634  if (!Op->hasOneUse()) {
635  // This value has uses not accounted for by the expression, so it is
636  // not safe to modify. Mark it as being a leaf.
637  DEBUG(dbgs() << "ADD USES LEAF: " << *Op << " (" << Weight << ")\n");
638  LeafOrder.push_back(Op);
639  Leaves[Op] = Weight;
640  continue;
641  }
642  // No uses outside the expression, try morphing it.
643  } else if (It != Leaves.end()) {
644  // Already in the leaf map.
645  assert(Visited.count(Op) && "In leaf map but not visited!");
646 
647  // Update the number of paths to the leaf.
648  IncorporateWeight(It->second, Weight, Opcode);
649 
650 #if 0 // TODO: Re-enable once PR13021 is fixed.
651  // The leaf already has one use from inside the expression. As we want
652  // exactly one such use, drop this new use of the leaf.
653  assert(!Op->hasOneUse() && "Only one use, but we got here twice!");
654  I->setOperand(OpIdx, UndefValue::get(I->getType()));
655  Changed = true;
656 
657  // If the leaf is a binary operation of the right kind and we now see
658  // that its multiple original uses were in fact all by nodes belonging
659  // to the expression, then no longer consider it to be a leaf and add
660  // its operands to the expression.
661  if (BinaryOperator *BO = isReassociableOp(Op, Opcode)) {
662  DEBUG(dbgs() << "UNLEAF: " << *Op << " (" << It->second << ")\n");
663  Worklist.push_back(std::make_pair(BO, It->second));
664  Leaves.erase(It);
665  continue;
666  }
667 #endif
668 
669  // If we still have uses that are not accounted for by the expression
670  // then it is not safe to modify the value.
671  if (!Op->hasOneUse())
672  continue;
673 
674  // No uses outside the expression, try morphing it.
675  Weight = It->second;
676  Leaves.erase(It); // Since the value may be morphed below.
677  }
678 
679  // At this point we have a value which, first of all, is not a binary
680  // expression of the right kind, and secondly, is only used inside the
681  // expression. This means that it can safely be modified. See if we
682  // can usefully morph it into an expression of the right kind.
683  assert((!isa<Instruction>(Op) ||
684  cast<Instruction>(Op)->getOpcode() != Opcode
685  || (isa<FPMathOperator>(Op) &&
686  !cast<Instruction>(Op)->hasUnsafeAlgebra())) &&
687  "Should have been handled above!");
688  assert(Op->hasOneUse() && "Has uses outside the expression tree!");
689 
690  // If this is a multiply expression, turn any internal negations into
691  // multiplies by -1 so they can be reassociated.
692  if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Op))
693  if ((Opcode == Instruction::Mul && BinaryOperator::isNeg(BO)) ||
694  (Opcode == Instruction::FMul && BinaryOperator::isFNeg(BO))) {
695  DEBUG(dbgs() << "MORPH LEAF: " << *Op << " (" << Weight << ") TO ");
696  BO = LowerNegateToMultiply(BO);
697  DEBUG(dbgs() << *BO << '\n');
698  Worklist.push_back(std::make_pair(BO, Weight));
699  Changed = true;
700  continue;
701  }
702 
703  // Failed to morph into an expression of the right type. This really is
704  // a leaf.
705  DEBUG(dbgs() << "ADD LEAF: " << *Op << " (" << Weight << ")\n");
706  assert(!isReassociableOp(Op, Opcode) && "Value was morphed?");
707  LeafOrder.push_back(Op);
708  Leaves[Op] = Weight;
709  }
710  }
711 
712  // The leaves, repeated according to their weights, represent the linearized
713  // form of the expression.
714  for (unsigned i = 0, e = LeafOrder.size(); i != e; ++i) {
715  Value *V = LeafOrder[i];
716  LeafMap::iterator It = Leaves.find(V);
717  if (It == Leaves.end())
718  // Node initially thought to be a leaf wasn't.
719  continue;
720  assert(!isReassociableOp(V, Opcode) && "Shouldn't be a leaf!");
721  APInt Weight = It->second;
722  if (Weight.isMinValue())
723  // Leaf already output or weight reduction eliminated it.
724  continue;
725  // Ensure the leaf is only output once.
726  It->second = 0;
727  Ops.push_back(std::make_pair(V, Weight));
728  }
729 
730  // For nilpotent operations or addition there may be no operands, for example
731  // because the expression was "X xor X" or consisted of 2^Bitwidth additions:
732  // in both cases the weight reduces to 0 causing the value to be skipped.
733  if (Ops.empty()) {
734  Constant *Identity = ConstantExpr::getBinOpIdentity(Opcode, I->getType());
735  assert(Identity && "Associative operation without identity!");
736  Ops.emplace_back(Identity, APInt(Bitwidth, 1));
737  }
738 
739  return Changed;
740 }
741 
742 /// Now that the operands for this expression tree are
743 /// linearized and optimized, emit them in-order.
744 void Reassociate::RewriteExprTree(BinaryOperator *I,
746  assert(Ops.size() > 1 && "Single values should be used directly!");
747 
748  // Since our optimizations should never increase the number of operations, the
749  // new expression can usually be written reusing the existing binary operators
750  // from the original expression tree, without creating any new instructions,
751  // though the rewritten expression may have a completely different topology.
752  // We take care to not change anything if the new expression will be the same
753  // as the original. If more than trivial changes (like commuting operands)
754  // were made then we are obliged to clear out any optional subclass data like
755  // nsw flags.
756 
757  /// NodesToRewrite - Nodes from the original expression available for writing
758  /// the new expression into.
759  SmallVector<BinaryOperator*, 8> NodesToRewrite;
760  unsigned Opcode = I->getOpcode();
761  BinaryOperator *Op = I;
762 
763  /// NotRewritable - The operands being written will be the leaves of the new
764  /// expression and must not be used as inner nodes (via NodesToRewrite) by
765  /// mistake. Inner nodes are always reassociable, and usually leaves are not
766  /// (if they were they would have been incorporated into the expression and so
767  /// would not be leaves), so most of the time there is no danger of this. But
768  /// in rare cases a leaf may become reassociable if an optimization kills uses
769  /// of it, or it may momentarily become reassociable during rewriting (below)
770  /// due it being removed as an operand of one of its uses. Ensure that misuse
771  /// of leaf nodes as inner nodes cannot occur by remembering all of the future
772  /// leaves and refusing to reuse any of them as inner nodes.
773  SmallPtrSet<Value*, 8> NotRewritable;
774  for (unsigned i = 0, e = Ops.size(); i != e; ++i)
775  NotRewritable.insert(Ops[i].Op);
776 
777  // ExpressionChanged - Non-null if the rewritten expression differs from the
778  // original in some non-trivial way, requiring the clearing of optional flags.
779  // Flags are cleared from the operator in ExpressionChanged up to I inclusive.
780  BinaryOperator *ExpressionChanged = nullptr;
781  for (unsigned i = 0; ; ++i) {
782  // The last operation (which comes earliest in the IR) is special as both
783  // operands will come from Ops, rather than just one with the other being
784  // a subexpression.
785  if (i+2 == Ops.size()) {
786  Value *NewLHS = Ops[i].Op;
787  Value *NewRHS = Ops[i+1].Op;
788  Value *OldLHS = Op->getOperand(0);
789  Value *OldRHS = Op->getOperand(1);
790 
791  if (NewLHS == OldLHS && NewRHS == OldRHS)
792  // Nothing changed, leave it alone.
793  break;
794 
795  if (NewLHS == OldRHS && NewRHS == OldLHS) {
796  // The order of the operands was reversed. Swap them.
797  DEBUG(dbgs() << "RA: " << *Op << '\n');
798  Op->swapOperands();
799  DEBUG(dbgs() << "TO: " << *Op << '\n');
800  MadeChange = true;
801  ++NumChanged;
802  break;
803  }
804 
805  // The new operation differs non-trivially from the original. Overwrite
806  // the old operands with the new ones.
807  DEBUG(dbgs() << "RA: " << *Op << '\n');
808  if (NewLHS != OldLHS) {
809  BinaryOperator *BO = isReassociableOp(OldLHS, Opcode);
810  if (BO && !NotRewritable.count(BO))
811  NodesToRewrite.push_back(BO);
812  Op->setOperand(0, NewLHS);
813  }
814  if (NewRHS != OldRHS) {
815  BinaryOperator *BO = isReassociableOp(OldRHS, Opcode);
816  if (BO && !NotRewritable.count(BO))
817  NodesToRewrite.push_back(BO);
818  Op->setOperand(1, NewRHS);
819  }
820  DEBUG(dbgs() << "TO: " << *Op << '\n');
821 
822  ExpressionChanged = Op;
823  MadeChange = true;
824  ++NumChanged;
825 
826  break;
827  }
828 
829  // Not the last operation. The left-hand side will be a sub-expression
830  // while the right-hand side will be the current element of Ops.
831  Value *NewRHS = Ops[i].Op;
832  if (NewRHS != Op->getOperand(1)) {
833  DEBUG(dbgs() << "RA: " << *Op << '\n');
834  if (NewRHS == Op->getOperand(0)) {
835  // The new right-hand side was already present as the left operand. If
836  // we are lucky then swapping the operands will sort out both of them.
837  Op->swapOperands();
838  } else {
839  // Overwrite with the new right-hand side.
840  BinaryOperator *BO = isReassociableOp(Op->getOperand(1), Opcode);
841  if (BO && !NotRewritable.count(BO))
842  NodesToRewrite.push_back(BO);
843  Op->setOperand(1, NewRHS);
844  ExpressionChanged = Op;
845  }
846  DEBUG(dbgs() << "TO: " << *Op << '\n');
847  MadeChange = true;
848  ++NumChanged;
849  }
850 
851  // Now deal with the left-hand side. If this is already an operation node
852  // from the original expression then just rewrite the rest of the expression
853  // into it.
854  BinaryOperator *BO = isReassociableOp(Op->getOperand(0), Opcode);
855  if (BO && !NotRewritable.count(BO)) {
856  Op = BO;
857  continue;
858  }
859 
860  // Otherwise, grab a spare node from the original expression and use that as
861  // the left-hand side. If there are no nodes left then the optimizers made
862  // an expression with more nodes than the original! This usually means that
863  // they did something stupid but it might mean that the problem was just too
864  // hard (finding the mimimal number of multiplications needed to realize a
865  // multiplication expression is NP-complete). Whatever the reason, smart or
866  // stupid, create a new node if there are none left.
867  BinaryOperator *NewOp;
868  if (NodesToRewrite.empty()) {
871  Undef, Undef, "", I);
872  if (NewOp->getType()->isFPOrFPVectorTy())
873  NewOp->setFastMathFlags(I->getFastMathFlags());
874  } else {
875  NewOp = NodesToRewrite.pop_back_val();
876  }
877 
878  DEBUG(dbgs() << "RA: " << *Op << '\n');
879  Op->setOperand(0, NewOp);
880  DEBUG(dbgs() << "TO: " << *Op << '\n');
881  ExpressionChanged = Op;
882  MadeChange = true;
883  ++NumChanged;
884  Op = NewOp;
885  }
886 
887  // If the expression changed non-trivially then clear out all subclass data
888  // starting from the operator specified in ExpressionChanged, and compactify
889  // the operators to just before the expression root to guarantee that the
890  // expression tree is dominated by all of Ops.
891  if (ExpressionChanged)
892  do {
893  // Preserve FastMathFlags.
894  if (isa<FPMathOperator>(I)) {
896  ExpressionChanged->clearSubclassOptionalData();
897  ExpressionChanged->setFastMathFlags(Flags);
898  } else
899  ExpressionChanged->clearSubclassOptionalData();
900 
901  if (ExpressionChanged == I)
902  break;
903  ExpressionChanged->moveBefore(I);
904  ExpressionChanged = cast<BinaryOperator>(*ExpressionChanged->user_begin());
905  } while (1);
906 
907  // Throw away any left over nodes from the original expression.
908  for (unsigned i = 0, e = NodesToRewrite.size(); i != e; ++i)
909  RedoInsts.insert(NodesToRewrite[i]);
910 }
911 
912 /// Insert instructions before the instruction pointed to by BI,
913 /// that computes the negative version of the value specified. The negative
914 /// version of the value is returned, and BI is left pointing at the instruction
915 /// that should be processed next by the reassociation pass.
916 static Value *NegateValue(Value *V, Instruction *BI) {
917  if (Constant *C = dyn_cast<Constant>(V)) {
918  if (C->getType()->isFPOrFPVectorTy()) {
919  return ConstantExpr::getFNeg(C);
920  }
921  return ConstantExpr::getNeg(C);
922  }
923 
924 
925  // We are trying to expose opportunity for reassociation. One of the things
926  // that we want to do to achieve this is to push a negation as deep into an
927  // expression chain as possible, to expose the add instructions. In practice,
928  // this means that we turn this:
929  // X = -(A+12+C+D) into X = -A + -12 + -C + -D = -12 + -A + -C + -D
930  // so that later, a: Y = 12+X could get reassociated with the -12 to eliminate
931  // the constants. We assume that instcombine will clean up the mess later if
932  // we introduce tons of unnecessary negation instructions.
933  //
934  if (BinaryOperator *I =
935  isReassociableOp(V, Instruction::Add, Instruction::FAdd)) {
936  // Push the negates through the add.
937  I->setOperand(0, NegateValue(I->getOperand(0), BI));
938  I->setOperand(1, NegateValue(I->getOperand(1), BI));
939  if (I->getOpcode() == Instruction::Add) {
940  I->setHasNoUnsignedWrap(false);
941  I->setHasNoSignedWrap(false);
942  }
943 
944  // We must move the add instruction here, because the neg instructions do
945  // not dominate the old add instruction in general. By moving it, we are
946  // assured that the neg instructions we just inserted dominate the
947  // instruction we are about to insert after them.
948  //
949  I->moveBefore(BI);
950  I->setName(I->getName()+".neg");
951  return I;
952  }
953 
954  // Okay, we need to materialize a negated version of V with an instruction.
955  // Scan the use lists of V to see if we have one already.
956  for (User *U : V->users()) {
958  continue;
959 
960  // We found one! Now we have to make sure that the definition dominates
961  // this use. We do this by moving it to the entry block (if it is a
962  // non-instruction value) or right after the definition. These negates will
963  // be zapped by reassociate later, so we don't need much finesse here.
964  BinaryOperator *TheNeg = cast<BinaryOperator>(U);
965 
966  // Verify that the negate is in this function, V might be a constant expr.
967  if (TheNeg->getParent()->getParent() != BI->getParent()->getParent())
968  continue;
969 
970  BasicBlock::iterator InsertPt;
971  if (Instruction *InstInput = dyn_cast<Instruction>(V)) {
972  if (InvokeInst *II = dyn_cast<InvokeInst>(InstInput)) {
973  InsertPt = II->getNormalDest()->begin();
974  } else {
975  InsertPt = InstInput;
976  ++InsertPt;
977  }
978  while (isa<PHINode>(InsertPt)) ++InsertPt;
979  } else {
980  InsertPt = TheNeg->getParent()->getParent()->getEntryBlock().begin();
981  }
982  TheNeg->moveBefore(InsertPt);
983  if (TheNeg->getOpcode() == Instruction::Sub) {
984  TheNeg->setHasNoUnsignedWrap(false);
985  TheNeg->setHasNoSignedWrap(false);
986  } else {
987  TheNeg->andIRFlags(BI);
988  }
989  return TheNeg;
990  }
991 
992  // Insert a 'neg' instruction that subtracts the value from zero to get the
993  // negation.
994  return CreateNeg(V, V->getName() + ".neg", BI, BI);
995 }
996 
997 /// Return true if we should break up this subtract of X-Y into (X + -Y).
999  // If this is a negation, we can't split it up!
1001  return false;
1002 
1003  // Don't breakup X - undef.
1004  if (isa<UndefValue>(Sub->getOperand(1)))
1005  return false;
1006 
1007  // Don't bother to break this up unless either the LHS is an associable add or
1008  // subtract or if this is only used by one.
1009  Value *V0 = Sub->getOperand(0);
1010  if (isReassociableOp(V0, Instruction::Add, Instruction::FAdd) ||
1011  isReassociableOp(V0, Instruction::Sub, Instruction::FSub))
1012  return true;
1013  Value *V1 = Sub->getOperand(1);
1014  if (isReassociableOp(V1, Instruction::Add, Instruction::FAdd) ||
1015  isReassociableOp(V1, Instruction::Sub, Instruction::FSub))
1016  return true;
1017  Value *VB = Sub->user_back();
1018  if (Sub->hasOneUse() &&
1019  (isReassociableOp(VB, Instruction::Add, Instruction::FAdd) ||
1020  isReassociableOp(VB, Instruction::Sub, Instruction::FSub)))
1021  return true;
1022 
1023  return false;
1024 }
1025 
1026 /// If we have (X-Y), and if either X is an add, or if this is only used by an
1027 /// add, transform this into (X+(0-Y)) to promote better reassociation.
1029  // Convert a subtract into an add and a neg instruction. This allows sub
1030  // instructions to be commuted with other add instructions.
1031  //
1032  // Calculate the negative value of Operand 1 of the sub instruction,
1033  // and set it as the RHS of the add instruction we just made.
1034  //
1035  Value *NegVal = NegateValue(Sub->getOperand(1), Sub);
1036  BinaryOperator *New = CreateAdd(Sub->getOperand(0), NegVal, "", Sub, Sub);
1037  Sub->setOperand(0, Constant::getNullValue(Sub->getType())); // Drop use of op.
1038  Sub->setOperand(1, Constant::getNullValue(Sub->getType())); // Drop use of op.
1039  New->takeName(Sub);
1040 
1041  // Everyone now refers to the add instruction.
1042  Sub->replaceAllUsesWith(New);
1043  New->setDebugLoc(Sub->getDebugLoc());
1044 
1045  DEBUG(dbgs() << "Negated: " << *New << '\n');
1046  return New;
1047 }
1048 
1049 /// If this is a shift of a reassociable multiply or is used by one, change
1050 /// this into a multiply by a constant to assist with further reassociation.
1052  Constant *MulCst = ConstantInt::get(Shl->getType(), 1);
1053  MulCst = ConstantExpr::getShl(MulCst, cast<Constant>(Shl->getOperand(1)));
1054 
1055  BinaryOperator *Mul =
1056  BinaryOperator::CreateMul(Shl->getOperand(0), MulCst, "", Shl);
1057  Shl->setOperand(0, UndefValue::get(Shl->getType())); // Drop use of op.
1058  Mul->takeName(Shl);
1059 
1060  // Everyone now refers to the mul instruction.
1061  Shl->replaceAllUsesWith(Mul);
1062  Mul->setDebugLoc(Shl->getDebugLoc());
1063 
1064  // We can safely preserve the nuw flag in all cases. It's also safe to turn a
1065  // nuw nsw shl into a nuw nsw mul. However, nsw in isolation requires special
1066  // handling.
1067  bool NSW = cast<BinaryOperator>(Shl)->hasNoSignedWrap();
1068  bool NUW = cast<BinaryOperator>(Shl)->hasNoUnsignedWrap();
1069  if (NSW && NUW)
1070  Mul->setHasNoSignedWrap(true);
1071  Mul->setHasNoUnsignedWrap(NUW);
1072  return Mul;
1073 }
1074 
1075 /// Scan backwards and forwards among values with the same rank as element i
1076 /// to see if X exists. If X does not exist, return i. This is useful when
1077 /// scanning for 'x' when we see '-x' because they both get the same rank.
1078 static unsigned FindInOperandList(SmallVectorImpl<ValueEntry> &Ops, unsigned i,
1079  Value *X) {
1080  unsigned XRank = Ops[i].Rank;
1081  unsigned e = Ops.size();
1082  for (unsigned j = i+1; j != e && Ops[j].Rank == XRank; ++j) {
1083  if (Ops[j].Op == X)
1084  return j;
1085  if (Instruction *I1 = dyn_cast<Instruction>(Ops[j].Op))
1086  if (Instruction *I2 = dyn_cast<Instruction>(X))
1087  if (I1->isIdenticalTo(I2))
1088  return j;
1089  }
1090  // Scan backwards.
1091  for (unsigned j = i-1; j != ~0U && Ops[j].Rank == XRank; --j) {
1092  if (Ops[j].Op == X)
1093  return j;
1094  if (Instruction *I1 = dyn_cast<Instruction>(Ops[j].Op))
1095  if (Instruction *I2 = dyn_cast<Instruction>(X))
1096  if (I1->isIdenticalTo(I2))
1097  return j;
1098  }
1099  return i;
1100 }
1101 
1102 /// Emit a tree of add instructions, summing Ops together
1103 /// and returning the result. Insert the tree before I.
1106  if (Ops.size() == 1) return Ops.back();
1107 
1108  Value *V1 = Ops.back();
1109  Ops.pop_back();
1110  Value *V2 = EmitAddTreeOfValues(I, Ops);
1111  return CreateAdd(V2, V1, "tmp", I, I);
1112 }
1113 
1114 /// If V is an expression tree that is a multiplication sequence,
1115 /// and if this sequence contains a multiply by Factor,
1116 /// remove Factor from the tree and return the new tree.
1117 Value *Reassociate::RemoveFactorFromExpression(Value *V, Value *Factor) {
1118  BinaryOperator *BO = isReassociableOp(V, Instruction::Mul, Instruction::FMul);
1119  if (!BO)
1120  return nullptr;
1121 
1123  MadeChange |= LinearizeExprTree(BO, Tree);
1125  Factors.reserve(Tree.size());
1126  for (unsigned i = 0, e = Tree.size(); i != e; ++i) {
1127  RepeatedValue E = Tree[i];
1128  Factors.append(E.second.getZExtValue(),
1129  ValueEntry(getRank(E.first), E.first));
1130  }
1131 
1132  bool FoundFactor = false;
1133  bool NeedsNegate = false;
1134  for (unsigned i = 0, e = Factors.size(); i != e; ++i) {
1135  if (Factors[i].Op == Factor) {
1136  FoundFactor = true;
1137  Factors.erase(Factors.begin()+i);
1138  break;
1139  }
1140 
1141  // If this is a negative version of this factor, remove it.
1142  if (ConstantInt *FC1 = dyn_cast<ConstantInt>(Factor)) {
1143  if (ConstantInt *FC2 = dyn_cast<ConstantInt>(Factors[i].Op))
1144  if (FC1->getValue() == -FC2->getValue()) {
1145  FoundFactor = NeedsNegate = true;
1146  Factors.erase(Factors.begin()+i);
1147  break;
1148  }
1149  } else if (ConstantFP *FC1 = dyn_cast<ConstantFP>(Factor)) {
1150  if (ConstantFP *FC2 = dyn_cast<ConstantFP>(Factors[i].Op)) {
1151  APFloat F1(FC1->getValueAPF());
1152  APFloat F2(FC2->getValueAPF());
1153  F2.changeSign();
1154  if (F1.compare(F2) == APFloat::cmpEqual) {
1155  FoundFactor = NeedsNegate = true;
1156  Factors.erase(Factors.begin() + i);
1157  break;
1158  }
1159  }
1160  }
1161  }
1162 
1163  if (!FoundFactor) {
1164  // Make sure to restore the operands to the expression tree.
1165  RewriteExprTree(BO, Factors);
1166  return nullptr;
1167  }
1168 
1169  BasicBlock::iterator InsertPt = BO; ++InsertPt;
1170 
1171  // If this was just a single multiply, remove the multiply and return the only
1172  // remaining operand.
1173  if (Factors.size() == 1) {
1174  RedoInsts.insert(BO);
1175  V = Factors[0].Op;
1176  } else {
1177  RewriteExprTree(BO, Factors);
1178  V = BO;
1179  }
1180 
1181  if (NeedsNegate)
1182  V = CreateNeg(V, "neg", InsertPt, BO);
1183 
1184  return V;
1185 }
1186 
1187 /// If V is a single-use multiply, recursively add its operands as factors,
1188 /// otherwise add V to the list of factors.
1189 ///
1190 /// Ops is the top-level list of add operands we're trying to factor.
1192  SmallVectorImpl<Value*> &Factors,
1193  const SmallVectorImpl<ValueEntry> &Ops) {
1194  BinaryOperator *BO = isReassociableOp(V, Instruction::Mul, Instruction::FMul);
1195  if (!BO) {
1196  Factors.push_back(V);
1197  return;
1198  }
1199 
1200  // Otherwise, add the LHS and RHS to the list of factors.
1201  FindSingleUseMultiplyFactors(BO->getOperand(1), Factors, Ops);
1202  FindSingleUseMultiplyFactors(BO->getOperand(0), Factors, Ops);
1203 }
1204 
1205 /// Optimize a series of operands to an 'and', 'or', or 'xor' instruction.
1206 /// This optimizes based on identities. If it can be reduced to a single Value,
1207 /// it is returned, otherwise the Ops list is mutated as necessary.
1208 static Value *OptimizeAndOrXor(unsigned Opcode,
1210  // Scan the operand lists looking for X and ~X pairs, along with X,X pairs.
1211  // If we find any, we can simplify the expression. X&~X == 0, X|~X == -1.
1212  for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
1213  // First, check for X and ~X in the operand list.
1214  assert(i < Ops.size());
1215  if (BinaryOperator::isNot(Ops[i].Op)) { // Cannot occur for ^.
1216  Value *X = BinaryOperator::getNotArgument(Ops[i].Op);
1217  unsigned FoundX = FindInOperandList(Ops, i, X);
1218  if (FoundX != i) {
1219  if (Opcode == Instruction::And) // ...&X&~X = 0
1220  return Constant::getNullValue(X->getType());
1221 
1222  if (Opcode == Instruction::Or) // ...|X|~X = -1
1223  return Constant::getAllOnesValue(X->getType());
1224  }
1225  }
1226 
1227  // Next, check for duplicate pairs of values, which we assume are next to
1228  // each other, due to our sorting criteria.
1229  assert(i < Ops.size());
1230  if (i+1 != Ops.size() && Ops[i+1].Op == Ops[i].Op) {
1231  if (Opcode == Instruction::And || Opcode == Instruction::Or) {
1232  // Drop duplicate values for And and Or.
1233  Ops.erase(Ops.begin()+i);
1234  --i; --e;
1235  ++NumAnnihil;
1236  continue;
1237  }
1238 
1239  // Drop pairs of values for Xor.
1240  assert(Opcode == Instruction::Xor);
1241  if (e == 2)
1242  return Constant::getNullValue(Ops[0].Op->getType());
1243 
1244  // Y ^ X^X -> Y
1245  Ops.erase(Ops.begin()+i, Ops.begin()+i+2);
1246  i -= 1; e -= 2;
1247  ++NumAnnihil;
1248  }
1249  }
1250  return nullptr;
1251 }
1252 
1253 /// Helper funciton of CombineXorOpnd(). It creates a bitwise-and
1254 /// instruction with the given two operands, and return the resulting
1255 /// instruction. There are two special cases: 1) if the constant operand is 0,
1256 /// it will return NULL. 2) if the constant is ~0, the symbolic operand will
1257 /// be returned.
1258 static Value *createAndInstr(Instruction *InsertBefore, Value *Opnd,
1259  const APInt &ConstOpnd) {
1260  if (ConstOpnd != 0) {
1261  if (!ConstOpnd.isAllOnesValue()) {
1262  LLVMContext &Ctx = Opnd->getType()->getContext();
1263  Instruction *I;
1264  I = BinaryOperator::CreateAnd(Opnd, ConstantInt::get(Ctx, ConstOpnd),
1265  "and.ra", InsertBefore);
1266  I->setDebugLoc(InsertBefore->getDebugLoc());
1267  return I;
1268  }
1269  return Opnd;
1270  }
1271  return nullptr;
1272 }
1273 
1274 // Helper function of OptimizeXor(). It tries to simplify "Opnd1 ^ ConstOpnd"
1275 // into "R ^ C", where C would be 0, and R is a symbolic value.
1276 //
1277 // If it was successful, true is returned, and the "R" and "C" is returned
1278 // via "Res" and "ConstOpnd", respectively; otherwise, false is returned,
1279 // and both "Res" and "ConstOpnd" remain unchanged.
1280 //
1281 bool Reassociate::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1,
1282  APInt &ConstOpnd, Value *&Res) {
1283  // Xor-Rule 1: (x | c1) ^ c2 = (x | c1) ^ (c1 ^ c1) ^ c2
1284  // = ((x | c1) ^ c1) ^ (c1 ^ c2)
1285  // = (x & ~c1) ^ (c1 ^ c2)
1286  // It is useful only when c1 == c2.
1287  if (Opnd1->isOrExpr() && Opnd1->getConstPart() != 0) {
1288  if (!Opnd1->getValue()->hasOneUse())
1289  return false;
1290 
1291  const APInt &C1 = Opnd1->getConstPart();
1292  if (C1 != ConstOpnd)
1293  return false;
1294 
1295  Value *X = Opnd1->getSymbolicPart();
1296  Res = createAndInstr(I, X, ~C1);
1297  // ConstOpnd was C2, now C1 ^ C2.
1298  ConstOpnd ^= C1;
1299 
1300  if (Instruction *T = dyn_cast<Instruction>(Opnd1->getValue()))
1301  RedoInsts.insert(T);
1302  return true;
1303  }
1304  return false;
1305 }
1306 
1307 
1308 // Helper function of OptimizeXor(). It tries to simplify
1309 // "Opnd1 ^ Opnd2 ^ ConstOpnd" into "R ^ C", where C would be 0, and R is a
1310 // symbolic value.
1311 //
1312 // If it was successful, true is returned, and the "R" and "C" is returned
1313 // via "Res" and "ConstOpnd", respectively (If the entire expression is
1314 // evaluated to a constant, the Res is set to NULL); otherwise, false is
1315 // returned, and both "Res" and "ConstOpnd" remain unchanged.
1316 bool Reassociate::CombineXorOpnd(Instruction *I, XorOpnd *Opnd1, XorOpnd *Opnd2,
1317  APInt &ConstOpnd, Value *&Res) {
1318  Value *X = Opnd1->getSymbolicPart();
1319  if (X != Opnd2->getSymbolicPart())
1320  return false;
1321 
1322  // This many instruction become dead.(At least "Opnd1 ^ Opnd2" will die.)
1323  int DeadInstNum = 1;
1324  if (Opnd1->getValue()->hasOneUse())
1325  DeadInstNum++;
1326  if (Opnd2->getValue()->hasOneUse())
1327  DeadInstNum++;
1328 
1329  // Xor-Rule 2:
1330  // (x | c1) ^ (x & c2)
1331  // = (x|c1) ^ (x&c2) ^ (c1 ^ c1) = ((x|c1) ^ c1) ^ (x & c2) ^ c1
1332  // = (x & ~c1) ^ (x & c2) ^ c1 // Xor-Rule 1
1333  // = (x & c3) ^ c1, where c3 = ~c1 ^ c2 // Xor-rule 3
1334  //
1335  if (Opnd1->isOrExpr() != Opnd2->isOrExpr()) {
1336  if (Opnd2->isOrExpr())
1337  std::swap(Opnd1, Opnd2);
1338 
1339  const APInt &C1 = Opnd1->getConstPart();
1340  const APInt &C2 = Opnd2->getConstPart();
1341  APInt C3((~C1) ^ C2);
1342 
1343  // Do not increase code size!
1344  if (C3 != 0 && !C3.isAllOnesValue()) {
1345  int NewInstNum = ConstOpnd != 0 ? 1 : 2;
1346  if (NewInstNum > DeadInstNum)
1347  return false;
1348  }
1349 
1350  Res = createAndInstr(I, X, C3);
1351  ConstOpnd ^= C1;
1352 
1353  } else if (Opnd1->isOrExpr()) {
1354  // Xor-Rule 3: (x | c1) ^ (x | c2) = (x & c3) ^ c3 where c3 = c1 ^ c2
1355  //
1356  const APInt &C1 = Opnd1->getConstPart();
1357  const APInt &C2 = Opnd2->getConstPart();
1358  APInt C3 = C1 ^ C2;
1359 
1360  // Do not increase code size
1361  if (C3 != 0 && !C3.isAllOnesValue()) {
1362  int NewInstNum = ConstOpnd != 0 ? 1 : 2;
1363  if (NewInstNum > DeadInstNum)
1364  return false;
1365  }
1366 
1367  Res = createAndInstr(I, X, C3);
1368  ConstOpnd ^= C3;
1369  } else {
1370  // Xor-Rule 4: (x & c1) ^ (x & c2) = (x & (c1^c2))
1371  //
1372  const APInt &C1 = Opnd1->getConstPart();
1373  const APInt &C2 = Opnd2->getConstPart();
1374  APInt C3 = C1 ^ C2;
1375  Res = createAndInstr(I, X, C3);
1376  }
1377 
1378  // Put the original operands in the Redo list; hope they will be deleted
1379  // as dead code.
1380  if (Instruction *T = dyn_cast<Instruction>(Opnd1->getValue()))
1381  RedoInsts.insert(T);
1382  if (Instruction *T = dyn_cast<Instruction>(Opnd2->getValue()))
1383  RedoInsts.insert(T);
1384 
1385  return true;
1386 }
1387 
1388 /// Optimize a series of operands to an 'xor' instruction. If it can be reduced
1389 /// to a single Value, it is returned, otherwise the Ops list is mutated as
1390 /// necessary.
1391 Value *Reassociate::OptimizeXor(Instruction *I,
1393  if (Value *V = OptimizeAndOrXor(Instruction::Xor, Ops))
1394  return V;
1395 
1396  if (Ops.size() == 1)
1397  return nullptr;
1398 
1400  SmallVector<XorOpnd*, 8> OpndPtrs;
1401  Type *Ty = Ops[0].Op->getType();
1402  APInt ConstOpnd(Ty->getIntegerBitWidth(), 0);
1403 
1404  // Step 1: Convert ValueEntry to XorOpnd
1405  for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
1406  Value *V = Ops[i].Op;
1407  if (!isa<ConstantInt>(V)) {
1408  XorOpnd O(V);
1409  O.setSymbolicRank(getRank(O.getSymbolicPart()));
1410  Opnds.push_back(O);
1411  } else
1412  ConstOpnd ^= cast<ConstantInt>(V)->getValue();
1413  }
1414 
1415  // NOTE: From this point on, do *NOT* add/delete element to/from "Opnds".
1416  // It would otherwise invalidate the "Opnds"'s iterator, and hence invalidate
1417  // the "OpndPtrs" as well. For the similar reason, do not fuse this loop
1418  // with the previous loop --- the iterator of the "Opnds" may be invalidated
1419  // when new elements are added to the vector.
1420  for (unsigned i = 0, e = Opnds.size(); i != e; ++i)
1421  OpndPtrs.push_back(&Opnds[i]);
1422 
1423  // Step 2: Sort the Xor-Operands in a way such that the operands containing
1424  // the same symbolic value cluster together. For instance, the input operand
1425  // sequence ("x | 123", "y & 456", "x & 789") will be sorted into:
1426  // ("x | 123", "x & 789", "y & 456").
1427  std::stable_sort(OpndPtrs.begin(), OpndPtrs.end(), XorOpnd::PtrSortFunctor());
1428 
1429  // Step 3: Combine adjacent operands
1430  XorOpnd *PrevOpnd = nullptr;
1431  bool Changed = false;
1432  for (unsigned i = 0, e = Opnds.size(); i < e; i++) {
1433  XorOpnd *CurrOpnd = OpndPtrs[i];
1434  // The combined value
1435  Value *CV;
1436 
1437  // Step 3.1: Try simplifying "CurrOpnd ^ ConstOpnd"
1438  if (ConstOpnd != 0 && CombineXorOpnd(I, CurrOpnd, ConstOpnd, CV)) {
1439  Changed = true;
1440  if (CV)
1441  *CurrOpnd = XorOpnd(CV);
1442  else {
1443  CurrOpnd->Invalidate();
1444  continue;
1445  }
1446  }
1447 
1448  if (!PrevOpnd || CurrOpnd->getSymbolicPart() != PrevOpnd->getSymbolicPart()) {
1449  PrevOpnd = CurrOpnd;
1450  continue;
1451  }
1452 
1453  // step 3.2: When previous and current operands share the same symbolic
1454  // value, try to simplify "PrevOpnd ^ CurrOpnd ^ ConstOpnd"
1455  //
1456  if (CombineXorOpnd(I, CurrOpnd, PrevOpnd, ConstOpnd, CV)) {
1457  // Remove previous operand
1458  PrevOpnd->Invalidate();
1459  if (CV) {
1460  *CurrOpnd = XorOpnd(CV);
1461  PrevOpnd = CurrOpnd;
1462  } else {
1463  CurrOpnd->Invalidate();
1464  PrevOpnd = nullptr;
1465  }
1466  Changed = true;
1467  }
1468  }
1469 
1470  // Step 4: Reassemble the Ops
1471  if (Changed) {
1472  Ops.clear();
1473  for (unsigned int i = 0, e = Opnds.size(); i < e; i++) {
1474  XorOpnd &O = Opnds[i];
1475  if (O.isInvalid())
1476  continue;
1477  ValueEntry VE(getRank(O.getValue()), O.getValue());
1478  Ops.push_back(VE);
1479  }
1480  if (ConstOpnd != 0) {
1481  Value *C = ConstantInt::get(Ty->getContext(), ConstOpnd);
1482  ValueEntry VE(getRank(C), C);
1483  Ops.push_back(VE);
1484  }
1485  int Sz = Ops.size();
1486  if (Sz == 1)
1487  return Ops.back().Op;
1488  else if (Sz == 0) {
1489  assert(ConstOpnd == 0);
1490  return ConstantInt::get(Ty->getContext(), ConstOpnd);
1491  }
1492  }
1493 
1494  return nullptr;
1495 }
1496 
1497 /// Optimize a series of operands to an 'add' instruction. This
1498 /// optimizes based on identities. If it can be reduced to a single Value, it
1499 /// is returned, otherwise the Ops list is mutated as necessary.
1500 Value *Reassociate::OptimizeAdd(Instruction *I,
1502  // Scan the operand lists looking for X and -X pairs. If we find any, we
1503  // can simplify expressions like X+-X == 0 and X+~X ==-1. While we're at it,
1504  // scan for any
1505  // duplicates. We want to canonicalize Y+Y+Y+Z -> 3*Y+Z.
1506 
1507  for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
1508  Value *TheOp = Ops[i].Op;
1509  // Check to see if we've seen this operand before. If so, we factor all
1510  // instances of the operand together. Due to our sorting criteria, we know
1511  // that these need to be next to each other in the vector.
1512  if (i+1 != Ops.size() && Ops[i+1].Op == TheOp) {
1513  // Rescan the list, remove all instances of this operand from the expr.
1514  unsigned NumFound = 0;
1515  do {
1516  Ops.erase(Ops.begin()+i);
1517  ++NumFound;
1518  } while (i != Ops.size() && Ops[i].Op == TheOp);
1519 
1520  DEBUG(dbgs() << "\nFACTORING [" << NumFound << "]: " << *TheOp << '\n');
1521  ++NumFactor;
1522 
1523  // Insert a new multiply.
1524  Type *Ty = TheOp->getType();
1525  Constant *C = Ty->isIntOrIntVectorTy() ?
1526  ConstantInt::get(Ty, NumFound) : ConstantFP::get(Ty, NumFound);
1527  Instruction *Mul = CreateMul(TheOp, C, "factor", I, I);
1528 
1529  // Now that we have inserted a multiply, optimize it. This allows us to
1530  // handle cases that require multiple factoring steps, such as this:
1531  // (X*2) + (X*2) + (X*2) -> (X*2)*3 -> X*6
1532  RedoInsts.insert(Mul);
1533 
1534  // If every add operand was a duplicate, return the multiply.
1535  if (Ops.empty())
1536  return Mul;
1537 
1538  // Otherwise, we had some input that didn't have the dupe, such as
1539  // "A + A + B" -> "A*2 + B". Add the new multiply to the list of
1540  // things being added by this operation.
1541  Ops.insert(Ops.begin(), ValueEntry(getRank(Mul), Mul));
1542 
1543  --i;
1544  e = Ops.size();
1545  continue;
1546  }
1547 
1548  // Check for X and -X or X and ~X in the operand list.
1549  if (!BinaryOperator::isNeg(TheOp) && !BinaryOperator::isFNeg(TheOp) &&
1550  !BinaryOperator::isNot(TheOp))
1551  continue;
1552 
1553  Value *X = nullptr;
1554  if (BinaryOperator::isNeg(TheOp) || BinaryOperator::isFNeg(TheOp))
1555  X = BinaryOperator::getNegArgument(TheOp);
1556  else if (BinaryOperator::isNot(TheOp))
1557  X = BinaryOperator::getNotArgument(TheOp);
1558 
1559  unsigned FoundX = FindInOperandList(Ops, i, X);
1560  if (FoundX == i)
1561  continue;
1562 
1563  // Remove X and -X from the operand list.
1564  if (Ops.size() == 2 &&
1565  (BinaryOperator::isNeg(TheOp) || BinaryOperator::isFNeg(TheOp)))
1566  return Constant::getNullValue(X->getType());
1567 
1568  // Remove X and ~X from the operand list.
1569  if (Ops.size() == 2 && BinaryOperator::isNot(TheOp))
1570  return Constant::getAllOnesValue(X->getType());
1571 
1572  Ops.erase(Ops.begin()+i);
1573  if (i < FoundX)
1574  --FoundX;
1575  else
1576  --i; // Need to back up an extra one.
1577  Ops.erase(Ops.begin()+FoundX);
1578  ++NumAnnihil;
1579  --i; // Revisit element.
1580  e -= 2; // Removed two elements.
1581 
1582  // if X and ~X we append -1 to the operand list.
1583  if (BinaryOperator::isNot(TheOp)) {
1585  Ops.insert(Ops.end(), ValueEntry(getRank(V), V));
1586  e += 1;
1587  }
1588  }
1589 
1590  // Scan the operand list, checking to see if there are any common factors
1591  // between operands. Consider something like A*A+A*B*C+D. We would like to
1592  // reassociate this to A*(A+B*C)+D, which reduces the number of multiplies.
1593  // To efficiently find this, we count the number of times a factor occurs
1594  // for any ADD operands that are MULs.
1595  DenseMap<Value*, unsigned> FactorOccurrences;
1596 
1597  // Keep track of each multiply we see, to avoid triggering on (X*4)+(X*4)
1598  // where they are actually the same multiply.
1599  unsigned MaxOcc = 0;
1600  Value *MaxOccVal = nullptr;
1601  for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
1602  BinaryOperator *BOp =
1603  isReassociableOp(Ops[i].Op, Instruction::Mul, Instruction::FMul);
1604  if (!BOp)
1605  continue;
1606 
1607  // Compute all of the factors of this added value.
1608  SmallVector<Value*, 8> Factors;
1609  FindSingleUseMultiplyFactors(BOp, Factors, Ops);
1610  assert(Factors.size() > 1 && "Bad linearize!");
1611 
1612  // Add one to FactorOccurrences for each unique factor in this op.
1613  SmallPtrSet<Value*, 8> Duplicates;
1614  for (unsigned i = 0, e = Factors.size(); i != e; ++i) {
1615  Value *Factor = Factors[i];
1616  if (!Duplicates.insert(Factor).second)
1617  continue;
1618 
1619  unsigned Occ = ++FactorOccurrences[Factor];
1620  if (Occ > MaxOcc) {
1621  MaxOcc = Occ;
1622  MaxOccVal = Factor;
1623  }
1624 
1625  // If Factor is a negative constant, add the negated value as a factor
1626  // because we can percolate the negate out. Watch for minint, which
1627  // cannot be positivified.
1628  if (ConstantInt *CI = dyn_cast<ConstantInt>(Factor)) {
1629  if (CI->isNegative() && !CI->isMinValue(true)) {
1630  Factor = ConstantInt::get(CI->getContext(), -CI->getValue());
1631  assert(!Duplicates.count(Factor) &&
1632  "Shouldn't have two constant factors, missed a canonicalize");
1633  unsigned Occ = ++FactorOccurrences[Factor];
1634  if (Occ > MaxOcc) {
1635  MaxOcc = Occ;
1636  MaxOccVal = Factor;
1637  }
1638  }
1639  } else if (ConstantFP *CF = dyn_cast<ConstantFP>(Factor)) {
1640  if (CF->isNegative()) {
1641  APFloat F(CF->getValueAPF());
1642  F.changeSign();
1643  Factor = ConstantFP::get(CF->getContext(), F);
1644  assert(!Duplicates.count(Factor) &&
1645  "Shouldn't have two constant factors, missed a canonicalize");
1646  unsigned Occ = ++FactorOccurrences[Factor];
1647  if (Occ > MaxOcc) {
1648  MaxOcc = Occ;
1649  MaxOccVal = Factor;
1650  }
1651  }
1652  }
1653  }
1654  }
1655 
1656  // If any factor occurred more than one time, we can pull it out.
1657  if (MaxOcc > 1) {
1658  DEBUG(dbgs() << "\nFACTORING [" << MaxOcc << "]: " << *MaxOccVal << '\n');
1659  ++NumFactor;
1660 
1661  // Create a new instruction that uses the MaxOccVal twice. If we don't do
1662  // this, we could otherwise run into situations where removing a factor
1663  // from an expression will drop a use of maxocc, and this can cause
1664  // RemoveFactorFromExpression on successive values to behave differently.
1665  Instruction *DummyInst =
1666  I->getType()->isIntOrIntVectorTy()
1667  ? BinaryOperator::CreateAdd(MaxOccVal, MaxOccVal)
1668  : BinaryOperator::CreateFAdd(MaxOccVal, MaxOccVal);
1669 
1670  SmallVector<WeakVH, 4> NewMulOps;
1671  for (unsigned i = 0; i != Ops.size(); ++i) {
1672  // Only try to remove factors from expressions we're allowed to.
1673  BinaryOperator *BOp =
1674  isReassociableOp(Ops[i].Op, Instruction::Mul, Instruction::FMul);
1675  if (!BOp)
1676  continue;
1677 
1678  if (Value *V = RemoveFactorFromExpression(Ops[i].Op, MaxOccVal)) {
1679  // The factorized operand may occur several times. Convert them all in
1680  // one fell swoop.
1681  for (unsigned j = Ops.size(); j != i;) {
1682  --j;
1683  if (Ops[j].Op == Ops[i].Op) {
1684  NewMulOps.push_back(V);
1685  Ops.erase(Ops.begin()+j);
1686  }
1687  }
1688  --i;
1689  }
1690  }
1691 
1692  // No need for extra uses anymore.
1693  delete DummyInst;
1694 
1695  unsigned NumAddedValues = NewMulOps.size();
1696  Value *V = EmitAddTreeOfValues(I, NewMulOps);
1697 
1698  // Now that we have inserted the add tree, optimize it. This allows us to
1699  // handle cases that require multiple factoring steps, such as this:
1700  // A*A*B + A*A*C --> A*(A*B+A*C) --> A*(A*(B+C))
1701  assert(NumAddedValues > 1 && "Each occurrence should contribute a value");
1702  (void)NumAddedValues;
1703  if (Instruction *VI = dyn_cast<Instruction>(V))
1704  RedoInsts.insert(VI);
1705 
1706  // Create the multiply.
1707  Instruction *V2 = CreateMul(V, MaxOccVal, "tmp", I, I);
1708 
1709  // Rerun associate on the multiply in case the inner expression turned into
1710  // a multiply. We want to make sure that we keep things in canonical form.
1711  RedoInsts.insert(V2);
1712 
1713  // If every add operand included the factor (e.g. "A*B + A*C"), then the
1714  // entire result expression is just the multiply "A*(B+C)".
1715  if (Ops.empty())
1716  return V2;
1717 
1718  // Otherwise, we had some input that didn't have the factor, such as
1719  // "A*B + A*C + D" -> "A*(B+C) + D". Add the new multiply to the list of
1720  // things being added by this operation.
1721  Ops.insert(Ops.begin(), ValueEntry(getRank(V2), V2));
1722  }
1723 
1724  return nullptr;
1725 }
1726 
1727 /// \brief Build up a vector of value/power pairs factoring a product.
1728 ///
1729 /// Given a series of multiplication operands, build a vector of factors and
1730 /// the powers each is raised to when forming the final product. Sort them in
1731 /// the order of descending power.
1732 ///
1733 /// (x*x) -> [(x, 2)]
1734 /// ((x*x)*x) -> [(x, 3)]
1735 /// ((((x*y)*x)*y)*x) -> [(x, 3), (y, 2)]
1736 ///
1737 /// \returns Whether any factors have a power greater than one.
1738 bool Reassociate::collectMultiplyFactors(SmallVectorImpl<ValueEntry> &Ops,
1739  SmallVectorImpl<Factor> &Factors) {
1740  // FIXME: Have Ops be (ValueEntry, Multiplicity) pairs, simplifying this.
1741  // Compute the sum of powers of simplifiable factors.
1742  unsigned FactorPowerSum = 0;
1743  for (unsigned Idx = 1, Size = Ops.size(); Idx < Size; ++Idx) {
1744  Value *Op = Ops[Idx-1].Op;
1745 
1746  // Count the number of occurrences of this value.
1747  unsigned Count = 1;
1748  for (; Idx < Size && Ops[Idx].Op == Op; ++Idx)
1749  ++Count;
1750  // Track for simplification all factors which occur 2 or more times.
1751  if (Count > 1)
1752  FactorPowerSum += Count;
1753  }
1754 
1755  // We can only simplify factors if the sum of the powers of our simplifiable
1756  // factors is 4 or higher. When that is the case, we will *always* have
1757  // a simplification. This is an important invariant to prevent cyclicly
1758  // trying to simplify already minimal formations.
1759  if (FactorPowerSum < 4)
1760  return false;
1761 
1762  // Now gather the simplifiable factors, removing them from Ops.
1763  FactorPowerSum = 0;
1764  for (unsigned Idx = 1; Idx < Ops.size(); ++Idx) {
1765  Value *Op = Ops[Idx-1].Op;
1766 
1767  // Count the number of occurrences of this value.
1768  unsigned Count = 1;
1769  for (; Idx < Ops.size() && Ops[Idx].Op == Op; ++Idx)
1770  ++Count;
1771  if (Count == 1)
1772  continue;
1773  // Move an even number of occurrences to Factors.
1774  Count &= ~1U;
1775  Idx -= Count;
1776  FactorPowerSum += Count;
1777  Factors.push_back(Factor(Op, Count));
1778  Ops.erase(Ops.begin()+Idx, Ops.begin()+Idx+Count);
1779  }
1780 
1781  // None of the adjustments above should have reduced the sum of factor powers
1782  // below our mininum of '4'.
1783  assert(FactorPowerSum >= 4);
1784 
1785  std::stable_sort(Factors.begin(), Factors.end(), Factor::PowerDescendingSorter());
1786  return true;
1787 }
1788 
1789 /// \brief Build a tree of multiplies, computing the product of Ops.
1791  SmallVectorImpl<Value*> &Ops) {
1792  if (Ops.size() == 1)
1793  return Ops.back();
1794 
1795  Value *LHS = Ops.pop_back_val();
1796  do {
1797  if (LHS->getType()->isIntOrIntVectorTy())
1798  LHS = Builder.CreateMul(LHS, Ops.pop_back_val());
1799  else
1800  LHS = Builder.CreateFMul(LHS, Ops.pop_back_val());
1801  } while (!Ops.empty());
1802 
1803  return LHS;
1804 }
1805 
1806 /// \brief Build a minimal multiplication DAG for (a^x)*(b^y)*(c^z)*...
1807 ///
1808 /// Given a vector of values raised to various powers, where no two values are
1809 /// equal and the powers are sorted in decreasing order, compute the minimal
1810 /// DAG of multiplies to compute the final product, and return that product
1811 /// value.
1812 Value *Reassociate::buildMinimalMultiplyDAG(IRBuilder<> &Builder,
1813  SmallVectorImpl<Factor> &Factors) {
1814  assert(Factors[0].Power);
1815  SmallVector<Value *, 4> OuterProduct;
1816  for (unsigned LastIdx = 0, Idx = 1, Size = Factors.size();
1817  Idx < Size && Factors[Idx].Power > 0; ++Idx) {
1818  if (Factors[Idx].Power != Factors[LastIdx].Power) {
1819  LastIdx = Idx;
1820  continue;
1821  }
1822 
1823  // We want to multiply across all the factors with the same power so that
1824  // we can raise them to that power as a single entity. Build a mini tree
1825  // for that.
1826  SmallVector<Value *, 4> InnerProduct;
1827  InnerProduct.push_back(Factors[LastIdx].Base);
1828  do {
1829  InnerProduct.push_back(Factors[Idx].Base);
1830  ++Idx;
1831  } while (Idx < Size && Factors[Idx].Power == Factors[LastIdx].Power);
1832 
1833  // Reset the base value of the first factor to the new expression tree.
1834  // We'll remove all the factors with the same power in a second pass.
1835  Value *M = Factors[LastIdx].Base = buildMultiplyTree(Builder, InnerProduct);
1836  if (Instruction *MI = dyn_cast<Instruction>(M))
1837  RedoInsts.insert(MI);
1838 
1839  LastIdx = Idx;
1840  }
1841  // Unique factors with equal powers -- we've folded them into the first one's
1842  // base.
1843  Factors.erase(std::unique(Factors.begin(), Factors.end(),
1844  Factor::PowerEqual()),
1845  Factors.end());
1846 
1847  // Iteratively collect the base of each factor with an add power into the
1848  // outer product, and halve each power in preparation for squaring the
1849  // expression.
1850  for (unsigned Idx = 0, Size = Factors.size(); Idx != Size; ++Idx) {
1851  if (Factors[Idx].Power & 1)
1852  OuterProduct.push_back(Factors[Idx].Base);
1853  Factors[Idx].Power >>= 1;
1854  }
1855  if (Factors[0].Power) {
1856  Value *SquareRoot = buildMinimalMultiplyDAG(Builder, Factors);
1857  OuterProduct.push_back(SquareRoot);
1858  OuterProduct.push_back(SquareRoot);
1859  }
1860  if (OuterProduct.size() == 1)
1861  return OuterProduct.front();
1862 
1863  Value *V = buildMultiplyTree(Builder, OuterProduct);
1864  return V;
1865 }
1866 
1867 Value *Reassociate::OptimizeMul(BinaryOperator *I,
1869  // We can only optimize the multiplies when there is a chain of more than
1870  // three, such that a balanced tree might require fewer total multiplies.
1871  if (Ops.size() < 4)
1872  return nullptr;
1873 
1874  // Try to turn linear trees of multiplies without other uses of the
1875  // intermediate stages into minimal multiply DAGs with perfect sub-expression
1876  // re-use.
1877  SmallVector<Factor, 4> Factors;
1878  if (!collectMultiplyFactors(Ops, Factors))
1879  return nullptr; // All distinct factors, so nothing left for us to do.
1880 
1881  IRBuilder<> Builder(I);
1882  Value *V = buildMinimalMultiplyDAG(Builder, Factors);
1883  if (Ops.empty())
1884  return V;
1885 
1886  ValueEntry NewEntry = ValueEntry(getRank(V), V);
1887  Ops.insert(std::lower_bound(Ops.begin(), Ops.end(), NewEntry), NewEntry);
1888  return nullptr;
1889 }
1890 
1891 Value *Reassociate::OptimizeExpression(BinaryOperator *I,
1893  // Now that we have the linearized expression tree, try to optimize it.
1894  // Start by folding any constants that we found.
1895  Constant *Cst = nullptr;
1896  unsigned Opcode = I->getOpcode();
1897  while (!Ops.empty() && isa<Constant>(Ops.back().Op)) {
1898  Constant *C = cast<Constant>(Ops.pop_back_val().Op);
1899  Cst = Cst ? ConstantExpr::get(Opcode, C, Cst) : C;
1900  }
1901  // If there was nothing but constants then we are done.
1902  if (Ops.empty())
1903  return Cst;
1904 
1905  // Put the combined constant back at the end of the operand list, except if
1906  // there is no point. For example, an add of 0 gets dropped here, while a
1907  // multiplication by zero turns the whole expression into zero.
1908  if (Cst && Cst != ConstantExpr::getBinOpIdentity(Opcode, I->getType())) {
1909  if (Cst == ConstantExpr::getBinOpAbsorber(Opcode, I->getType()))
1910  return Cst;
1911  Ops.push_back(ValueEntry(0, Cst));
1912  }
1913 
1914  if (Ops.size() == 1) return Ops[0].Op;
1915 
1916  // Handle destructive annihilation due to identities between elements in the
1917  // argument list here.
1918  unsigned NumOps = Ops.size();
1919  switch (Opcode) {
1920  default: break;
1921  case Instruction::And:
1922  case Instruction::Or:
1923  if (Value *Result = OptimizeAndOrXor(Opcode, Ops))
1924  return Result;
1925  break;
1926 
1927  case Instruction::Xor:
1928  if (Value *Result = OptimizeXor(I, Ops))
1929  return Result;
1930  break;
1931 
1932  case Instruction::Add:
1933  case Instruction::FAdd:
1934  if (Value *Result = OptimizeAdd(I, Ops))
1935  return Result;
1936  break;
1937 
1938  case Instruction::Mul:
1939  case Instruction::FMul:
1940  if (Value *Result = OptimizeMul(I, Ops))
1941  return Result;
1942  break;
1943  }
1944 
1945  if (Ops.size() != NumOps)
1946  return OptimizeExpression(I, Ops);
1947  return nullptr;
1948 }
1949 
1950 /// Zap the given instruction, adding interesting operands to the work list.
1951 void Reassociate::EraseInst(Instruction *I) {
1952  assert(isInstructionTriviallyDead(I) && "Trivially dead instructions only!");
1953  SmallVector<Value*, 8> Ops(I->op_begin(), I->op_end());
1954  // Erase the dead instruction.
1955  ValueRankMap.erase(I);
1956  RedoInsts.remove(I);
1957  I->eraseFromParent();
1958  // Optimize its operands.
1959  SmallPtrSet<Instruction *, 8> Visited; // Detect self-referential nodes.
1960  for (unsigned i = 0, e = Ops.size(); i != e; ++i)
1961  if (Instruction *Op = dyn_cast<Instruction>(Ops[i])) {
1962  // If this is a node in an expression tree, climb to the expression root
1963  // and add that since that's where optimization actually happens.
1964  unsigned Opcode = Op->getOpcode();
1965  while (Op->hasOneUse() && Op->user_back()->getOpcode() == Opcode &&
1966  Visited.insert(Op).second)
1967  Op = Op->user_back();
1968  RedoInsts.insert(Op);
1969  }
1970 }
1971 
1972 // Canonicalize expressions of the following form:
1973 // x + (-Constant * y) -> x - (Constant * y)
1974 // x - (-Constant * y) -> x + (Constant * y)
1975 Instruction *Reassociate::canonicalizeNegConstExpr(Instruction *I) {
1976  if (!I->hasOneUse() || I->getType()->isVectorTy())
1977  return nullptr;
1978 
1979  // Must be a fmul or fdiv instruction.
1980  unsigned Opcode = I->getOpcode();
1981  if (Opcode != Instruction::FMul && Opcode != Instruction::FDiv)
1982  return nullptr;
1983 
1984  auto *C0 = dyn_cast<ConstantFP>(I->getOperand(0));
1985  auto *C1 = dyn_cast<ConstantFP>(I->getOperand(1));
1986 
1987  // Both operands are constant, let it get constant folded away.
1988  if (C0 && C1)
1989  return nullptr;
1990 
1991  ConstantFP *CF = C0 ? C0 : C1;
1992 
1993  // Must have one constant operand.
1994  if (!CF)
1995  return nullptr;
1996 
1997  // Must be a negative ConstantFP.
1998  if (!CF->isNegative())
1999  return nullptr;
2000 
2001  // User must be a binary operator with one or more uses.
2002  Instruction *User = I->user_back();
2003  if (!isa<BinaryOperator>(User) || !User->hasNUsesOrMore(1))
2004  return nullptr;
2005 
2006  unsigned UserOpcode = User->getOpcode();
2007  if (UserOpcode != Instruction::FAdd && UserOpcode != Instruction::FSub)
2008  return nullptr;
2009 
2010  // Subtraction is not commutative. Explicitly, the following transform is
2011  // not valid: (-Constant * y) - x -> x + (Constant * y)
2012  if (!User->isCommutative() && User->getOperand(1) != I)
2013  return nullptr;
2014 
2015  // Change the sign of the constant.
2016  APFloat Val = CF->getValueAPF();
2017  Val.changeSign();
2018  I->setOperand(C0 ? 0 : 1, ConstantFP::get(CF->getContext(), Val));
2019 
2020  // Canonicalize I to RHS to simplify the next bit of logic. E.g.,
2021  // ((-Const*y) + x) -> (x + (-Const*y)).
2022  if (User->getOperand(0) == I && User->isCommutative())
2023  cast<BinaryOperator>(User)->swapOperands();
2024 
2025  Value *Op0 = User->getOperand(0);
2026  Value *Op1 = User->getOperand(1);
2027  BinaryOperator *NI;
2028  switch (UserOpcode) {
2029  default:
2030  llvm_unreachable("Unexpected Opcode!");
2031  case Instruction::FAdd:
2032  NI = BinaryOperator::CreateFSub(Op0, Op1);
2033  NI->setFastMathFlags(cast<FPMathOperator>(User)->getFastMathFlags());
2034  break;
2035  case Instruction::FSub:
2036  NI = BinaryOperator::CreateFAdd(Op0, Op1);
2037  NI->setFastMathFlags(cast<FPMathOperator>(User)->getFastMathFlags());
2038  break;
2039  }
2040 
2041  NI->insertBefore(User);
2042  NI->setName(User->getName());
2043  User->replaceAllUsesWith(NI);
2044  NI->setDebugLoc(I->getDebugLoc());
2045  RedoInsts.insert(I);
2046  MadeChange = true;
2047  return NI;
2048 }
2049 
2050 /// Inspect and optimize the given instruction. Note that erasing
2051 /// instructions is not allowed.
2052 void Reassociate::OptimizeInst(Instruction *I) {
2053  // Only consider operations that we understand.
2054  if (!isa<BinaryOperator>(I))
2055  return;
2056 
2057  if (I->getOpcode() == Instruction::Shl && isa<ConstantInt>(I->getOperand(1)))
2058  // If an operand of this shift is a reassociable multiply, or if the shift
2059  // is used by a reassociable multiply or add, turn into a multiply.
2060  if (isReassociableOp(I->getOperand(0), Instruction::Mul) ||
2061  (I->hasOneUse() &&
2062  (isReassociableOp(I->user_back(), Instruction::Mul) ||
2063  isReassociableOp(I->user_back(), Instruction::Add)))) {
2064  Instruction *NI = ConvertShiftToMul(I);
2065  RedoInsts.insert(I);
2066  MadeChange = true;
2067  I = NI;
2068  }
2069 
2070  // Canonicalize negative constants out of expressions.
2071  if (Instruction *Res = canonicalizeNegConstExpr(I))
2072  I = Res;
2073 
2074  // Commute binary operators, to canonicalize the order of their operands.
2075  // This can potentially expose more CSE opportunities, and makes writing other
2076  // transformations simpler.
2077  if (I->isCommutative())
2078  canonicalizeOperands(I);
2079 
2080  // TODO: We should optimize vector Xor instructions, but they are
2081  // currently unsupported.
2082  if (I->getType()->isVectorTy() && I->getOpcode() == Instruction::Xor)
2083  return;
2084 
2085  // Don't optimize floating point instructions that don't have unsafe algebra.
2086  if (I->getType()->isFloatingPointTy() && !I->hasUnsafeAlgebra())
2087  return;
2088 
2089  // Do not reassociate boolean (i1) expressions. We want to preserve the
2090  // original order of evaluation for short-circuited comparisons that
2091  // SimplifyCFG has folded to AND/OR expressions. If the expression
2092  // is not further optimized, it is likely to be transformed back to a
2093  // short-circuited form for code gen, and the source order may have been
2094  // optimized for the most likely conditions.
2095  if (I->getType()->isIntegerTy(1))
2096  return;
2097 
2098  // If this is a subtract instruction which is not already in negate form,
2099  // see if we can convert it to X+-Y.
2100  if (I->getOpcode() == Instruction::Sub) {
2101  if (ShouldBreakUpSubtract(I)) {
2102  Instruction *NI = BreakUpSubtract(I);
2103  RedoInsts.insert(I);
2104  MadeChange = true;
2105  I = NI;
2106  } else if (BinaryOperator::isNeg(I)) {
2107  // Otherwise, this is a negation. See if the operand is a multiply tree
2108  // and if this is not an inner node of a multiply tree.
2109  if (isReassociableOp(I->getOperand(1), Instruction::Mul) &&
2110  (!I->hasOneUse() ||
2111  !isReassociableOp(I->user_back(), Instruction::Mul))) {
2113  RedoInsts.insert(I);
2114  MadeChange = true;
2115  I = NI;
2116  }
2117  }
2118  } else if (I->getOpcode() == Instruction::FSub) {
2119  if (ShouldBreakUpSubtract(I)) {
2120  Instruction *NI = BreakUpSubtract(I);
2121  RedoInsts.insert(I);
2122  MadeChange = true;
2123  I = NI;
2124  } else if (BinaryOperator::isFNeg(I)) {
2125  // Otherwise, this is a negation. See if the operand is a multiply tree
2126  // and if this is not an inner node of a multiply tree.
2127  if (isReassociableOp(I->getOperand(1), Instruction::FMul) &&
2128  (!I->hasOneUse() ||
2129  !isReassociableOp(I->user_back(), Instruction::FMul))) {
2131  RedoInsts.insert(I);
2132  MadeChange = true;
2133  I = NI;
2134  }
2135  }
2136  }
2137 
2138  // If this instruction is an associative binary operator, process it.
2139  if (!I->isAssociative()) return;
2140  BinaryOperator *BO = cast<BinaryOperator>(I);
2141 
2142  // If this is an interior node of a reassociable tree, ignore it until we
2143  // get to the root of the tree, to avoid N^2 analysis.
2144  unsigned Opcode = BO->getOpcode();
2145  if (BO->hasOneUse() && BO->user_back()->getOpcode() == Opcode)
2146  return;
2147 
2148  // If this is an add tree that is used by a sub instruction, ignore it
2149  // until we process the subtract.
2150  if (BO->hasOneUse() && BO->getOpcode() == Instruction::Add &&
2151  cast<Instruction>(BO->user_back())->getOpcode() == Instruction::Sub)
2152  return;
2153  if (BO->hasOneUse() && BO->getOpcode() == Instruction::FAdd &&
2154  cast<Instruction>(BO->user_back())->getOpcode() == Instruction::FSub)
2155  return;
2156 
2157  ReassociateExpression(BO);
2158 }
2159 
2160 void Reassociate::ReassociateExpression(BinaryOperator *I) {
2161  // First, walk the expression tree, linearizing the tree, collecting the
2162  // operand information.
2164  MadeChange |= LinearizeExprTree(I, Tree);
2166  Ops.reserve(Tree.size());
2167  for (unsigned i = 0, e = Tree.size(); i != e; ++i) {
2168  RepeatedValue E = Tree[i];
2169  Ops.append(E.second.getZExtValue(),
2170  ValueEntry(getRank(E.first), E.first));
2171  }
2172 
2173  DEBUG(dbgs() << "RAIn:\t"; PrintOps(I, Ops); dbgs() << '\n');
2174 
2175  // Now that we have linearized the tree to a list and have gathered all of
2176  // the operands and their ranks, sort the operands by their rank. Use a
2177  // stable_sort so that values with equal ranks will have their relative
2178  // positions maintained (and so the compiler is deterministic). Note that
2179  // this sorts so that the highest ranking values end up at the beginning of
2180  // the vector.
2181  std::stable_sort(Ops.begin(), Ops.end());
2182 
2183  // Now that we have the expression tree in a convenient
2184  // sorted form, optimize it globally if possible.
2185  if (Value *V = OptimizeExpression(I, Ops)) {
2186  if (V == I)
2187  // Self-referential expression in unreachable code.
2188  return;
2189  // This expression tree simplified to something that isn't a tree,
2190  // eliminate it.
2191  DEBUG(dbgs() << "Reassoc to scalar: " << *V << '\n');
2192  I->replaceAllUsesWith(V);
2193  if (Instruction *VI = dyn_cast<Instruction>(V))
2194  VI->setDebugLoc(I->getDebugLoc());
2195  RedoInsts.insert(I);
2196  ++NumAnnihil;
2197  return;
2198  }
2199 
2200  // We want to sink immediates as deeply as possible except in the case where
2201  // this is a multiply tree used only by an add, and the immediate is a -1.
2202  // In this case we reassociate to put the negation on the outside so that we
2203  // can fold the negation into the add: (-X)*Y + Z -> Z-X*Y
2204  if (I->hasOneUse()) {
2205  if (I->getOpcode() == Instruction::Mul &&
2206  cast<Instruction>(I->user_back())->getOpcode() == Instruction::Add &&
2207  isa<ConstantInt>(Ops.back().Op) &&
2208  cast<ConstantInt>(Ops.back().Op)->isAllOnesValue()) {
2209  ValueEntry Tmp = Ops.pop_back_val();
2210  Ops.insert(Ops.begin(), Tmp);
2211  } else if (I->getOpcode() == Instruction::FMul &&
2212  cast<Instruction>(I->user_back())->getOpcode() ==
2213  Instruction::FAdd &&
2214  isa<ConstantFP>(Ops.back().Op) &&
2215  cast<ConstantFP>(Ops.back().Op)->isExactlyValue(-1.0)) {
2216  ValueEntry Tmp = Ops.pop_back_val();
2217  Ops.insert(Ops.begin(), Tmp);
2218  }
2219  }
2220 
2221  DEBUG(dbgs() << "RAOut:\t"; PrintOps(I, Ops); dbgs() << '\n');
2222 
2223  if (Ops.size() == 1) {
2224  if (Ops[0].Op == I)
2225  // Self-referential expression in unreachable code.
2226  return;
2227 
2228  // This expression tree simplified to something that isn't a tree,
2229  // eliminate it.
2230  I->replaceAllUsesWith(Ops[0].Op);
2231  if (Instruction *OI = dyn_cast<Instruction>(Ops[0].Op))
2232  OI->setDebugLoc(I->getDebugLoc());
2233  RedoInsts.insert(I);
2234  return;
2235  }
2236 
2237  // Now that we ordered and optimized the expressions, splat them back into
2238  // the expression tree, removing any unneeded nodes.
2239  RewriteExprTree(I, Ops);
2240 }
2241 
2242 bool Reassociate::runOnFunction(Function &F) {
2243  if (skipOptnoneFunction(F))
2244  return false;
2245 
2246  // Calculate the rank map for F
2247  BuildRankMap(F);
2248 
2249  MadeChange = false;
2250  for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) {
2251  // Optimize every instruction in the basic block.
2252  for (BasicBlock::iterator II = BI->begin(), IE = BI->end(); II != IE; )
2253  if (isInstructionTriviallyDead(II)) {
2254  EraseInst(II++);
2255  } else {
2256  OptimizeInst(II);
2257  assert(II->getParent() == BI && "Moved to a different block!");
2258  ++II;
2259  }
2260 
2261  // If this produced extra instructions to optimize, handle them now.
2262  while (!RedoInsts.empty()) {
2263  Instruction *I = RedoInsts.pop_back_val();
2265  EraseInst(I);
2266  else
2267  OptimizeInst(I);
2268  }
2269  }
2270 
2271  // We are done with the rank map.
2272  RankMap.clear();
2273  ValueRankMap.clear();
2274 
2275  return MadeChange;
2276 }
bool isNilpotent() const
isNilpotent - Return true if the instruction is nilpotent:
Definition: Instruction.h:348
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
Definition: Instruction.cpp:70
void push_back(const T &Elt)
Definition: SmallVector.h:222
void setHasNoSignedWrap(bool b=true)
Set or clear the nsw flag on this instruction, which must be an operator which supports this flag...
void setFastMathFlags(FastMathFlags FMF)
Convenience function for setting multiple fast-math flags on this instruction, which must be an opera...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1327
STATISTIC(NumFunctions,"Total number of functions")
static BinaryOperator * LowerNegateToMultiply(Instruction *Neg)
Replace 0-X with X*-1.
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:114
iterator end()
Definition: Function.h:459
static Constant * getBinOpIdentity(unsigned Opcode, Type *Ty)
getBinOpIdentity - Return the identity for the given binary operation, i.e.
Definition: Constants.cpp:2349
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:276
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:111
arg_iterator arg_end()
Definition: Function.h:480
F(f)
void reserve(size_type N)
Definition: SmallVector.h:401
op_iterator op_begin()
Definition: User.h:183
static void FindSingleUseMultiplyFactors(Value *V, SmallVectorImpl< Value * > &Factors, const SmallVectorImpl< ValueEntry > &Ops)
If V is a single-use multiply, recursively add its operands as factors, otherwise add V to the list o...
static Constant * getNullValue(Type *Ty)
Definition: Constants.cpp:178
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:188
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:231
bool swapOperands()
Exchange the two operands to this instruction.
static const Value * getNegArgument(const Value *BinOp)
Helper functions to extract the unary argument of a NEG, FNEG or NOT operation implemented via Sub...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:79
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
Definition: SmallVector.h:406
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
void setHasNoUnsignedWrap(bool b=true)
Set or clear the nsw flag on this instruction, which must be an operator which supports this flag...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
static bool isUnmovableInstruction(Instruction *I)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:517
FunctionPass * createReassociatePass()
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:250
static Constant * get(unsigned Opcode, Constant *C1, Constant *C2, unsigned Flags=0, Type *OnlyIfReducedTy=nullptr)
get - Return a binary or shift operator constant expression, folding if possible. ...
Definition: Constants.cpp:1868
static Value * OptimizeAndOrXor(unsigned Opcode, SmallVectorImpl< ValueEntry > &Ops)
Optimize a series of operands to an 'and', 'or', or 'xor' instruction.
LLVMContext & getContext() const
getContext - Return the LLVMContext in which this type was uniqued.
Definition: Type.h:125
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
std::pair< Value *, APInt > RepeatedValue
bool isAssociative() const
isAssociative - Return true if the instruction is associative:
bool hasUnsafeAlgebra() const
Determine whether the unsafe-algebra flag is set.
static Value * createAndInstr(Instruction *InsertBefore, Value *Opnd, const APInt &ConstOpnd)
Helper funciton of CombineXorOpnd().
bool isFloatingPointTy() const
isFloatingPointTy - Return true if this is one of the six floating point types
Definition: Type.h:159
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:351
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:256
iterator begin()
Definition: Function.h:457
const char * getOpcodeName() const
Definition: Instruction.h:114
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, Instruction *InsertBefore, Value *FlagsOp)
static const Value * getNotArgument(const Value *BinOp)
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.cpp:520
A self-contained host- and target-independent arbitrary-precision floating-point software implementat...
Definition: APFloat.h:122
#define P(N)
bool isIntOrIntVectorTy() const
isIntOrIntVectorTy - Return true if this is an integer type or a vector of integer types...
Definition: Type.h:201
static Constant * getFNeg(Constant *C)
Definition: Constants.cpp:2246
static Value * buildMultiplyTree(IRBuilder<> &Builder, SmallVectorImpl< Value * > &Ops)
Build a tree of multiplies, computing the product of Ops.
void setDebugLoc(DebugLoc Loc)
setDebugLoc - Set the debug location information for this instruction.
Definition: Instruction.h:227
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
Definition: Instruction.cpp:76
INITIALIZE_PASS(Reassociate,"reassociate","Reassociate expressions", false, false) FunctionPass *llvm
LLVM Basic Block Representation.
Definition: BasicBlock.h:65
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:41
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
Definition: Type.h:226
This is an important base class in LLVM.
Definition: Constant.h:41
static Value * NegateValue(Value *V, Instruction *BI)
Insert instructions before the instruction pointed to by BI, that computes the negative version of th...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1895
static Value * EmitAddTreeOfValues(Instruction *I, SmallVectorImpl< WeakVH > &Ops)
Emit a tree of add instructions, summing Ops together and returning the result.
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:233
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
Definition: APInt.h:1900
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:479
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:230
Represent the analysis usage information of a pass.
op_iterator op_end()
Definition: User.h:185
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1273
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:294
Value * getOperand(unsigned i) const
Definition: User.h:118
bool isCommutative() const
isCommutative - Return true if the instruction is commutative:
Definition: Instruction.h:327
arg_iterator arg_begin()
Definition: Function.h:472
static bool isNot(const Value *V)
void changeSign()
Definition: APFloat.cpp:1623
static Constant * getAllOnesValue(Type *Ty)
Get the all ones value.
Definition: Constants.cpp:230
NUW NUW NUW NUW Exact static Exact BinaryOperator * CreateNeg(Value *Op, const Twine &Name="", Instruction *InsertBefore=nullptr)
Helper functions to construct and inspect unary operations (NEG and NOT) via binary operators SUB and...
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:416
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1473
void andIRFlags(const Value *V)
Logical 'and' of any supported wrapping, exact, and fast-math flags of V and this instruction...
iterator erase(iterator I)
Definition: SmallVector.h:455
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:519
static unsigned CarmichaelShift(unsigned Bitwidth)
Returns k such that lambda(2^Bitwidth) = 2^k, where lambda is the Carmichael function.
std::vector< NodeType * >::reverse_iterator rpo_iterator
BinaryOps getOpcode() const
Definition: InstrTypes.h:323
static BinaryOperator * CreateFNeg(Value *Op, const Twine &Name="", Instruction *InsertBefore=nullptr)
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
unsigned getIntegerBitWidth() const
Definition: Type.cpp:176
This is the shared class of boolean and integer constants.
Definition: Constants.h:47
iterator end()
Definition: BasicBlock.h:233
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
Instruction * user_back()
user_back - Specialize the methods defined in Value, as we know that an instruction can only be used ...
Definition: Instruction.h:69
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:748
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:582
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:263
const BasicBlock & getEntryBlock() const
Definition: Function.h:442
static Constant * get(Type *Ty, double V)
get() - This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in the specified type.
Definition: Constants.cpp:652
void setOperand(unsigned i, Value *Val)
Definition: User.h:122
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
void initializeReassociatePass(PassRegistry &)
static bool isNeg(const Value *V)
Check if the given Value is a NEG, FNeg, or NOT instruction.
Class for arbitrary precision integers.
Definition: APInt.h:73
bool isIntegerTy() const
isIntegerTy - True if this is an instance of IntegerType.
Definition: Type.h:193
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), Instruction *InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
iterator_range< user_iterator > users()
Definition: Value.h:300
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:481
bool isMinValue() const
Determine if this is the smallest unsigned value.
Definition: APInt.h:361
const Type * getScalarType() const LLVM_READONLY
getScalarType - If this is a vector type, return the element type, otherwise return 'this'...
Definition: Type.cpp:51
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1890
static Constant * getNeg(Constant *C, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2239
bool isAllOnesValue() const
Determine if all bits are set.
Definition: APInt.h:337
bool isNegative() const
isNegative - Return true if the sign bit is set.
Definition: Constants.h:276
void emplace_back(ArgTypes &&...Args)
Definition: SmallVector.h:652
static BinaryOperator * BreakUpSubtract(Instruction *Sub)
If we have (X-Y), and if either X is an add, or if this is only used by an add, transform this into (...
static bool isFNeg(const Value *V, bool IgnoreZeroSign=false)
bool isIdempotent() const
isIdempotent - Return true if the instruction is idempotent:
Definition: Instruction.h:336
static void PrintOps(Instruction *I, const SmallVectorImpl< ValueEntry > &Ops)
Print out the expression identified in the Ops list.
Definition: Reassociate.cpp:64
static void IncorporateWeight(APInt &LHS, const APInt &RHS, unsigned Opcode)
Add the extra weight 'RHS' to the existing weight 'LHS', reducing the combined weight using any speci...
#define I(x, y, z)
Definition: MD5.cpp:54
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:311
static Constant * getShl(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2329
static BinaryOperator * CreateNeg(Value *S1, const Twine &Name, Instruction *InsertBefore, Value *FlagsOp)
bool hasNUsesOrMore(unsigned N) const
Return true if this value has N users or more.
Definition: Value.cpp:104
const APFloat & getValueAPF() const
Definition: Constants.h:270
static int const Threshold
TODO: Write a new FunctionPass AliasAnalysis so that it can keep a cache.
Value * CreateFMul(Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:762
static BinaryOperator * ConvertShiftToMul(Instruction *Shl)
If this is a shift of a reassociable multiply or is used by one, change this into a multiply by a con...
bool use_empty() const
Definition: Value.h:275
user_iterator user_begin()
Definition: Value.h:294
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:332
unsigned getPrimitiveSizeInBits() const LLVM_READONLY
getPrimitiveSizeInBits - Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:121
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:365
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
isInstructionTriviallyDead - Return true if the result produced by the instruction is not used...
Definition: Local.cpp:282
LLVM Value Representation.
Definition: Value.h:69
static unsigned FindInOperandList(SmallVectorImpl< ValueEntry > &Ops, unsigned i, Value *X)
Scan backwards and forwards among values with the same rank as element i to see if X exists...
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:112
A vector that has set insertion semantics.
Definition: SetVector.h:37
void clearSubclassOptionalData()
Clear the optional flags contained in this value.
Definition: Value.h:374
void moveBefore(Instruction *MovePos)
moveBefore - Unlink this instruction from its current basic block and insert it into the basic block ...
Definition: Instruction.cpp:89
static BinaryOperator * isReassociableOp(Value *V, unsigned Opcode)
Return true if V is an instruction of the specified opcode and if it only has one use...
InvokeInst - Invoke instruction.
#define DEBUG(X)
Definition: Debug.h:92
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Convenience struct for specifying and reasoning about fast-math flags.
Definition: Operator.h:164
static bool ShouldBreakUpSubtract(Instruction *Sub)
Return true if we should break up this subtract of X-Y into (X + -Y).
static APInt getNullValue(unsigned numBits)
Get the '0' value.
Definition: APInt.h:460
static BinaryOperator * CreateMul(Value *S1, Value *S2, const Twine &Name, Instruction *InsertBefore, Value *FlagsOp)
static Constant * getBinOpAbsorber(unsigned Opcode, Type *Ty)
getBinOpAbsorber - Return the absorbing element for the given binary operation, i.e.
Definition: Constants.cpp:2372
static bool LinearizeExprTree(BinaryOperator *I, SmallVectorImpl< RepeatedValue > &Ops)
Given an associative binary expression, return the leaf nodes in Ops along with their weights (how ma...
const BasicBlock * getParent() const
Definition: Instruction.h:72
User * user_back()
Definition: Value.h:298