LLVM  6.0.0svn
NewGVN.cpp
Go to the documentation of this file.
1 //===---- NewGVN.cpp - Global Value Numbering Pass --------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 /// \file
10 /// This file implements the new LLVM's Global Value Numbering pass.
11 /// GVN partitions values computed by a function into congruence classes.
12 /// Values ending up in the same congruence class are guaranteed to be the same
13 /// for every execution of the program. In that respect, congruency is a
14 /// compile-time approximation of equivalence of values at runtime.
15 /// The algorithm implemented here uses a sparse formulation and it's based
16 /// on the ideas described in the paper:
17 /// "A Sparse Algorithm for Predicated Global Value Numbering" from
18 /// Karthik Gargi.
19 ///
20 /// A brief overview of the algorithm: The algorithm is essentially the same as
21 /// the standard RPO value numbering algorithm (a good reference is the paper
22 /// "SCC based value numbering" by L. Taylor Simpson) with one major difference:
23 /// The RPO algorithm proceeds, on every iteration, to process every reachable
24 /// block and every instruction in that block. This is because the standard RPO
25 /// algorithm does not track what things have the same value number, it only
26 /// tracks what the value number of a given operation is (the mapping is
27 /// operation -> value number). Thus, when a value number of an operation
28 /// changes, it must reprocess everything to ensure all uses of a value number
29 /// get updated properly. In constrast, the sparse algorithm we use *also*
30 /// tracks what operations have a given value number (IE it also tracks the
31 /// reverse mapping from value number -> operations with that value number), so
32 /// that it only needs to reprocess the instructions that are affected when
33 /// something's value number changes. The vast majority of complexity and code
34 /// in this file is devoted to tracking what value numbers could change for what
35 /// instructions when various things happen. The rest of the algorithm is
36 /// devoted to performing symbolic evaluation, forward propagation, and
37 /// simplification of operations based on the value numbers deduced so far
38 ///
39 /// In order to make the GVN mostly-complete, we use a technique derived from
40 /// "Detection of Redundant Expressions: A Complete and Polynomial-time
41 /// Algorithm in SSA" by R.R. Pai. The source of incompleteness in most SSA
42 /// based GVN algorithms is related to their inability to detect equivalence
43 /// between phi of ops (IE phi(a+b, c+d)) and op of phis (phi(a,c) + phi(b, d)).
44 /// We resolve this issue by generating the equivalent "phi of ops" form for
45 /// each op of phis we see, in a way that only takes polynomial time to resolve.
46 ///
47 /// We also do not perform elimination by using any published algorithm. All
48 /// published algorithms are O(Instructions). Instead, we use a technique that
49 /// is O(number of operations with the same value number), enabling us to skip
50 /// trying to eliminate things that have unique value numbers.
51 //===----------------------------------------------------------------------===//
52 
54 #include "llvm/ADT/BitVector.h"
56 #include "llvm/ADT/MapVector.h"
58 #include "llvm/ADT/SmallSet.h"
60 #include "llvm/ADT/Statistic.h"
63 #include "llvm/Analysis/CFG.h"
70 #include "llvm/IR/PatternMatch.h"
72 #include "llvm/Transforms/Scalar.h"
78 #include <numeric>
79 #include <unordered_map>
80 using namespace llvm;
81 using namespace PatternMatch;
82 using namespace llvm::GVNExpression;
83 using namespace llvm::VNCoercion;
84 #define DEBUG_TYPE "newgvn"
85 
86 STATISTIC(NumGVNInstrDeleted, "Number of instructions deleted");
87 STATISTIC(NumGVNBlocksDeleted, "Number of blocks deleted");
88 STATISTIC(NumGVNOpsSimplified, "Number of Expressions simplified");
89 STATISTIC(NumGVNPhisAllSame, "Number of PHIs whos arguments are all the same");
90 STATISTIC(NumGVNMaxIterations,
91  "Maximum Number of iterations it took to converge GVN");
92 STATISTIC(NumGVNLeaderChanges, "Number of leader changes");
93 STATISTIC(NumGVNSortedLeaderChanges, "Number of sorted leader changes");
94 STATISTIC(NumGVNAvoidedSortedLeaderChanges,
95  "Number of avoided sorted leader changes");
96 STATISTIC(NumGVNDeadStores, "Number of redundant/dead stores eliminated");
97 STATISTIC(NumGVNPHIOfOpsCreated, "Number of PHI of ops created");
98 STATISTIC(NumGVNPHIOfOpsEliminations,
99  "Number of things eliminated using PHI of ops");
100 DEBUG_COUNTER(VNCounter, "newgvn-vn",
101  "Controls which instructions are value numbered");
102 DEBUG_COUNTER(PHIOfOpsCounter, "newgvn-phi",
103  "Controls which instructions we create phi of ops for");
104 // Currently store defining access refinement is too slow due to basicaa being
105 // egregiously slow. This flag lets us keep it working while we work on this
106 // issue.
107 static cl::opt<bool> EnableStoreRefinement("enable-store-refinement",
108  cl::init(false), cl::Hidden);
109 
110 /// Currently, the generation "phi of ops" can result in correctness issues.
111 static cl::opt<bool> EnablePhiOfOps("enable-phi-of-ops", cl::init(true),
112  cl::Hidden);
113 
114 //===----------------------------------------------------------------------===//
115 // GVN Pass
116 //===----------------------------------------------------------------------===//
117 
118 // Anchor methods.
119 namespace llvm {
120 namespace GVNExpression {
121 Expression::~Expression() = default;
122 BasicExpression::~BasicExpression() = default;
123 CallExpression::~CallExpression() = default;
124 LoadExpression::~LoadExpression() = default;
125 StoreExpression::~StoreExpression() = default;
126 AggregateValueExpression::~AggregateValueExpression() = default;
127 PHIExpression::~PHIExpression() = default;
128 }
129 }
130 
131 namespace {
132 // Tarjan's SCC finding algorithm with Nuutila's improvements
133 // SCCIterator is actually fairly complex for the simple thing we want.
134 // It also wants to hand us SCC's that are unrelated to the phi node we ask
135 // about, and have us process them there or risk redoing work.
136 // Graph traits over a filter iterator also doesn't work that well here.
137 // This SCC finder is specialized to walk use-def chains, and only follows
138 // instructions,
139 // not generic values (arguments, etc).
140 struct TarjanSCC {
141 
142  TarjanSCC() : Components(1) {}
143 
144  void Start(const Instruction *Start) {
145  if (Root.lookup(Start) == 0)
146  FindSCC(Start);
147  }
148 
149  const SmallPtrSetImpl<const Value *> &getComponentFor(const Value *V) const {
150  unsigned ComponentID = ValueToComponent.lookup(V);
151 
152  assert(ComponentID > 0 &&
153  "Asking for a component for a value we never processed");
154  return Components[ComponentID];
155  }
156 
157 private:
158  void FindSCC(const Instruction *I) {
159  Root[I] = ++DFSNum;
160  // Store the DFS Number we had before it possibly gets incremented.
161  unsigned int OurDFS = DFSNum;
162  for (auto &Op : I->operands()) {
163  if (auto *InstOp = dyn_cast<Instruction>(Op)) {
164  if (Root.lookup(Op) == 0)
165  FindSCC(InstOp);
166  if (!InComponent.count(Op))
167  Root[I] = std::min(Root.lookup(I), Root.lookup(Op));
168  }
169  }
170  // See if we really were the root of a component, by seeing if we still have
171  // our DFSNumber. If we do, we are the root of the component, and we have
172  // completed a component. If we do not, we are not the root of a component,
173  // and belong on the component stack.
174  if (Root.lookup(I) == OurDFS) {
175  unsigned ComponentID = Components.size();
176  Components.resize(Components.size() + 1);
177  auto &Component = Components.back();
178  Component.insert(I);
179  DEBUG(dbgs() << "Component root is " << *I << "\n");
180  InComponent.insert(I);
181  ValueToComponent[I] = ComponentID;
182  // Pop a component off the stack and label it.
183  while (!Stack.empty() && Root.lookup(Stack.back()) >= OurDFS) {
184  auto *Member = Stack.back();
185  DEBUG(dbgs() << "Component member is " << *Member << "\n");
186  Component.insert(Member);
187  InComponent.insert(Member);
188  ValueToComponent[Member] = ComponentID;
189  Stack.pop_back();
190  }
191  } else {
192  // Part of a component, push to stack
193  Stack.push_back(I);
194  }
195  }
196  unsigned int DFSNum = 1;
197  SmallPtrSet<const Value *, 8> InComponent;
200  // Store the components as vector of ptr sets, because we need the topo order
201  // of SCC's, but not individual member order
203  DenseMap<const Value *, unsigned> ValueToComponent;
204 };
205 // Congruence classes represent the set of expressions/instructions
206 // that are all the same *during some scope in the function*.
207 // That is, because of the way we perform equality propagation, and
208 // because of memory value numbering, it is not correct to assume
209 // you can willy-nilly replace any member with any other at any
210 // point in the function.
211 //
212 // For any Value in the Member set, it is valid to replace any dominated member
213 // with that Value.
214 //
215 // Every congruence class has a leader, and the leader is used to symbolize
216 // instructions in a canonical way (IE every operand of an instruction that is a
217 // member of the same congruence class will always be replaced with leader
218 // during symbolization). To simplify symbolization, we keep the leader as a
219 // constant if class can be proved to be a constant value. Otherwise, the
220 // leader is the member of the value set with the smallest DFS number. Each
221 // congruence class also has a defining expression, though the expression may be
222 // null. If it exists, it can be used for forward propagation and reassociation
223 // of values.
224 
225 // For memory, we also track a representative MemoryAccess, and a set of memory
226 // members for MemoryPhis (which have no real instructions). Note that for
227 // memory, it seems tempting to try to split the memory members into a
228 // MemoryCongruenceClass or something. Unfortunately, this does not work
229 // easily. The value numbering of a given memory expression depends on the
230 // leader of the memory congruence class, and the leader of memory congruence
231 // class depends on the value numbering of a given memory expression. This
232 // leads to wasted propagation, and in some cases, missed optimization. For
233 // example: If we had value numbered two stores together before, but now do not,
234 // we move them to a new value congruence class. This in turn will move at one
235 // of the memorydefs to a new memory congruence class. Which in turn, affects
236 // the value numbering of the stores we just value numbered (because the memory
237 // congruence class is part of the value number). So while theoretically
238 // possible to split them up, it turns out to be *incredibly* complicated to get
239 // it to work right, because of the interdependency. While structurally
240 // slightly messier, it is algorithmically much simpler and faster to do what we
241 // do here, and track them both at once in the same class.
242 // Note: The default iterators for this class iterate over values
243 class CongruenceClass {
244 public:
245  using MemberType = Value;
246  using MemberSet = SmallPtrSet<MemberType *, 4>;
247  using MemoryMemberType = MemoryPhi;
248  using MemoryMemberSet = SmallPtrSet<const MemoryMemberType *, 2>;
249 
250  explicit CongruenceClass(unsigned ID) : ID(ID) {}
251  CongruenceClass(unsigned ID, Value *Leader, const Expression *E)
252  : ID(ID), RepLeader(Leader), DefiningExpr(E) {}
253  unsigned getID() const { return ID; }
254  // True if this class has no members left. This is mainly used for assertion
255  // purposes, and for skipping empty classes.
256  bool isDead() const {
257  // If it's both dead from a value perspective, and dead from a memory
258  // perspective, it's really dead.
259  return empty() && memory_empty();
260  }
261  // Leader functions
262  Value *getLeader() const { return RepLeader; }
263  void setLeader(Value *Leader) { RepLeader = Leader; }
264  const std::pair<Value *, unsigned int> &getNextLeader() const {
265  return NextLeader;
266  }
267  void resetNextLeader() { NextLeader = {nullptr, ~0}; }
268 
269  void addPossibleNextLeader(std::pair<Value *, unsigned int> LeaderPair) {
270  if (LeaderPair.second < NextLeader.second)
271  NextLeader = LeaderPair;
272  }
273 
274  Value *getStoredValue() const { return RepStoredValue; }
275  void setStoredValue(Value *Leader) { RepStoredValue = Leader; }
276  const MemoryAccess *getMemoryLeader() const { return RepMemoryAccess; }
277  void setMemoryLeader(const MemoryAccess *Leader) { RepMemoryAccess = Leader; }
278 
279  // Forward propagation info
280  const Expression *getDefiningExpr() const { return DefiningExpr; }
281 
282  // Value member set
283  bool empty() const { return Members.empty(); }
284  unsigned size() const { return Members.size(); }
285  MemberSet::const_iterator begin() const { return Members.begin(); }
286  MemberSet::const_iterator end() const { return Members.end(); }
287  void insert(MemberType *M) { Members.insert(M); }
288  void erase(MemberType *M) { Members.erase(M); }
289  void swap(MemberSet &Other) { Members.swap(Other); }
290 
291  // Memory member set
292  bool memory_empty() const { return MemoryMembers.empty(); }
293  unsigned memory_size() const { return MemoryMembers.size(); }
294  MemoryMemberSet::const_iterator memory_begin() const {
295  return MemoryMembers.begin();
296  }
297  MemoryMemberSet::const_iterator memory_end() const {
298  return MemoryMembers.end();
299  }
301  return make_range(memory_begin(), memory_end());
302  }
303  void memory_insert(const MemoryMemberType *M) { MemoryMembers.insert(M); }
304  void memory_erase(const MemoryMemberType *M) { MemoryMembers.erase(M); }
305 
306  // Store count
307  unsigned getStoreCount() const { return StoreCount; }
308  void incStoreCount() { ++StoreCount; }
309  void decStoreCount() {
310  assert(StoreCount != 0 && "Store count went negative");
311  --StoreCount;
312  }
313 
314  // True if this class has no memory members.
315  bool definesNoMemory() const { return StoreCount == 0 && memory_empty(); }
316 
317  // Return true if two congruence classes are equivalent to each other. This
318  // means
319  // that every field but the ID number and the dead field are equivalent.
320  bool isEquivalentTo(const CongruenceClass *Other) const {
321  if (!Other)
322  return false;
323  if (this == Other)
324  return true;
325 
326  if (std::tie(StoreCount, RepLeader, RepStoredValue, RepMemoryAccess) !=
327  std::tie(Other->StoreCount, Other->RepLeader, Other->RepStoredValue,
328  Other->RepMemoryAccess))
329  return false;
330  if (DefiningExpr != Other->DefiningExpr)
331  if (!DefiningExpr || !Other->DefiningExpr ||
332  *DefiningExpr != *Other->DefiningExpr)
333  return false;
334  // We need some ordered set
335  std::set<Value *> AMembers(Members.begin(), Members.end());
336  std::set<Value *> BMembers(Members.begin(), Members.end());
337  return AMembers == BMembers;
338  }
339 
340 private:
341  unsigned ID;
342  // Representative leader.
343  Value *RepLeader = nullptr;
344  // The most dominating leader after our current leader, because the member set
345  // is not sorted and is expensive to keep sorted all the time.
346  std::pair<Value *, unsigned int> NextLeader = {nullptr, ~0U};
347  // If this is represented by a store, the value of the store.
348  Value *RepStoredValue = nullptr;
349  // If this class contains MemoryDefs or MemoryPhis, this is the leading memory
350  // access.
351  const MemoryAccess *RepMemoryAccess = nullptr;
352  // Defining Expression.
353  const Expression *DefiningExpr = nullptr;
354  // Actual members of this class.
355  MemberSet Members;
356  // This is the set of MemoryPhis that exist in the class. MemoryDefs and
357  // MemoryUses have real instructions representing them, so we only need to
358  // track MemoryPhis here.
359  MemoryMemberSet MemoryMembers;
360  // Number of stores in this congruence class.
361  // This is used so we can detect store equivalence changes properly.
362  int StoreCount = 0;
363 };
364 } // namespace
365 
366 namespace llvm {
368  const Expression &E;
369  explicit ExactEqualsExpression(const Expression &E) : E(E) {}
370  hash_code getComputedHash() const { return E.getComputedHash(); }
371  bool operator==(const Expression &Other) const {
372  return E.exactlyEquals(Other);
373  }
374 };
375 
376 template <> struct DenseMapInfo<const Expression *> {
377  static const Expression *getEmptyKey() {
378  auto Val = static_cast<uintptr_t>(-1);
379  Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
380  return reinterpret_cast<const Expression *>(Val);
381  }
382  static const Expression *getTombstoneKey() {
383  auto Val = static_cast<uintptr_t>(~1U);
384  Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
385  return reinterpret_cast<const Expression *>(Val);
386  }
387  static unsigned getHashValue(const Expression *E) {
388  return E->getComputedHash();
389  }
390  static unsigned getHashValue(const ExactEqualsExpression &E) {
391  return E.getComputedHash();
392  }
393  static bool isEqual(const ExactEqualsExpression &LHS, const Expression *RHS) {
394  if (RHS == getTombstoneKey() || RHS == getEmptyKey())
395  return false;
396  return LHS == *RHS;
397  }
398 
399  static bool isEqual(const Expression *LHS, const Expression *RHS) {
400  if (LHS == RHS)
401  return true;
402  if (LHS == getTombstoneKey() || RHS == getTombstoneKey() ||
403  LHS == getEmptyKey() || RHS == getEmptyKey())
404  return false;
405  // Compare hashes before equality. This is *not* what the hashtable does,
406  // since it is computing it modulo the number of buckets, whereas we are
407  // using the full hash keyspace. Since the hashes are precomputed, this
408  // check is *much* faster than equality.
409  if (LHS->getComputedHash() != RHS->getComputedHash())
410  return false;
411  return *LHS == *RHS;
412  }
413 };
414 } // end namespace llvm
415 
416 namespace {
417 class NewGVN {
418  Function &F;
419  DominatorTree *DT;
420  const TargetLibraryInfo *TLI;
421  AliasAnalysis *AA;
422  MemorySSA *MSSA;
423  MemorySSAWalker *MSSAWalker;
424  const DataLayout &DL;
425  std::unique_ptr<PredicateInfo> PredInfo;
426 
427  // These are the only two things the create* functions should have
428  // side-effects on due to allocating memory.
429  mutable BumpPtrAllocator ExpressionAllocator;
430  mutable ArrayRecycler<Value *> ArgRecycler;
431  mutable TarjanSCC SCCFinder;
432  const SimplifyQuery SQ;
433 
434  // Number of function arguments, used by ranking
435  unsigned int NumFuncArgs;
436 
437  // RPOOrdering of basic blocks
439 
440  // Congruence class info.
441 
442  // This class is called INITIAL in the paper. It is the class everything
443  // startsout in, and represents any value. Being an optimistic analysis,
444  // anything in the TOP class has the value TOP, which is indeterminate and
445  // equivalent to everything.
446  CongruenceClass *TOPClass;
447  std::vector<CongruenceClass *> CongruenceClasses;
448  unsigned NextCongruenceNum;
449 
450  // Value Mappings.
452  DenseMap<Value *, const Expression *> ValueToExpression;
453  // Value PHI handling, used to make equivalence between phi(op, op) and
454  // op(phi, phi).
455  // These mappings just store various data that would normally be part of the
456  // IR.
458 
459  DenseMap<const Value *, bool> OpSafeForPHIOfOps;
460  // Map a temporary instruction we created to a parent block.
462  // Map between the already in-program instructions and the temporary phis we
463  // created that they are known equivalent to.
465  // In order to know when we should re-process instructions that have
466  // phi-of-ops, we track the set of expressions that they needed as
467  // leaders. When we discover new leaders for those expressions, we process the
468  // associated phi-of-op instructions again in case they have changed. The
469  // other way they may change is if they had leaders, and those leaders
470  // disappear. However, at the point they have leaders, there are uses of the
471  // relevant operands in the created phi node, and so they will get reprocessed
472  // through the normal user marking we perform.
473  mutable DenseMap<const Value *, SmallPtrSet<Value *, 2>> AdditionalUsers;
475  ExpressionToPhiOfOps;
476  // Map from temporary operation to MemoryAccess.
478  // Set of all temporary instructions we created.
479  // Note: This will include instructions that were just created during value
480  // numbering. The way to test if something is using them is to check
481  // RealToTemp.
482 
483  DenseSet<Instruction *> AllTempInstructions;
484 
485  // This is the set of instructions to revisit on a reachability change. At
486  // the end of the main iteration loop it will contain at least all the phi of
487  // ops instructions that will be changed to phis, as well as regular phis.
488  // During the iteration loop, it may contain other things, such as phi of ops
489  // instructions that used edge reachability to reach a result, and so need to
490  // be revisited when the edge changes, independent of whether the phi they
491  // depended on changes.
492  DenseMap<BasicBlock *, SparseBitVector<>> RevisitOnReachabilityChange;
493 
494  // Mapping from predicate info we used to the instructions we used it with.
495  // In order to correctly ensure propagation, we must keep track of what
496  // comparisons we used, so that when the values of the comparisons change, we
497  // propagate the information to the places we used the comparison.
499  PredicateToUsers;
500  // the same reasoning as PredicateToUsers. When we skip MemoryAccesses for
501  // stores, we no longer can rely solely on the def-use chains of MemorySSA.
503  MemoryToUsers;
504 
505  // A table storing which memorydefs/phis represent a memory state provably
506  // equivalent to another memory state.
507  // We could use the congruence class machinery, but the MemoryAccess's are
508  // abstract memory states, so they can only ever be equivalent to each other,
509  // and not to constants, etc.
511 
512  // We could, if we wanted, build MemoryPhiExpressions and
513  // MemoryVariableExpressions, etc, and value number them the same way we value
514  // number phi expressions. For the moment, this seems like overkill. They
515  // can only exist in one of three states: they can be TOP (equal to
516  // everything), Equivalent to something else, or unique. Because we do not
517  // create expressions for them, we need to simulate leader change not just
518  // when they change class, but when they change state. Note: We can do the
519  // same thing for phis, and avoid having phi expressions if we wanted, We
520  // should eventually unify in one direction or the other, so this is a little
521  // bit of an experiment in which turns out easier to maintain.
522  enum MemoryPhiState { MPS_Invalid, MPS_TOP, MPS_Equivalent, MPS_Unique };
524 
525  enum InstCycleState { ICS_Unknown, ICS_CycleFree, ICS_Cycle };
526  mutable DenseMap<const Instruction *, InstCycleState> InstCycleState;
527  // Expression to class mapping.
528  using ExpressionClassMap = DenseMap<const Expression *, CongruenceClass *>;
529  ExpressionClassMap ExpressionToClass;
530 
531  // We have a single expression that represents currently DeadExpressions.
532  // For dead expressions we can prove will stay dead, we mark them with
533  // DFS number zero. However, it's possible in the case of phi nodes
534  // for us to assume/prove all arguments are dead during fixpointing.
535  // We use DeadExpression for that case.
536  DeadExpression *SingletonDeadExpression = nullptr;
537 
538  // Which values have changed as a result of leader changes.
539  SmallPtrSet<Value *, 8> LeaderChanges;
540 
541  // Reachability info.
542  using BlockEdge = BasicBlockEdge;
543  DenseSet<BlockEdge> ReachableEdges;
544  SmallPtrSet<const BasicBlock *, 8> ReachableBlocks;
545 
546  // This is a bitvector because, on larger functions, we may have
547  // thousands of touched instructions at once (entire blocks,
548  // instructions with hundreds of uses, etc). Even with optimization
549  // for when we mark whole blocks as touched, when this was a
550  // SmallPtrSet or DenseSet, for some functions, we spent >20% of all
551  // the time in GVN just managing this list. The bitvector, on the
552  // other hand, efficiently supports test/set/clear of both
553  // individual and ranges, as well as "find next element" This
554  // enables us to use it as a worklist with essentially 0 cost.
555  BitVector TouchedInstructions;
556 
558 
559 #ifndef NDEBUG
560  // Debugging for how many times each block and instruction got processed.
561  DenseMap<const Value *, unsigned> ProcessedCount;
562 #endif
563 
564  // DFS info.
565  // This contains a mapping from Instructions to DFS numbers.
566  // The numbering starts at 1. An instruction with DFS number zero
567  // means that the instruction is dead.
569 
570  // This contains the mapping DFS numbers to instructions.
571  SmallVector<Value *, 32> DFSToInstr;
572 
573  // Deletion info.
574  SmallPtrSet<Instruction *, 8> InstructionsToErase;
575 
576 public:
577  NewGVN(Function &F, DominatorTree *DT, AssumptionCache *AC,
578  TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA,
579  const DataLayout &DL)
580  : F(F), DT(DT), TLI(TLI), AA(AA), MSSA(MSSA), DL(DL),
581  PredInfo(make_unique<PredicateInfo>(F, *DT, *AC)), SQ(DL, TLI, DT, AC) {
582  }
583  bool runGVN();
584 
585 private:
586  // Expression handling.
587  const Expression *createExpression(Instruction *) const;
588  const Expression *createBinaryExpression(unsigned, Type *, Value *, Value *,
589  Instruction *) const;
590  // Our canonical form for phi arguments is a pair of incoming value, incoming
591  // basic block.
592  typedef std::pair<Value *, BasicBlock *> ValPair;
593  PHIExpression *createPHIExpression(ArrayRef<ValPair>, const Instruction *,
594  BasicBlock *, bool &HasBackEdge,
595  bool &OriginalOpsConstant) const;
596  const DeadExpression *createDeadExpression() const;
597  const VariableExpression *createVariableExpression(Value *) const;
598  const ConstantExpression *createConstantExpression(Constant *) const;
599  const Expression *createVariableOrConstant(Value *V) const;
600  const UnknownExpression *createUnknownExpression(Instruction *) const;
601  const StoreExpression *createStoreExpression(StoreInst *,
602  const MemoryAccess *) const;
603  LoadExpression *createLoadExpression(Type *, Value *, LoadInst *,
604  const MemoryAccess *) const;
605  const CallExpression *createCallExpression(CallInst *,
606  const MemoryAccess *) const;
608  createAggregateValueExpression(Instruction *) const;
609  bool setBasicExpressionInfo(Instruction *, BasicExpression *) const;
610 
611  // Congruence class handling.
612  CongruenceClass *createCongruenceClass(Value *Leader, const Expression *E) {
613  auto *result = new CongruenceClass(NextCongruenceNum++, Leader, E);
614  CongruenceClasses.emplace_back(result);
615  return result;
616  }
617 
618  CongruenceClass *createMemoryClass(MemoryAccess *MA) {
619  auto *CC = createCongruenceClass(nullptr, nullptr);
620  CC->setMemoryLeader(MA);
621  return CC;
622  }
623  CongruenceClass *ensureLeaderOfMemoryClass(MemoryAccess *MA) {
624  auto *CC = getMemoryClass(MA);
625  if (CC->getMemoryLeader() != MA)
626  CC = createMemoryClass(MA);
627  return CC;
628  }
629 
630  CongruenceClass *createSingletonCongruenceClass(Value *Member) {
631  CongruenceClass *CClass = createCongruenceClass(Member, nullptr);
632  CClass->insert(Member);
633  ValueToClass[Member] = CClass;
634  return CClass;
635  }
636  void initializeCongruenceClasses(Function &F);
637  const Expression *makePossiblePHIOfOps(Instruction *,
639  Value *findLeaderForInst(Instruction *ValueOp,
640  SmallPtrSetImpl<Value *> &Visited,
641  MemoryAccess *MemAccess, Instruction *OrigInst,
642  BasicBlock *PredBB);
643  bool OpIsSafeForPHIOfOpsHelper(Value *V, const BasicBlock *PHIBlock,
646  bool OpIsSafeForPHIOfOps(Value *Op, const BasicBlock *PHIBlock,
648  void addPhiOfOps(PHINode *Op, BasicBlock *BB, Instruction *ExistingValue);
649  void removePhiOfOps(Instruction *I, PHINode *PHITemp);
650 
651  // Value number an Instruction or MemoryPhi.
652  void valueNumberMemoryPhi(MemoryPhi *);
653  void valueNumberInstruction(Instruction *);
654 
655  // Symbolic evaluation.
656  const Expression *checkSimplificationResults(Expression *, Instruction *,
657  Value *) const;
658  const Expression *performSymbolicEvaluation(Value *,
659  SmallPtrSetImpl<Value *> &) const;
660  const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *,
661  Instruction *,
662  MemoryAccess *) const;
663  const Expression *performSymbolicLoadEvaluation(Instruction *) const;
664  const Expression *performSymbolicStoreEvaluation(Instruction *) const;
665  const Expression *performSymbolicCallEvaluation(Instruction *) const;
666  void sortPHIOps(MutableArrayRef<ValPair> Ops) const;
667  const Expression *performSymbolicPHIEvaluation(ArrayRef<ValPair>,
668  Instruction *I,
669  BasicBlock *PHIBlock) const;
670  const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;
671  const Expression *performSymbolicCmpEvaluation(Instruction *) const;
672  const Expression *performSymbolicPredicateInfoEvaluation(Instruction *) const;
673 
674  // Congruence finding.
675  bool someEquivalentDominates(const Instruction *, const Instruction *) const;
676  Value *lookupOperandLeader(Value *) const;
677  CongruenceClass *getClassForExpression(const Expression *E) const;
678  void performCongruenceFinding(Instruction *, const Expression *);
679  void moveValueToNewCongruenceClass(Instruction *, const Expression *,
680  CongruenceClass *, CongruenceClass *);
681  void moveMemoryToNewCongruenceClass(Instruction *, MemoryAccess *,
682  CongruenceClass *, CongruenceClass *);
683  Value *getNextValueLeader(CongruenceClass *) const;
684  const MemoryAccess *getNextMemoryLeader(CongruenceClass *) const;
685  bool setMemoryClass(const MemoryAccess *From, CongruenceClass *To);
686  CongruenceClass *getMemoryClass(const MemoryAccess *MA) const;
687  const MemoryAccess *lookupMemoryLeader(const MemoryAccess *) const;
688  bool isMemoryAccessTOP(const MemoryAccess *) const;
689 
690  // Ranking
691  unsigned int getRank(const Value *) const;
692  bool shouldSwapOperands(const Value *, const Value *) const;
693 
694  // Reachability handling.
695  void updateReachableEdge(BasicBlock *, BasicBlock *);
696  void processOutgoingEdges(TerminatorInst *, BasicBlock *);
697  Value *findConditionEquivalence(Value *) const;
698 
699  // Elimination.
700  struct ValueDFS;
701  void convertClassToDFSOrdered(const CongruenceClass &,
705  void convertClassToLoadsAndStores(const CongruenceClass &,
706  SmallVectorImpl<ValueDFS> &) const;
707 
708  bool eliminateInstructions(Function &);
709  void replaceInstruction(Instruction *, Value *);
710  void markInstructionForDeletion(Instruction *);
711  void deleteInstructionsInBlock(BasicBlock *);
712  Value *findPHIOfOpsLeader(const Expression *, const Instruction *,
713  const BasicBlock *) const;
714 
715  // New instruction creation.
716  void handleNewInstruction(Instruction *){};
717 
718  // Various instruction touch utilities
719  template <typename Map, typename KeyType, typename Func>
720  void for_each_found(Map &, const KeyType &, Func);
721  template <typename Map, typename KeyType>
722  void touchAndErase(Map &, const KeyType &);
723  void markUsersTouched(Value *);
724  void markMemoryUsersTouched(const MemoryAccess *);
725  void markMemoryDefTouched(const MemoryAccess *);
726  void markPredicateUsersTouched(Instruction *);
727  void markValueLeaderChangeTouched(CongruenceClass *CC);
728  void markMemoryLeaderChangeTouched(CongruenceClass *CC);
729  void markPhiOfOpsChanged(const Expression *E);
730  void addPredicateUsers(const PredicateBase *, Instruction *) const;
731  void addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const;
732  void addAdditionalUsers(Value *To, Value *User) const;
733 
734  // Main loop of value numbering
735  void iterateTouchedInstructions();
736 
737  // Utilities.
738  void cleanupTables();
739  std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned);
740  void updateProcessedCount(const Value *V);
741  void verifyMemoryCongruency() const;
742  void verifyIterationSettled(Function &F);
743  void verifyStoreExpressions() const;
744  bool singleReachablePHIPath(SmallPtrSet<const MemoryAccess *, 8> &,
745  const MemoryAccess *, const MemoryAccess *) const;
746  BasicBlock *getBlockForValue(Value *V) const;
747  void deleteExpression(const Expression *E) const;
748  MemoryUseOrDef *getMemoryAccess(const Instruction *) const;
749  MemoryAccess *getDefiningAccess(const MemoryAccess *) const;
750  MemoryPhi *getMemoryAccess(const BasicBlock *) const;
751  template <class T, class Range> T *getMinDFSOfRange(const Range &) const;
752  unsigned InstrToDFSNum(const Value *V) const {
753  assert(isa<Instruction>(V) && "This should not be used for MemoryAccesses");
754  return InstrDFS.lookup(V);
755  }
756 
757  unsigned InstrToDFSNum(const MemoryAccess *MA) const {
758  return MemoryToDFSNum(MA);
759  }
760  Value *InstrFromDFSNum(unsigned DFSNum) { return DFSToInstr[DFSNum]; }
761  // Given a MemoryAccess, return the relevant instruction DFS number. Note:
762  // This deliberately takes a value so it can be used with Use's, which will
763  // auto-convert to Value's but not to MemoryAccess's.
764  unsigned MemoryToDFSNum(const Value *MA) const {
765  assert(isa<MemoryAccess>(MA) &&
766  "This should not be used with instructions");
767  return isa<MemoryUseOrDef>(MA)
768  ? InstrToDFSNum(cast<MemoryUseOrDef>(MA)->getMemoryInst())
769  : InstrDFS.lookup(MA);
770  }
771  bool isCycleFree(const Instruction *) const;
772  bool isBackedge(BasicBlock *From, BasicBlock *To) const;
773  // Debug counter info. When verifying, we have to reset the value numbering
774  // debug counter to the same state it started in to get the same results.
775  std::pair<int, int> StartingVNCounter;
776 };
777 } // end anonymous namespace
778 
779 template <typename T>
780 static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {
781  if (!isa<LoadExpression>(RHS) && !isa<StoreExpression>(RHS))
782  return false;
783  return LHS.MemoryExpression::equals(RHS);
784 }
785 
786 bool LoadExpression::equals(const Expression &Other) const {
787  return equalsLoadStoreHelper(*this, Other);
788 }
789 
790 bool StoreExpression::equals(const Expression &Other) const {
791  if (!equalsLoadStoreHelper(*this, Other))
792  return false;
793  // Make sure that store vs store includes the value operand.
794  if (const auto *S = dyn_cast<StoreExpression>(&Other))
795  if (getStoredValue() != S->getStoredValue())
796  return false;
797  return true;
798 }
799 
800 // Determine if the edge From->To is a backedge
801 bool NewGVN::isBackedge(BasicBlock *From, BasicBlock *To) const {
802  return From == To ||
803  RPOOrdering.lookup(DT->getNode(From)) >=
804  RPOOrdering.lookup(DT->getNode(To));
805 }
806 
807 #ifndef NDEBUG
808 static std::string getBlockName(const BasicBlock *B) {
810 }
811 #endif
812 
813 // Get a MemoryAccess for an instruction, fake or real.
814 MemoryUseOrDef *NewGVN::getMemoryAccess(const Instruction *I) const {
815  auto *Result = MSSA->getMemoryAccess(I);
816  return Result ? Result : TempToMemory.lookup(I);
817 }
818 
819 // Get a MemoryPhi for a basic block. These are all real.
820 MemoryPhi *NewGVN::getMemoryAccess(const BasicBlock *BB) const {
821  return MSSA->getMemoryAccess(BB);
822 }
823 
824 // Get the basic block from an instruction/memory value.
825 BasicBlock *NewGVN::getBlockForValue(Value *V) const {
826  if (auto *I = dyn_cast<Instruction>(V)) {
827  auto *Parent = I->getParent();
828  if (Parent)
829  return Parent;
830  Parent = TempToBlock.lookup(V);
831  assert(Parent && "Every fake instruction should have a block");
832  return Parent;
833  }
834 
835  auto *MP = dyn_cast<MemoryPhi>(V);
836  assert(MP && "Should have been an instruction or a MemoryPhi");
837  return MP->getBlock();
838 }
839 
840 // Delete a definitely dead expression, so it can be reused by the expression
841 // allocator. Some of these are not in creation functions, so we have to accept
842 // const versions.
843 void NewGVN::deleteExpression(const Expression *E) const {
844  assert(isa<BasicExpression>(E));
845  auto *BE = cast<BasicExpression>(E);
846  const_cast<BasicExpression *>(BE)->deallocateOperands(ArgRecycler);
847  ExpressionAllocator.Deallocate(E);
848 }
849 
850 // If V is a predicateinfo copy, get the thing it is a copy of.
851 static Value *getCopyOf(const Value *V) {
852  if (auto *II = dyn_cast<IntrinsicInst>(V))
853  if (II->getIntrinsicID() == Intrinsic::ssa_copy)
854  return II->getOperand(0);
855  return nullptr;
856 }
857 
858 // Return true if V is really PN, even accounting for predicateinfo copies.
859 static bool isCopyOfPHI(const Value *V, const PHINode *PN) {
860  return V == PN || getCopyOf(V) == PN;
861 }
862 
863 static bool isCopyOfAPHI(const Value *V) {
864  auto *CO = getCopyOf(V);
865  return CO && isa<PHINode>(CO);
866 }
867 
868 // Sort PHI Operands into a canonical order. What we use here is an RPO
869 // order. The BlockInstRange numbers are generated in an RPO walk of the basic
870 // blocks.
871 void NewGVN::sortPHIOps(MutableArrayRef<ValPair> Ops) const {
872  std::sort(Ops.begin(), Ops.end(), [&](const ValPair &P1, const ValPair &P2) {
873  return BlockInstRange.lookup(P1.second).first <
874  BlockInstRange.lookup(P2.second).first;
875  });
876 }
877 
878 // Return true if V is a value that will always be available (IE can
879 // be placed anywhere) in the function. We don't do globals here
880 // because they are often worse to put in place.
881 static bool alwaysAvailable(Value *V) {
882  return isa<Constant>(V) || isa<Argument>(V);
883 }
884 
885 // Create a PHIExpression from an array of {incoming edge, value} pairs. I is
886 // the original instruction we are creating a PHIExpression for (but may not be
887 // a phi node). We require, as an invariant, that all the PHIOperands in the
888 // same block are sorted the same way. sortPHIOps will sort them into a
889 // canonical order.
890 PHIExpression *NewGVN::createPHIExpression(ArrayRef<ValPair> PHIOperands,
891  const Instruction *I,
892  BasicBlock *PHIBlock,
893  bool &HasBackedge,
894  bool &OriginalOpsConstant) const {
895  unsigned NumOps = PHIOperands.size();
896  auto *E = new (ExpressionAllocator) PHIExpression(NumOps, PHIBlock);
897 
898  E->allocateOperands(ArgRecycler, ExpressionAllocator);
899  E->setType(PHIOperands.begin()->first->getType());
900  E->setOpcode(Instruction::PHI);
901 
902  // Filter out unreachable phi operands.
903  auto Filtered = make_filter_range(PHIOperands, [&](const ValPair &P) {
904  auto *BB = P.second;
905  if (auto *PHIOp = dyn_cast<PHINode>(I))
906  if (isCopyOfPHI(P.first, PHIOp))
907  return false;
908  if (!ReachableEdges.count({BB, PHIBlock}))
909  return false;
910  // Things in TOPClass are equivalent to everything.
911  if (ValueToClass.lookup(P.first) == TOPClass)
912  return false;
913  OriginalOpsConstant = OriginalOpsConstant && isa<Constant>(P.first);
914  HasBackedge = HasBackedge || isBackedge(BB, PHIBlock);
915  return lookupOperandLeader(P.first) != I;
916  });
917  std::transform(Filtered.begin(), Filtered.end(), op_inserter(E),
918  [&](const ValPair &P) -> Value * {
919  return lookupOperandLeader(P.first);
920  });
921  return E;
922 }
923 
924 // Set basic expression info (Arguments, type, opcode) for Expression
925 // E from Instruction I in block B.
926 bool NewGVN::setBasicExpressionInfo(Instruction *I, BasicExpression *E) const {
927  bool AllConstant = true;
928  if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
929  E->setType(GEP->getSourceElementType());
930  else
931  E->setType(I->getType());
932  E->setOpcode(I->getOpcode());
933  E->allocateOperands(ArgRecycler, ExpressionAllocator);
934 
935  // Transform the operand array into an operand leader array, and keep track of
936  // whether all members are constant.
937  std::transform(I->op_begin(), I->op_end(), op_inserter(E), [&](Value *O) {
938  auto Operand = lookupOperandLeader(O);
939  AllConstant = AllConstant && isa<Constant>(Operand);
940  return Operand;
941  });
942 
943  return AllConstant;
944 }
945 
946 const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T,
947  Value *Arg1, Value *Arg2,
948  Instruction *I) const {
949  auto *E = new (ExpressionAllocator) BasicExpression(2);
950 
951  E->setType(T);
952  E->setOpcode(Opcode);
953  E->allocateOperands(ArgRecycler, ExpressionAllocator);
954  if (Instruction::isCommutative(Opcode)) {
955  // Ensure that commutative instructions that only differ by a permutation
956  // of their operands get the same value number by sorting the operand value
957  // numbers. Since all commutative instructions have two operands it is more
958  // efficient to sort by hand rather than using, say, std::sort.
959  if (shouldSwapOperands(Arg1, Arg2))
960  std::swap(Arg1, Arg2);
961  }
962  E->op_push_back(lookupOperandLeader(Arg1));
963  E->op_push_back(lookupOperandLeader(Arg2));
964 
965  Value *V = SimplifyBinOp(Opcode, E->getOperand(0), E->getOperand(1), SQ);
966  if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
967  return SimplifiedE;
968  return E;
969 }
970 
971 // Take a Value returned by simplification of Expression E/Instruction
972 // I, and see if it resulted in a simpler expression. If so, return
973 // that expression.
974 const Expression *NewGVN::checkSimplificationResults(Expression *E,
975  Instruction *I,
976  Value *V) const {
977  if (!V)
978  return nullptr;
979  if (auto *C = dyn_cast<Constant>(V)) {
980  if (I)
981  DEBUG(dbgs() << "Simplified " << *I << " to "
982  << " constant " << *C << "\n");
983  NumGVNOpsSimplified++;
984  assert(isa<BasicExpression>(E) &&
985  "We should always have had a basic expression here");
986  deleteExpression(E);
987  return createConstantExpression(C);
988  } else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
989  if (I)
990  DEBUG(dbgs() << "Simplified " << *I << " to "
991  << " variable " << *V << "\n");
992  deleteExpression(E);
993  return createVariableExpression(V);
994  }
995 
996  CongruenceClass *CC = ValueToClass.lookup(V);
997  if (CC) {
998  if (CC->getLeader() && CC->getLeader() != I) {
999  // Don't add temporary instructions to the user lists.
1000  if (!AllTempInstructions.count(I))
1001  addAdditionalUsers(V, I);
1002  return createVariableOrConstant(CC->getLeader());
1003  }
1004  if (CC->getDefiningExpr()) {
1005  // If we simplified to something else, we need to communicate
1006  // that we're users of the value we simplified to.
1007  if (I != V) {
1008  // Don't add temporary instructions to the user lists.
1009  if (!AllTempInstructions.count(I))
1010  addAdditionalUsers(V, I);
1011  }
1012 
1013  if (I)
1014  DEBUG(dbgs() << "Simplified " << *I << " to "
1015  << " expression " << *CC->getDefiningExpr() << "\n");
1016  NumGVNOpsSimplified++;
1017  deleteExpression(E);
1018  return CC->getDefiningExpr();
1019  }
1020  }
1021 
1022  return nullptr;
1023 }
1024 
1025 // Create a value expression from the instruction I, replacing operands with
1026 // their leaders.
1027 
1028 const Expression *NewGVN::createExpression(Instruction *I) const {
1029  auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands());
1030 
1031  bool AllConstant = setBasicExpressionInfo(I, E);
1032 
1033  if (I->isCommutative()) {
1034  // Ensure that commutative instructions that only differ by a permutation
1035  // of their operands get the same value number by sorting the operand value
1036  // numbers. Since all commutative instructions have two operands it is more
1037  // efficient to sort by hand rather than using, say, std::sort.
1038  assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!");
1039  if (shouldSwapOperands(E->getOperand(0), E->getOperand(1)))
1040  E->swapOperands(0, 1);
1041  }
1042  // Perform simplification.
1043  if (auto *CI = dyn_cast<CmpInst>(I)) {
1044  // Sort the operand value numbers so x<y and y>x get the same value
1045  // number.
1046  CmpInst::Predicate Predicate = CI->getPredicate();
1047  if (shouldSwapOperands(E->getOperand(0), E->getOperand(1))) {
1048  E->swapOperands(0, 1);
1049  Predicate = CmpInst::getSwappedPredicate(Predicate);
1050  }
1051  E->setOpcode((CI->getOpcode() << 8) | Predicate);
1052  // TODO: 25% of our time is spent in SimplifyCmpInst with pointer operands
1053  assert(I->getOperand(0)->getType() == I->getOperand(1)->getType() &&
1054  "Wrong types on cmp instruction");
1055  assert((E->getOperand(0)->getType() == I->getOperand(0)->getType() &&
1056  E->getOperand(1)->getType() == I->getOperand(1)->getType()));
1057  Value *V =
1058  SimplifyCmpInst(Predicate, E->getOperand(0), E->getOperand(1), SQ);
1059  if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1060  return SimplifiedE;
1061  } else if (isa<SelectInst>(I)) {
1062  if (isa<Constant>(E->getOperand(0)) ||
1063  E->getOperand(1) == E->getOperand(2)) {
1064  assert(E->getOperand(1)->getType() == I->getOperand(1)->getType() &&
1065  E->getOperand(2)->getType() == I->getOperand(2)->getType());
1066  Value *V = SimplifySelectInst(E->getOperand(0), E->getOperand(1),
1067  E->getOperand(2), SQ);
1068  if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1069  return SimplifiedE;
1070  }
1071  } else if (I->isBinaryOp()) {
1072  Value *V =
1073  SimplifyBinOp(E->getOpcode(), E->getOperand(0), E->getOperand(1), SQ);
1074  if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1075  return SimplifiedE;
1076  } else if (auto *BI = dyn_cast<BitCastInst>(I)) {
1077  Value *V =
1078  SimplifyCastInst(BI->getOpcode(), BI->getOperand(0), BI->getType(), SQ);
1079  if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1080  return SimplifiedE;
1081  } else if (isa<GetElementPtrInst>(I)) {
1082  Value *V = SimplifyGEPInst(
1083  E->getType(), ArrayRef<Value *>(E->op_begin(), E->op_end()), SQ);
1084  if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1085  return SimplifiedE;
1086  } else if (AllConstant) {
1087  // We don't bother trying to simplify unless all of the operands
1088  // were constant.
1089  // TODO: There are a lot of Simplify*'s we could call here, if we
1090  // wanted to. The original motivating case for this code was a
1091  // zext i1 false to i8, which we don't have an interface to
1092  // simplify (IE there is no SimplifyZExt).
1093 
1095  for (Value *Arg : E->operands())
1096  C.emplace_back(cast<Constant>(Arg));
1097 
1098  if (Value *V = ConstantFoldInstOperands(I, C, DL, TLI))
1099  if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1100  return SimplifiedE;
1101  }
1102  return E;
1103 }
1104 
1106 NewGVN::createAggregateValueExpression(Instruction *I) const {
1107  if (auto *II = dyn_cast<InsertValueInst>(I)) {
1108  auto *E = new (ExpressionAllocator)
1109  AggregateValueExpression(I->getNumOperands(), II->getNumIndices());
1110  setBasicExpressionInfo(I, E);
1111  E->allocateIntOperands(ExpressionAllocator);
1112  std::copy(II->idx_begin(), II->idx_end(), int_op_inserter(E));
1113  return E;
1114  } else if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
1115  auto *E = new (ExpressionAllocator)
1116  AggregateValueExpression(I->getNumOperands(), EI->getNumIndices());
1117  setBasicExpressionInfo(EI, E);
1118  E->allocateIntOperands(ExpressionAllocator);
1119  std::copy(EI->idx_begin(), EI->idx_end(), int_op_inserter(E));
1120  return E;
1121  }
1122  llvm_unreachable("Unhandled type of aggregate value operation");
1123 }
1124 
1125 const DeadExpression *NewGVN::createDeadExpression() const {
1126  // DeadExpression has no arguments and all DeadExpression's are the same,
1127  // so we only need one of them.
1128  return SingletonDeadExpression;
1129 }
1130 
1131 const VariableExpression *NewGVN::createVariableExpression(Value *V) const {
1132  auto *E = new (ExpressionAllocator) VariableExpression(V);
1133  E->setOpcode(V->getValueID());
1134  return E;
1135 }
1136 
1137 const Expression *NewGVN::createVariableOrConstant(Value *V) const {
1138  if (auto *C = dyn_cast<Constant>(V))
1139  return createConstantExpression(C);
1140  return createVariableExpression(V);
1141 }
1142 
1143 const ConstantExpression *NewGVN::createConstantExpression(Constant *C) const {
1144  auto *E = new (ExpressionAllocator) ConstantExpression(C);
1145  E->setOpcode(C->getValueID());
1146  return E;
1147 }
1148 
1149 const UnknownExpression *NewGVN::createUnknownExpression(Instruction *I) const {
1150  auto *E = new (ExpressionAllocator) UnknownExpression(I);
1151  E->setOpcode(I->getOpcode());
1152  return E;
1153 }
1154 
1155 const CallExpression *
1156 NewGVN::createCallExpression(CallInst *CI, const MemoryAccess *MA) const {
1157  // FIXME: Add operand bundles for calls.
1158  auto *E =
1159  new (ExpressionAllocator) CallExpression(CI->getNumOperands(), CI, MA);
1160  setBasicExpressionInfo(CI, E);
1161  return E;
1162 }
1163 
1164 // Return true if some equivalent of instruction Inst dominates instruction U.
1165 bool NewGVN::someEquivalentDominates(const Instruction *Inst,
1166  const Instruction *U) const {
1167  auto *CC = ValueToClass.lookup(Inst);
1168  // This must be an instruction because we are only called from phi nodes
1169  // in the case that the value it needs to check against is an instruction.
1170 
1171  // The most likely candiates for dominance are the leader and the next leader.
1172  // The leader or nextleader will dominate in all cases where there is an
1173  // equivalent that is higher up in the dom tree.
1174  // We can't *only* check them, however, because the
1175  // dominator tree could have an infinite number of non-dominating siblings
1176  // with instructions that are in the right congruence class.
1177  // A
1178  // B C D E F G
1179  // |
1180  // H
1181  // Instruction U could be in H, with equivalents in every other sibling.
1182  // Depending on the rpo order picked, the leader could be the equivalent in
1183  // any of these siblings.
1184  if (!CC)
1185  return false;
1186  if (alwaysAvailable(CC->getLeader()))
1187  return true;
1188  if (DT->dominates(cast<Instruction>(CC->getLeader()), U))
1189  return true;
1190  if (CC->getNextLeader().first &&
1191  DT->dominates(cast<Instruction>(CC->getNextLeader().first), U))
1192  return true;
1193  return llvm::any_of(*CC, [&](const Value *Member) {
1194  return Member != CC->getLeader() &&
1195  DT->dominates(cast<Instruction>(Member), U);
1196  });
1197 }
1198 
1199 // See if we have a congruence class and leader for this operand, and if so,
1200 // return it. Otherwise, return the operand itself.
1201 Value *NewGVN::lookupOperandLeader(Value *V) const {
1202  CongruenceClass *CC = ValueToClass.lookup(V);
1203  if (CC) {
1204  // Everything in TOP is represented by undef, as it can be any value.
1205  // We do have to make sure we get the type right though, so we can't set the
1206  // RepLeader to undef.
1207  if (CC == TOPClass)
1208  return UndefValue::get(V->getType());
1209  return CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
1210  }
1211 
1212  return V;
1213 }
1214 
1215 const MemoryAccess *NewGVN::lookupMemoryLeader(const MemoryAccess *MA) const {
1216  auto *CC = getMemoryClass(MA);
1217  assert(CC->getMemoryLeader() &&
1218  "Every MemoryAccess should be mapped to a congruence class with a "
1219  "representative memory access");
1220  return CC->getMemoryLeader();
1221 }
1222 
1223 // Return true if the MemoryAccess is really equivalent to everything. This is
1224 // equivalent to the lattice value "TOP" in most lattices. This is the initial
1225 // state of all MemoryAccesses.
1226 bool NewGVN::isMemoryAccessTOP(const MemoryAccess *MA) const {
1227  return getMemoryClass(MA) == TOPClass;
1228 }
1229 
1230 LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp,
1231  LoadInst *LI,
1232  const MemoryAccess *MA) const {
1233  auto *E =
1234  new (ExpressionAllocator) LoadExpression(1, LI, lookupMemoryLeader(MA));
1235  E->allocateOperands(ArgRecycler, ExpressionAllocator);
1236  E->setType(LoadType);
1237 
1238  // Give store and loads same opcode so they value number together.
1239  E->setOpcode(0);
1240  E->op_push_back(PointerOp);
1241  if (LI)
1242  E->setAlignment(LI->getAlignment());
1243 
1244  // TODO: Value number heap versions. We may be able to discover
1245  // things alias analysis can't on it's own (IE that a store and a
1246  // load have the same value, and thus, it isn't clobbering the load).
1247  return E;
1248 }
1249 
1250 const StoreExpression *
1251 NewGVN::createStoreExpression(StoreInst *SI, const MemoryAccess *MA) const {
1252  auto *StoredValueLeader = lookupOperandLeader(SI->getValueOperand());
1253  auto *E = new (ExpressionAllocator)
1254  StoreExpression(SI->getNumOperands(), SI, StoredValueLeader, MA);
1255  E->allocateOperands(ArgRecycler, ExpressionAllocator);
1256  E->setType(SI->getValueOperand()->getType());
1257 
1258  // Give store and loads same opcode so they value number together.
1259  E->setOpcode(0);
1260  E->op_push_back(lookupOperandLeader(SI->getPointerOperand()));
1261 
1262  // TODO: Value number heap versions. We may be able to discover
1263  // things alias analysis can't on it's own (IE that a store and a
1264  // load have the same value, and thus, it isn't clobbering the load).
1265  return E;
1266 }
1267 
1268 const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I) const {
1269  // Unlike loads, we never try to eliminate stores, so we do not check if they
1270  // are simple and avoid value numbering them.
1271  auto *SI = cast<StoreInst>(I);
1272  auto *StoreAccess = getMemoryAccess(SI);
1273  // Get the expression, if any, for the RHS of the MemoryDef.
1274  const MemoryAccess *StoreRHS = StoreAccess->getDefiningAccess();
1276  StoreRHS = MSSAWalker->getClobberingMemoryAccess(StoreAccess);
1277  // If we bypassed the use-def chains, make sure we add a use.
1278  StoreRHS = lookupMemoryLeader(StoreRHS);
1279  if (StoreRHS != StoreAccess->getDefiningAccess())
1280  addMemoryUsers(StoreRHS, StoreAccess);
1281  // If we are defined by ourselves, use the live on entry def.
1282  if (StoreRHS == StoreAccess)
1283  StoreRHS = MSSA->getLiveOnEntryDef();
1284 
1285  if (SI->isSimple()) {
1286  // See if we are defined by a previous store expression, it already has a
1287  // value, and it's the same value as our current store. FIXME: Right now, we
1288  // only do this for simple stores, we should expand to cover memcpys, etc.
1289  const auto *LastStore = createStoreExpression(SI, StoreRHS);
1290  const auto *LastCC = ExpressionToClass.lookup(LastStore);
1291  // We really want to check whether the expression we matched was a store. No
1292  // easy way to do that. However, we can check that the class we found has a
1293  // store, which, assuming the value numbering state is not corrupt, is
1294  // sufficient, because we must also be equivalent to that store's expression
1295  // for it to be in the same class as the load.
1296  if (LastCC && LastCC->getStoredValue() == LastStore->getStoredValue())
1297  return LastStore;
1298  // Also check if our value operand is defined by a load of the same memory
1299  // location, and the memory state is the same as it was then (otherwise, it
1300  // could have been overwritten later. See test32 in
1301  // transforms/DeadStoreElimination/simple.ll).
1302  if (auto *LI = dyn_cast<LoadInst>(LastStore->getStoredValue()))
1303  if ((lookupOperandLeader(LI->getPointerOperand()) ==
1304  LastStore->getOperand(0)) &&
1305  (lookupMemoryLeader(getMemoryAccess(LI)->getDefiningAccess()) ==
1306  StoreRHS))
1307  return LastStore;
1308  deleteExpression(LastStore);
1309  }
1310 
1311  // If the store is not equivalent to anything, value number it as a store that
1312  // produces a unique memory state (instead of using it's MemoryUse, we use
1313  // it's MemoryDef).
1314  return createStoreExpression(SI, StoreAccess);
1315 }
1316 
1317 // See if we can extract the value of a loaded pointer from a load, a store, or
1318 // a memory instruction.
1319 const Expression *
1320 NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
1321  LoadInst *LI, Instruction *DepInst,
1322  MemoryAccess *DefiningAccess) const {
1323  assert((!LI || LI->isSimple()) && "Not a simple load");
1324  if (auto *DepSI = dyn_cast<StoreInst>(DepInst)) {
1325  // Can't forward from non-atomic to atomic without violating memory model.
1326  // Also don't need to coerce if they are the same type, we will just
1327  // propagate.
1328  if (LI->isAtomic() > DepSI->isAtomic() ||
1329  LoadType == DepSI->getValueOperand()->getType())
1330  return nullptr;
1331  int Offset = analyzeLoadFromClobberingStore(LoadType, LoadPtr, DepSI, DL);
1332  if (Offset >= 0) {
1333  if (auto *C = dyn_cast<Constant>(
1334  lookupOperandLeader(DepSI->getValueOperand()))) {
1335  DEBUG(dbgs() << "Coercing load from store " << *DepSI << " to constant "
1336  << *C << "\n");
1337  return createConstantExpression(
1338  getConstantStoreValueForLoad(C, Offset, LoadType, DL));
1339  }
1340  }
1341 
1342  } else if (auto *DepLI = dyn_cast<LoadInst>(DepInst)) {
1343  // Can't forward from non-atomic to atomic without violating memory model.
1344  if (LI->isAtomic() > DepLI->isAtomic())
1345  return nullptr;
1346  int Offset = analyzeLoadFromClobberingLoad(LoadType, LoadPtr, DepLI, DL);
1347  if (Offset >= 0) {
1348  // We can coerce a constant load into a load.
1349  if (auto *C = dyn_cast<Constant>(lookupOperandLeader(DepLI)))
1350  if (auto *PossibleConstant =
1351  getConstantLoadValueForLoad(C, Offset, LoadType, DL)) {
1352  DEBUG(dbgs() << "Coercing load from load " << *LI << " to constant "
1353  << *PossibleConstant << "\n");
1354  return createConstantExpression(PossibleConstant);
1355  }
1356  }
1357 
1358  } else if (auto *DepMI = dyn_cast<MemIntrinsic>(DepInst)) {
1359  int Offset = analyzeLoadFromClobberingMemInst(LoadType, LoadPtr, DepMI, DL);
1360  if (Offset >= 0) {
1361  if (auto *PossibleConstant =
1362  getConstantMemInstValueForLoad(DepMI, Offset, LoadType, DL)) {
1363  DEBUG(dbgs() << "Coercing load from meminst " << *DepMI
1364  << " to constant " << *PossibleConstant << "\n");
1365  return createConstantExpression(PossibleConstant);
1366  }
1367  }
1368  }
1369 
1370  // All of the below are only true if the loaded pointer is produced
1371  // by the dependent instruction.
1372  if (LoadPtr != lookupOperandLeader(DepInst) &&
1373  !AA->isMustAlias(LoadPtr, DepInst))
1374  return nullptr;
1375  // If this load really doesn't depend on anything, then we must be loading an
1376  // undef value. This can happen when loading for a fresh allocation with no
1377  // intervening stores, for example. Note that this is only true in the case
1378  // that the result of the allocation is pointer equal to the load ptr.
1379  if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI)) {
1380  return createConstantExpression(UndefValue::get(LoadType));
1381  }
1382  // If this load occurs either right after a lifetime begin,
1383  // then the loaded value is undefined.
1384  else if (auto *II = dyn_cast<IntrinsicInst>(DepInst)) {
1385  if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1386  return createConstantExpression(UndefValue::get(LoadType));
1387  }
1388  // If this load follows a calloc (which zero initializes memory),
1389  // then the loaded value is zero
1390  else if (isCallocLikeFn(DepInst, TLI)) {
1391  return createConstantExpression(Constant::getNullValue(LoadType));
1392  }
1393 
1394  return nullptr;
1395 }
1396 
1397 const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
1398  auto *LI = cast<LoadInst>(I);
1399 
1400  // We can eliminate in favor of non-simple loads, but we won't be able to
1401  // eliminate the loads themselves.
1402  if (!LI->isSimple())
1403  return nullptr;
1404 
1405  Value *LoadAddressLeader = lookupOperandLeader(LI->getPointerOperand());
1406  // Load of undef is undef.
1407  if (isa<UndefValue>(LoadAddressLeader))
1408  return createConstantExpression(UndefValue::get(LI->getType()));
1409  MemoryAccess *OriginalAccess = getMemoryAccess(I);
1410  MemoryAccess *DefiningAccess =
1411  MSSAWalker->getClobberingMemoryAccess(OriginalAccess);
1412 
1413  if (!MSSA->isLiveOnEntryDef(DefiningAccess)) {
1414  if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {
1415  Instruction *DefiningInst = MD->getMemoryInst();
1416  // If the defining instruction is not reachable, replace with undef.
1417  if (!ReachableBlocks.count(DefiningInst->getParent()))
1418  return createConstantExpression(UndefValue::get(LI->getType()));
1419  // This will handle stores and memory insts. We only do if it the
1420  // defining access has a different type, or it is a pointer produced by
1421  // certain memory operations that cause the memory to have a fixed value
1422  // (IE things like calloc).
1423  if (const auto *CoercionResult =
1424  performSymbolicLoadCoercion(LI->getType(), LoadAddressLeader, LI,
1425  DefiningInst, DefiningAccess))
1426  return CoercionResult;
1427  }
1428  }
1429 
1430  const auto *LE = createLoadExpression(LI->getType(), LoadAddressLeader, LI,
1431  DefiningAccess);
1432  // If our MemoryLeader is not our defining access, add a use to the
1433  // MemoryLeader, so that we get reprocessed when it changes.
1434  if (LE->getMemoryLeader() != DefiningAccess)
1435  addMemoryUsers(LE->getMemoryLeader(), OriginalAccess);
1436  return LE;
1437 }
1438 
1439 const Expression *
1440 NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) const {
1441  auto *PI = PredInfo->getPredicateInfoFor(I);
1442  if (!PI)
1443  return nullptr;
1444 
1445  DEBUG(dbgs() << "Found predicate info from instruction !\n");
1446 
1447  auto *PWC = dyn_cast<PredicateWithCondition>(PI);
1448  if (!PWC)
1449  return nullptr;
1450 
1451  auto *CopyOf = I->getOperand(0);
1452  auto *Cond = PWC->Condition;
1453 
1454  // If this a copy of the condition, it must be either true or false depending
1455  // on the predicate info type and edge.
1456  if (CopyOf == Cond) {
1457  // We should not need to add predicate users because the predicate info is
1458  // already a use of this operand.
1459  if (isa<PredicateAssume>(PI))
1460  return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1461  if (auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1462  if (PBranch->TrueEdge)
1463  return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1464  return createConstantExpression(ConstantInt::getFalse(Cond->getType()));
1465  }
1466  if (auto *PSwitch = dyn_cast<PredicateSwitch>(PI))
1467  return createConstantExpression(cast<Constant>(PSwitch->CaseValue));
1468  }
1469 
1470  // Not a copy of the condition, so see what the predicates tell us about this
1471  // value. First, though, we check to make sure the value is actually a copy
1472  // of one of the condition operands. It's possible, in certain cases, for it
1473  // to be a copy of a predicateinfo copy. In particular, if two branch
1474  // operations use the same condition, and one branch dominates the other, we
1475  // will end up with a copy of a copy. This is currently a small deficiency in
1476  // predicateinfo. What will end up happening here is that we will value
1477  // number both copies the same anyway.
1478 
1479  // Everything below relies on the condition being a comparison.
1480  auto *Cmp = dyn_cast<CmpInst>(Cond);
1481  if (!Cmp)
1482  return nullptr;
1483 
1484  if (CopyOf != Cmp->getOperand(0) && CopyOf != Cmp->getOperand(1)) {
1485  DEBUG(dbgs() << "Copy is not of any condition operands!\n");
1486  return nullptr;
1487  }
1488  Value *FirstOp = lookupOperandLeader(Cmp->getOperand(0));
1489  Value *SecondOp = lookupOperandLeader(Cmp->getOperand(1));
1490  bool SwappedOps = false;
1491  // Sort the ops.
1492  if (shouldSwapOperands(FirstOp, SecondOp)) {
1493  std::swap(FirstOp, SecondOp);
1494  SwappedOps = true;
1495  }
1497  SwappedOps ? Cmp->getSwappedPredicate() : Cmp->getPredicate();
1498 
1499  if (isa<PredicateAssume>(PI)) {
1500  // If the comparison is true when the operands are equal, then we know the
1501  // operands are equal, because assumes must always be true.
1502  if (CmpInst::isTrueWhenEqual(Predicate)) {
1503  addPredicateUsers(PI, I);
1504  addAdditionalUsers(Cmp->getOperand(0), I);
1505  return createVariableOrConstant(FirstOp);
1506  }
1507  }
1508  if (const auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1509  // If we are *not* a copy of the comparison, we may equal to the other
1510  // operand when the predicate implies something about equality of
1511  // operations. In particular, if the comparison is true/false when the
1512  // operands are equal, and we are on the right edge, we know this operation
1513  // is equal to something.
1514  if ((PBranch->TrueEdge && Predicate == CmpInst::ICMP_EQ) ||
1515  (!PBranch->TrueEdge && Predicate == CmpInst::ICMP_NE)) {
1516  addPredicateUsers(PI, I);
1517  addAdditionalUsers(SwappedOps ? Cmp->getOperand(1) : Cmp->getOperand(0),
1518  I);
1519  return createVariableOrConstant(FirstOp);
1520  }
1521  // Handle the special case of floating point.
1522  if (((PBranch->TrueEdge && Predicate == CmpInst::FCMP_OEQ) ||
1523  (!PBranch->TrueEdge && Predicate == CmpInst::FCMP_UNE)) &&
1524  isa<ConstantFP>(FirstOp) && !cast<ConstantFP>(FirstOp)->isZero()) {
1525  addPredicateUsers(PI, I);
1526  addAdditionalUsers(SwappedOps ? Cmp->getOperand(1) : Cmp->getOperand(0),
1527  I);
1528  return createConstantExpression(cast<Constant>(FirstOp));
1529  }
1530  }
1531  return nullptr;
1532 }
1533 
1534 // Evaluate read only and pure calls, and create an expression result.
1535 const Expression *NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
1536  auto *CI = cast<CallInst>(I);
1537  if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1538  // Instrinsics with the returned attribute are copies of arguments.
1539  if (auto *ReturnedValue = II->getReturnedArgOperand()) {
1540  if (II->getIntrinsicID() == Intrinsic::ssa_copy)
1541  if (const auto *Result = performSymbolicPredicateInfoEvaluation(I))
1542  return Result;
1543  return createVariableOrConstant(ReturnedValue);
1544  }
1545  }
1546  if (AA->doesNotAccessMemory(CI)) {
1547  return createCallExpression(CI, TOPClass->getMemoryLeader());
1548  } else if (AA->onlyReadsMemory(CI)) {
1549  MemoryAccess *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(CI);
1550  return createCallExpression(CI, DefiningAccess);
1551  }
1552  return nullptr;
1553 }
1554 
1555 // Retrieve the memory class for a given MemoryAccess.
1556 CongruenceClass *NewGVN::getMemoryClass(const MemoryAccess *MA) const {
1557 
1558  auto *Result = MemoryAccessToClass.lookup(MA);
1559  assert(Result && "Should have found memory class");
1560  return Result;
1561 }
1562 
1563 // Update the MemoryAccess equivalence table to say that From is equal to To,
1564 // and return true if this is different from what already existed in the table.
1565 bool NewGVN::setMemoryClass(const MemoryAccess *From,
1566  CongruenceClass *NewClass) {
1567  assert(NewClass &&
1568  "Every MemoryAccess should be getting mapped to a non-null class");
1569  DEBUG(dbgs() << "Setting " << *From);
1570  DEBUG(dbgs() << " equivalent to congruence class ");
1571  DEBUG(dbgs() << NewClass->getID() << " with current MemoryAccess leader ");
1572  DEBUG(dbgs() << *NewClass->getMemoryLeader() << "\n");
1573 
1574  auto LookupResult = MemoryAccessToClass.find(From);
1575  bool Changed = false;
1576  // If it's already in the table, see if the value changed.
1577  if (LookupResult != MemoryAccessToClass.end()) {
1578  auto *OldClass = LookupResult->second;
1579  if (OldClass != NewClass) {
1580  // If this is a phi, we have to handle memory member updates.
1581  if (auto *MP = dyn_cast<MemoryPhi>(From)) {
1582  OldClass->memory_erase(MP);
1583  NewClass->memory_insert(MP);
1584  // This may have killed the class if it had no non-memory members
1585  if (OldClass->getMemoryLeader() == From) {
1586  if (OldClass->definesNoMemory()) {
1587  OldClass->setMemoryLeader(nullptr);
1588  } else {
1589  OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
1590  DEBUG(dbgs() << "Memory class leader change for class "
1591  << OldClass->getID() << " to "
1592  << *OldClass->getMemoryLeader()
1593  << " due to removal of a memory member " << *From
1594  << "\n");
1595  markMemoryLeaderChangeTouched(OldClass);
1596  }
1597  }
1598  }
1599  // It wasn't equivalent before, and now it is.
1600  LookupResult->second = NewClass;
1601  Changed = true;
1602  }
1603  }
1604 
1605  return Changed;
1606 }
1607 
1608 // Determine if a instruction is cycle-free. That means the values in the
1609 // instruction don't depend on any expressions that can change value as a result
1610 // of the instruction. For example, a non-cycle free instruction would be v =
1611 // phi(0, v+1).
1612 bool NewGVN::isCycleFree(const Instruction *I) const {
1613  // In order to compute cycle-freeness, we do SCC finding on the instruction,
1614  // and see what kind of SCC it ends up in. If it is a singleton, it is
1615  // cycle-free. If it is not in a singleton, it is only cycle free if the
1616  // other members are all phi nodes (as they do not compute anything, they are
1617  // copies).
1618  auto ICS = InstCycleState.lookup(I);
1619  if (ICS == ICS_Unknown) {
1620  SCCFinder.Start(I);
1621  auto &SCC = SCCFinder.getComponentFor(I);
1622  // It's cycle free if it's size 1 or or the SCC is *only* phi nodes.
1623  if (SCC.size() == 1)
1624  InstCycleState.insert({I, ICS_CycleFree});
1625  else {
1626  bool AllPhis = llvm::all_of(SCC, [](const Value *V) {
1627  return isa<PHINode>(V) || isCopyOfAPHI(V);
1628  });
1629  ICS = AllPhis ? ICS_CycleFree : ICS_Cycle;
1630  for (auto *Member : SCC)
1631  if (auto *MemberPhi = dyn_cast<PHINode>(Member))
1632  InstCycleState.insert({MemberPhi, ICS});
1633  }
1634  }
1635  if (ICS == ICS_Cycle)
1636  return false;
1637  return true;
1638 }
1639 
1640 // Evaluate PHI nodes symbolically and create an expression result.
1641 const Expression *
1642 NewGVN::performSymbolicPHIEvaluation(ArrayRef<ValPair> PHIOps,
1643  Instruction *I,
1644  BasicBlock *PHIBlock) const {
1645  // True if one of the incoming phi edges is a backedge.
1646  bool HasBackedge = false;
1647  // All constant tracks the state of whether all the *original* phi operands
1648  // This is really shorthand for "this phi cannot cycle due to forward
1649  // change in value of the phi is guaranteed not to later change the value of
1650  // the phi. IE it can't be v = phi(undef, v+1)
1651  bool OriginalOpsConstant = true;
1652  auto *E = cast<PHIExpression>(createPHIExpression(
1653  PHIOps, I, PHIBlock, HasBackedge, OriginalOpsConstant));
1654  // We match the semantics of SimplifyPhiNode from InstructionSimplify here.
1655  // See if all arguments are the same.
1656  // We track if any were undef because they need special handling.
1657  bool HasUndef = false;
1658  auto Filtered = make_filter_range(E->operands(), [&](Value *Arg) {
1659  if (isa<UndefValue>(Arg)) {
1660  HasUndef = true;
1661  return false;
1662  }
1663  return true;
1664  });
1665  // If we are left with no operands, it's dead.
1666  if (Filtered.begin() == Filtered.end()) {
1667  // If it has undef at this point, it means there are no-non-undef arguments,
1668  // and thus, the value of the phi node must be undef.
1669  if (HasUndef) {
1670  DEBUG(dbgs() << "PHI Node " << *I
1671  << " has no non-undef arguments, valuing it as undef\n");
1672  return createConstantExpression(UndefValue::get(I->getType()));
1673  }
1674 
1675  DEBUG(dbgs() << "No arguments of PHI node " << *I << " are live\n");
1676  deleteExpression(E);
1677  return createDeadExpression();
1678  }
1679  Value *AllSameValue = *(Filtered.begin());
1680  ++Filtered.begin();
1681  // Can't use std::equal here, sadly, because filter.begin moves.
1682  if (llvm::all_of(Filtered, [&](Value *Arg) { return Arg == AllSameValue; })) {
1683  // In LLVM's non-standard representation of phi nodes, it's possible to have
1684  // phi nodes with cycles (IE dependent on other phis that are .... dependent
1685  // on the original phi node), especially in weird CFG's where some arguments
1686  // are unreachable, or uninitialized along certain paths. This can cause
1687  // infinite loops during evaluation. We work around this by not trying to
1688  // really evaluate them independently, but instead using a variable
1689  // expression to say if one is equivalent to the other.
1690  // We also special case undef, so that if we have an undef, we can't use the
1691  // common value unless it dominates the phi block.
1692  if (HasUndef) {
1693  // If we have undef and at least one other value, this is really a
1694  // multivalued phi, and we need to know if it's cycle free in order to
1695  // evaluate whether we can ignore the undef. The other parts of this are
1696  // just shortcuts. If there is no backedge, or all operands are
1697  // constants, it also must be cycle free.
1698  if (HasBackedge && !OriginalOpsConstant &&
1699  !isa<UndefValue>(AllSameValue) && !isCycleFree(I))
1700  return E;
1701 
1702  // Only have to check for instructions
1703  if (auto *AllSameInst = dyn_cast<Instruction>(AllSameValue))
1704  if (!someEquivalentDominates(AllSameInst, I))
1705  return E;
1706  }
1707  // Can't simplify to something that comes later in the iteration.
1708  // Otherwise, when and if it changes congruence class, we will never catch
1709  // up. We will always be a class behind it.
1710  if (isa<Instruction>(AllSameValue) &&
1711  InstrToDFSNum(AllSameValue) > InstrToDFSNum(I))
1712  return E;
1713  NumGVNPhisAllSame++;
1714  DEBUG(dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue
1715  << "\n");
1716  deleteExpression(E);
1717  return createVariableOrConstant(AllSameValue);
1718  }
1719  return E;
1720 }
1721 
1722 const Expression *
1723 NewGVN::performSymbolicAggrValueEvaluation(Instruction *I) const {
1724  if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
1725  auto *II = dyn_cast<IntrinsicInst>(EI->getAggregateOperand());
1726  if (II && EI->getNumIndices() == 1 && *EI->idx_begin() == 0) {
1727  unsigned Opcode = 0;
1728  // EI might be an extract from one of our recognised intrinsics. If it
1729  // is we'll synthesize a semantically equivalent expression instead on
1730  // an extract value expression.
1731  switch (II->getIntrinsicID()) {
1732  case Intrinsic::sadd_with_overflow:
1733  case Intrinsic::uadd_with_overflow:
1734  Opcode = Instruction::Add;
1735  break;
1736  case Intrinsic::ssub_with_overflow:
1737  case Intrinsic::usub_with_overflow:
1738  Opcode = Instruction::Sub;
1739  break;
1740  case Intrinsic::smul_with_overflow:
1741  case Intrinsic::umul_with_overflow:
1742  Opcode = Instruction::Mul;
1743  break;
1744  default:
1745  break;
1746  }
1747 
1748  if (Opcode != 0) {
1749  // Intrinsic recognized. Grab its args to finish building the
1750  // expression.
1751  assert(II->getNumArgOperands() == 2 &&
1752  "Expect two args for recognised intrinsics.");
1753  return createBinaryExpression(Opcode, EI->getType(),
1754  II->getArgOperand(0),
1755  II->getArgOperand(1), I);
1756  }
1757  }
1758  }
1759 
1760  return createAggregateValueExpression(I);
1761 }
1762 const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
1763  assert(isa<CmpInst>(I) && "Expected a cmp instruction.");
1764 
1765  auto *CI = cast<CmpInst>(I);
1766  // See if our operands are equal to those of a previous predicate, and if so,
1767  // if it implies true or false.
1768  auto Op0 = lookupOperandLeader(CI->getOperand(0));
1769  auto Op1 = lookupOperandLeader(CI->getOperand(1));
1770  auto OurPredicate = CI->getPredicate();
1771  if (shouldSwapOperands(Op0, Op1)) {
1772  std::swap(Op0, Op1);
1773  OurPredicate = CI->getSwappedPredicate();
1774  }
1775 
1776  // Avoid processing the same info twice.
1777  const PredicateBase *LastPredInfo = nullptr;
1778  // See if we know something about the comparison itself, like it is the target
1779  // of an assume.
1780  auto *CmpPI = PredInfo->getPredicateInfoFor(I);
1781  if (dyn_cast_or_null<PredicateAssume>(CmpPI))
1782  return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1783 
1784  if (Op0 == Op1) {
1785  // This condition does not depend on predicates, no need to add users
1786  if (CI->isTrueWhenEqual())
1787  return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1788  else if (CI->isFalseWhenEqual())
1789  return createConstantExpression(ConstantInt::getFalse(CI->getType()));
1790  }
1791 
1792  // NOTE: Because we are comparing both operands here and below, and using
1793  // previous comparisons, we rely on fact that predicateinfo knows to mark
1794  // comparisons that use renamed operands as users of the earlier comparisons.
1795  // It is *not* enough to just mark predicateinfo renamed operands as users of
1796  // the earlier comparisons, because the *other* operand may have changed in a
1797  // previous iteration.
1798  // Example:
1799  // icmp slt %a, %b
1800  // %b.0 = ssa.copy(%b)
1801  // false branch:
1802  // icmp slt %c, %b.0
1803 
1804  // %c and %a may start out equal, and thus, the code below will say the second
1805  // %icmp is false. c may become equal to something else, and in that case the
1806  // %second icmp *must* be reexamined, but would not if only the renamed
1807  // %operands are considered users of the icmp.
1808 
1809  // *Currently* we only check one level of comparisons back, and only mark one
1810  // level back as touched when changes happen. If you modify this code to look
1811  // back farther through comparisons, you *must* mark the appropriate
1812  // comparisons as users in PredicateInfo.cpp, or you will cause bugs. See if
1813  // we know something just from the operands themselves
1814 
1815  // See if our operands have predicate info, so that we may be able to derive
1816  // something from a previous comparison.
1817  for (const auto &Op : CI->operands()) {
1818  auto *PI = PredInfo->getPredicateInfoFor(Op);
1819  if (const auto *PBranch = dyn_cast_or_null<PredicateBranch>(PI)) {
1820  if (PI == LastPredInfo)
1821  continue;
1822  LastPredInfo = PI;
1823  // In phi of ops cases, we may have predicate info that we are evaluating
1824  // in a different context.
1825  if (!DT->dominates(PBranch->To, getBlockForValue(I)))
1826  continue;
1827  // TODO: Along the false edge, we may know more things too, like
1828  // icmp of
1829  // same operands is false.
1830  // TODO: We only handle actual comparison conditions below, not
1831  // and/or.
1832  auto *BranchCond = dyn_cast<CmpInst>(PBranch->Condition);
1833  if (!BranchCond)
1834  continue;
1835  auto *BranchOp0 = lookupOperandLeader(BranchCond->getOperand(0));
1836  auto *BranchOp1 = lookupOperandLeader(BranchCond->getOperand(1));
1837  auto BranchPredicate = BranchCond->getPredicate();
1838  if (shouldSwapOperands(BranchOp0, BranchOp1)) {
1839  std::swap(BranchOp0, BranchOp1);
1840  BranchPredicate = BranchCond->getSwappedPredicate();
1841  }
1842  if (BranchOp0 == Op0 && BranchOp1 == Op1) {
1843  if (PBranch->TrueEdge) {
1844  // If we know the previous predicate is true and we are in the true
1845  // edge then we may be implied true or false.
1846  if (CmpInst::isImpliedTrueByMatchingCmp(BranchPredicate,
1847  OurPredicate)) {
1848  addPredicateUsers(PI, I);
1849  return createConstantExpression(
1850  ConstantInt::getTrue(CI->getType()));
1851  }
1852 
1853  if (CmpInst::isImpliedFalseByMatchingCmp(BranchPredicate,
1854  OurPredicate)) {
1855  addPredicateUsers(PI, I);
1856  return createConstantExpression(
1858  }
1859 
1860  } else {
1861  // Just handle the ne and eq cases, where if we have the same
1862  // operands, we may know something.
1863  if (BranchPredicate == OurPredicate) {
1864  addPredicateUsers(PI, I);
1865  // Same predicate, same ops,we know it was false, so this is false.
1866  return createConstantExpression(
1868  } else if (BranchPredicate ==
1869  CmpInst::getInversePredicate(OurPredicate)) {
1870  addPredicateUsers(PI, I);
1871  // Inverse predicate, we know the other was false, so this is true.
1872  return createConstantExpression(
1873  ConstantInt::getTrue(CI->getType()));
1874  }
1875  }
1876  }
1877  }
1878  }
1879  // Create expression will take care of simplifyCmpInst
1880  return createExpression(I);
1881 }
1882 
1883 // Substitute and symbolize the value before value numbering.
1884 const Expression *
1885 NewGVN::performSymbolicEvaluation(Value *V,
1886  SmallPtrSetImpl<Value *> &Visited) const {
1887  const Expression *E = nullptr;
1888  if (auto *C = dyn_cast<Constant>(V))
1889  E = createConstantExpression(C);
1890  else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
1891  E = createVariableExpression(V);
1892  } else {
1893  // TODO: memory intrinsics.
1894  // TODO: Some day, we should do the forward propagation and reassociation
1895  // parts of the algorithm.
1896  auto *I = cast<Instruction>(V);
1897  switch (I->getOpcode()) {
1898  case Instruction::ExtractValue:
1899  case Instruction::InsertValue:
1900  E = performSymbolicAggrValueEvaluation(I);
1901  break;
1902  case Instruction::PHI: {
1904  auto *PN = cast<PHINode>(I);
1905  for (unsigned i = 0; i < PN->getNumOperands(); ++i)
1906  Ops.push_back({PN->getIncomingValue(i), PN->getIncomingBlock(i)});
1907  // Sort to ensure the invariant createPHIExpression requires is met.
1908  sortPHIOps(Ops);
1909  E = performSymbolicPHIEvaluation(Ops, I, getBlockForValue(I));
1910  } break;
1911  case Instruction::Call:
1912  E = performSymbolicCallEvaluation(I);
1913  break;
1914  case Instruction::Store:
1915  E = performSymbolicStoreEvaluation(I);
1916  break;
1917  case Instruction::Load:
1918  E = performSymbolicLoadEvaluation(I);
1919  break;
1920  case Instruction::BitCast: {
1921  E = createExpression(I);
1922  } break;
1923  case Instruction::ICmp:
1924  case Instruction::FCmp: {
1925  E = performSymbolicCmpEvaluation(I);
1926  } break;
1927  case Instruction::Add:
1928  case Instruction::FAdd:
1929  case Instruction::Sub:
1930  case Instruction::FSub:
1931  case Instruction::Mul:
1932  case Instruction::FMul:
1933  case Instruction::UDiv:
1934  case Instruction::SDiv:
1935  case Instruction::FDiv:
1936  case Instruction::URem:
1937  case Instruction::SRem:
1938  case Instruction::FRem:
1939  case Instruction::Shl:
1940  case Instruction::LShr:
1941  case Instruction::AShr:
1942  case Instruction::And:
1943  case Instruction::Or:
1944  case Instruction::Xor:
1945  case Instruction::Trunc:
1946  case Instruction::ZExt:
1947  case Instruction::SExt:
1948  case Instruction::FPToUI:
1949  case Instruction::FPToSI:
1950  case Instruction::UIToFP:
1951  case Instruction::SIToFP:
1952  case Instruction::FPTrunc:
1953  case Instruction::FPExt:
1954  case Instruction::PtrToInt:
1955  case Instruction::IntToPtr:
1956  case Instruction::Select:
1957  case Instruction::ExtractElement:
1958  case Instruction::InsertElement:
1959  case Instruction::ShuffleVector:
1960  case Instruction::GetElementPtr:
1961  E = createExpression(I);
1962  break;
1963  default:
1964  return nullptr;
1965  }
1966  }
1967  return E;
1968 }
1969 
1970 // Look up a container in a map, and then call a function for each thing in the
1971 // found container.
1972 template <typename Map, typename KeyType, typename Func>
1973 void NewGVN::for_each_found(Map &M, const KeyType &Key, Func F) {
1974  const auto Result = M.find_as(Key);
1975  if (Result != M.end())
1976  for (typename Map::mapped_type::value_type Mapped : Result->second)
1977  F(Mapped);
1978 }
1979 
1980 // Look up a container of values/instructions in a map, and touch all the
1981 // instructions in the container. Then erase value from the map.
1982 template <typename Map, typename KeyType>
1983 void NewGVN::touchAndErase(Map &M, const KeyType &Key) {
1984  const auto Result = M.find_as(Key);
1985  if (Result != M.end()) {
1986  for (const typename Map::mapped_type::value_type Mapped : Result->second)
1987  TouchedInstructions.set(InstrToDFSNum(Mapped));
1988  M.erase(Result);
1989  }
1990 }
1991 
1992 void NewGVN::addAdditionalUsers(Value *To, Value *User) const {
1993  assert(User && To != User);
1994  if (isa<Instruction>(To))
1995  AdditionalUsers[To].insert(User);
1996 }
1997 
1998 void NewGVN::markUsersTouched(Value *V) {
1999  // Now mark the users as touched.
2000  for (auto *User : V->users()) {
2001  assert(isa<Instruction>(User) && "Use of value not within an instruction?");
2002  TouchedInstructions.set(InstrToDFSNum(User));
2003  }
2004  touchAndErase(AdditionalUsers, V);
2005 }
2006 
2007 void NewGVN::addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const {
2008  DEBUG(dbgs() << "Adding memory user " << *U << " to " << *To << "\n");
2009  MemoryToUsers[To].insert(U);
2010 }
2011 
2012 void NewGVN::markMemoryDefTouched(const MemoryAccess *MA) {
2013  TouchedInstructions.set(MemoryToDFSNum(MA));
2014 }
2015 
2016 void NewGVN::markMemoryUsersTouched(const MemoryAccess *MA) {
2017  if (isa<MemoryUse>(MA))
2018  return;
2019  for (auto U : MA->users())
2020  TouchedInstructions.set(MemoryToDFSNum(U));
2021  touchAndErase(MemoryToUsers, MA);
2022 }
2023 
2024 // Add I to the set of users of a given predicate.
2025 void NewGVN::addPredicateUsers(const PredicateBase *PB, Instruction *I) const {
2026  // Don't add temporary instructions to the user lists.
2027  if (AllTempInstructions.count(I))
2028  return;
2029 
2030  if (auto *PBranch = dyn_cast<PredicateBranch>(PB))
2031  PredicateToUsers[PBranch->Condition].insert(I);
2032  else if (auto *PAssume = dyn_cast<PredicateBranch>(PB))
2033  PredicateToUsers[PAssume->Condition].insert(I);
2034 }
2035 
2036 // Touch all the predicates that depend on this instruction.
2037 void NewGVN::markPredicateUsersTouched(Instruction *I) {
2038  touchAndErase(PredicateToUsers, I);
2039 }
2040 
2041 // Mark users affected by a memory leader change.
2042 void NewGVN::markMemoryLeaderChangeTouched(CongruenceClass *CC) {
2043  for (auto M : CC->memory())
2044  markMemoryDefTouched(M);
2045 }
2046 
2047 // Touch the instructions that need to be updated after a congruence class has a
2048 // leader change, and mark changed values.
2049 void NewGVN::markValueLeaderChangeTouched(CongruenceClass *CC) {
2050  for (auto M : *CC) {
2051  if (auto *I = dyn_cast<Instruction>(M))
2052  TouchedInstructions.set(InstrToDFSNum(I));
2053  LeaderChanges.insert(M);
2054  }
2055 }
2056 
2057 // Give a range of things that have instruction DFS numbers, this will return
2058 // the member of the range with the smallest dfs number.
2059 template <class T, class Range>
2060 T *NewGVN::getMinDFSOfRange(const Range &R) const {
2061  std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};
2062  for (const auto X : R) {
2063  auto DFSNum = InstrToDFSNum(X);
2064  if (DFSNum < MinDFS.second)
2065  MinDFS = {X, DFSNum};
2066  }
2067  return MinDFS.first;
2068 }
2069 
2070 // This function returns the MemoryAccess that should be the next leader of
2071 // congruence class CC, under the assumption that the current leader is going to
2072 // disappear.
2073 const MemoryAccess *NewGVN::getNextMemoryLeader(CongruenceClass *CC) const {
2074  // TODO: If this ends up to slow, we can maintain a next memory leader like we
2075  // do for regular leaders.
2076  // Make sure there will be a leader to find.
2077  assert(!CC->definesNoMemory() && "Can't get next leader if there is none");
2078  if (CC->getStoreCount() > 0) {
2079  if (auto *NL = dyn_cast_or_null<StoreInst>(CC->getNextLeader().first))
2080  return getMemoryAccess(NL);
2081  // Find the store with the minimum DFS number.
2082  auto *V = getMinDFSOfRange<Value>(make_filter_range(
2083  *CC, [&](const Value *V) { return isa<StoreInst>(V); }));
2084  return getMemoryAccess(cast<StoreInst>(V));
2085  }
2086  assert(CC->getStoreCount() == 0);
2087 
2088  // Given our assertion, hitting this part must mean
2089  // !OldClass->memory_empty()
2090  if (CC->memory_size() == 1)
2091  return *CC->memory_begin();
2092  return getMinDFSOfRange<const MemoryPhi>(CC->memory());
2093 }
2094 
2095 // This function returns the next value leader of a congruence class, under the
2096 // assumption that the current leader is going away. This should end up being
2097 // the next most dominating member.
2098 Value *NewGVN::getNextValueLeader(CongruenceClass *CC) const {
2099  // We don't need to sort members if there is only 1, and we don't care about
2100  // sorting the TOP class because everything either gets out of it or is
2101  // unreachable.
2102 
2103  if (CC->size() == 1 || CC == TOPClass) {
2104  return *(CC->begin());
2105  } else if (CC->getNextLeader().first) {
2106  ++NumGVNAvoidedSortedLeaderChanges;
2107  return CC->getNextLeader().first;
2108  } else {
2109  ++NumGVNSortedLeaderChanges;
2110  // NOTE: If this ends up to slow, we can maintain a dual structure for
2111  // member testing/insertion, or keep things mostly sorted, and sort only
2112  // here, or use SparseBitVector or ....
2113  return getMinDFSOfRange<Value>(*CC);
2114  }
2115 }
2116 
2117 // Move a MemoryAccess, currently in OldClass, to NewClass, including updates to
2118 // the memory members, etc for the move.
2119 //
2120 // The invariants of this function are:
2121 //
2122 // - I must be moving to NewClass from OldClass
2123 // - The StoreCount of OldClass and NewClass is expected to have been updated
2124 // for I already if it is is a store.
2125 // - The OldClass memory leader has not been updated yet if I was the leader.
2126 void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I,
2127  MemoryAccess *InstMA,
2128  CongruenceClass *OldClass,
2129  CongruenceClass *NewClass) {
2130  // If the leader is I, and we had a represenative MemoryAccess, it should
2131  // be the MemoryAccess of OldClass.
2132  assert((!InstMA || !OldClass->getMemoryLeader() ||
2133  OldClass->getLeader() != I ||
2134  MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) ==
2135  MemoryAccessToClass.lookup(InstMA)) &&
2136  "Representative MemoryAccess mismatch");
2137  // First, see what happens to the new class
2138  if (!NewClass->getMemoryLeader()) {
2139  // Should be a new class, or a store becoming a leader of a new class.
2140  assert(NewClass->size() == 1 ||
2141  (isa<StoreInst>(I) && NewClass->getStoreCount() == 1));
2142  NewClass->setMemoryLeader(InstMA);
2143  // Mark it touched if we didn't just create a singleton
2144  DEBUG(dbgs() << "Memory class leader change for class " << NewClass->getID()
2145  << " due to new memory instruction becoming leader\n");
2146  markMemoryLeaderChangeTouched(NewClass);
2147  }
2148  setMemoryClass(InstMA, NewClass);
2149  // Now, fixup the old class if necessary
2150  if (OldClass->getMemoryLeader() == InstMA) {
2151  if (!OldClass->definesNoMemory()) {
2152  OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
2153  DEBUG(dbgs() << "Memory class leader change for class "
2154  << OldClass->getID() << " to "
2155  << *OldClass->getMemoryLeader()
2156  << " due to removal of old leader " << *InstMA << "\n");
2157  markMemoryLeaderChangeTouched(OldClass);
2158  } else
2159  OldClass->setMemoryLeader(nullptr);
2160  }
2161 }
2162 
2163 // Move a value, currently in OldClass, to be part of NewClass
2164 // Update OldClass and NewClass for the move (including changing leaders, etc).
2165 void NewGVN::moveValueToNewCongruenceClass(Instruction *I, const Expression *E,
2166  CongruenceClass *OldClass,
2167  CongruenceClass *NewClass) {
2168  if (I == OldClass->getNextLeader().first)
2169  OldClass->resetNextLeader();
2170 
2171  OldClass->erase(I);
2172  NewClass->insert(I);
2173 
2174  if (NewClass->getLeader() != I)
2175  NewClass->addPossibleNextLeader({I, InstrToDFSNum(I)});
2176  // Handle our special casing of stores.
2177  if (auto *SI = dyn_cast<StoreInst>(I)) {
2178  OldClass->decStoreCount();
2179  // Okay, so when do we want to make a store a leader of a class?
2180  // If we have a store defined by an earlier load, we want the earlier load
2181  // to lead the class.
2182  // If we have a store defined by something else, we want the store to lead
2183  // the class so everything else gets the "something else" as a value.
2184  // If we have a store as the single member of the class, we want the store
2185  // as the leader
2186  if (NewClass->getStoreCount() == 0 && !NewClass->getStoredValue()) {
2187  // If it's a store expression we are using, it means we are not equivalent
2188  // to something earlier.
2189  if (auto *SE = dyn_cast<StoreExpression>(E)) {
2190  NewClass->setStoredValue(SE->getStoredValue());
2191  markValueLeaderChangeTouched(NewClass);
2192  // Shift the new class leader to be the store
2193  DEBUG(dbgs() << "Changing leader of congruence class "
2194  << NewClass->getID() << " from " << *NewClass->getLeader()
2195  << " to " << *SI << " because store joined class\n");
2196  // If we changed the leader, we have to mark it changed because we don't
2197  // know what it will do to symbolic evaluation.
2198  NewClass->setLeader(SI);
2199  }
2200  // We rely on the code below handling the MemoryAccess change.
2201  }
2202  NewClass->incStoreCount();
2203  }
2204  // True if there is no memory instructions left in a class that had memory
2205  // instructions before.
2206 
2207  // If it's not a memory use, set the MemoryAccess equivalence
2208  auto *InstMA = dyn_cast_or_null<MemoryDef>(getMemoryAccess(I));
2209  if (InstMA)
2210  moveMemoryToNewCongruenceClass(I, InstMA, OldClass, NewClass);
2211  ValueToClass[I] = NewClass;
2212  // See if we destroyed the class or need to swap leaders.
2213  if (OldClass->empty() && OldClass != TOPClass) {
2214  if (OldClass->getDefiningExpr()) {
2215  DEBUG(dbgs() << "Erasing expression " << *OldClass->getDefiningExpr()
2216  << " from table\n");
2217  // We erase it as an exact expression to make sure we don't just erase an
2218  // equivalent one.
2219  auto Iter = ExpressionToClass.find_as(
2220  ExactEqualsExpression(*OldClass->getDefiningExpr()));
2221  if (Iter != ExpressionToClass.end())
2222  ExpressionToClass.erase(Iter);
2223 #ifdef EXPENSIVE_CHECKS
2224  assert(
2225  (*OldClass->getDefiningExpr() != *E || ExpressionToClass.lookup(E)) &&
2226  "We erased the expression we just inserted, which should not happen");
2227 #endif
2228  }
2229  } else if (OldClass->getLeader() == I) {
2230  // When the leader changes, the value numbering of
2231  // everything may change due to symbolization changes, so we need to
2232  // reprocess.
2233  DEBUG(dbgs() << "Value class leader change for class " << OldClass->getID()
2234  << "\n");
2235  ++NumGVNLeaderChanges;
2236  // Destroy the stored value if there are no more stores to represent it.
2237  // Note that this is basically clean up for the expression removal that
2238  // happens below. If we remove stores from a class, we may leave it as a
2239  // class of equivalent memory phis.
2240  if (OldClass->getStoreCount() == 0) {
2241  if (OldClass->getStoredValue())
2242  OldClass->setStoredValue(nullptr);
2243  }
2244  OldClass->setLeader(getNextValueLeader(OldClass));
2245  OldClass->resetNextLeader();
2246  markValueLeaderChangeTouched(OldClass);
2247  }
2248 }
2249 
2250 // For a given expression, mark the phi of ops instructions that could have
2251 // changed as a result.
2252 void NewGVN::markPhiOfOpsChanged(const Expression *E) {
2253  touchAndErase(ExpressionToPhiOfOps, E);
2254 }
2255 
2256 // Perform congruence finding on a given value numbering expression.
2257 void NewGVN::performCongruenceFinding(Instruction *I, const Expression *E) {
2258  // This is guaranteed to return something, since it will at least find
2259  // TOP.
2260 
2261  CongruenceClass *IClass = ValueToClass.lookup(I);
2262  assert(IClass && "Should have found a IClass");
2263  // Dead classes should have been eliminated from the mapping.
2264  assert(!IClass->isDead() && "Found a dead class");
2265 
2266  CongruenceClass *EClass = nullptr;
2267  if (const auto *VE = dyn_cast<VariableExpression>(E)) {
2268  EClass = ValueToClass.lookup(VE->getVariableValue());
2269  } else if (isa<DeadExpression>(E)) {
2270  EClass = TOPClass;
2271  }
2272  if (!EClass) {
2273  auto lookupResult = ExpressionToClass.insert({E, nullptr});
2274 
2275  // If it's not in the value table, create a new congruence class.
2276  if (lookupResult.second) {
2277  CongruenceClass *NewClass = createCongruenceClass(nullptr, E);
2278  auto place = lookupResult.first;
2279  place->second = NewClass;
2280 
2281  // Constants and variables should always be made the leader.
2282  if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2283  NewClass->setLeader(CE->getConstantValue());
2284  } else if (const auto *SE = dyn_cast<StoreExpression>(E)) {
2285  StoreInst *SI = SE->getStoreInst();
2286  NewClass->setLeader(SI);
2287  NewClass->setStoredValue(SE->getStoredValue());
2288  // The RepMemoryAccess field will be filled in properly by the
2289  // moveValueToNewCongruenceClass call.
2290  } else {
2291  NewClass->setLeader(I);
2292  }
2293  assert(!isa<VariableExpression>(E) &&
2294  "VariableExpression should have been handled already");
2295 
2296  EClass = NewClass;
2297  DEBUG(dbgs() << "Created new congruence class for " << *I
2298  << " using expression " << *E << " at " << NewClass->getID()
2299  << " and leader " << *(NewClass->getLeader()));
2300  if (NewClass->getStoredValue())
2301  DEBUG(dbgs() << " and stored value " << *(NewClass->getStoredValue()));
2302  DEBUG(dbgs() << "\n");
2303  } else {
2304  EClass = lookupResult.first->second;
2305  if (isa<ConstantExpression>(E))
2306  assert((isa<Constant>(EClass->getLeader()) ||
2307  (EClass->getStoredValue() &&
2308  isa<Constant>(EClass->getStoredValue()))) &&
2309  "Any class with a constant expression should have a "
2310  "constant leader");
2311 
2312  assert(EClass && "Somehow don't have an eclass");
2313 
2314  assert(!EClass->isDead() && "We accidentally looked up a dead class");
2315  }
2316  }
2317  bool ClassChanged = IClass != EClass;
2318  bool LeaderChanged = LeaderChanges.erase(I);
2319  if (ClassChanged || LeaderChanged) {
2320  DEBUG(dbgs() << "New class " << EClass->getID() << " for expression " << *E
2321  << "\n");
2322  if (ClassChanged) {
2323  moveValueToNewCongruenceClass(I, E, IClass, EClass);
2324  markPhiOfOpsChanged(E);
2325  }
2326 
2327  markUsersTouched(I);
2328  if (MemoryAccess *MA = getMemoryAccess(I))
2329  markMemoryUsersTouched(MA);
2330  if (auto *CI = dyn_cast<CmpInst>(I))
2331  markPredicateUsersTouched(CI);
2332  }
2333  // If we changed the class of the store, we want to ensure nothing finds the
2334  // old store expression. In particular, loads do not compare against stored
2335  // value, so they will find old store expressions (and associated class
2336  // mappings) if we leave them in the table.
2337  if (ClassChanged && isa<StoreInst>(I)) {
2338  auto *OldE = ValueToExpression.lookup(I);
2339  // It could just be that the old class died. We don't want to erase it if we
2340  // just moved classes.
2341  if (OldE && isa<StoreExpression>(OldE) && *E != *OldE) {
2342  // Erase this as an exact expression to ensure we don't erase expressions
2343  // equivalent to it.
2344  auto Iter = ExpressionToClass.find_as(ExactEqualsExpression(*OldE));
2345  if (Iter != ExpressionToClass.end())
2346  ExpressionToClass.erase(Iter);
2347  }
2348  }
2349  ValueToExpression[I] = E;
2350 }
2351 
2352 // Process the fact that Edge (from, to) is reachable, including marking
2353 // any newly reachable blocks and instructions for processing.
2354 void NewGVN::updateReachableEdge(BasicBlock *From, BasicBlock *To) {
2355  // Check if the Edge was reachable before.
2356  if (ReachableEdges.insert({From, To}).second) {
2357  // If this block wasn't reachable before, all instructions are touched.
2358  if (ReachableBlocks.insert(To).second) {
2359  DEBUG(dbgs() << "Block " << getBlockName(To) << " marked reachable\n");
2360  const auto &InstRange = BlockInstRange.lookup(To);
2361  TouchedInstructions.set(InstRange.first, InstRange.second);
2362  } else {
2363  DEBUG(dbgs() << "Block " << getBlockName(To)
2364  << " was reachable, but new edge {" << getBlockName(From)
2365  << "," << getBlockName(To) << "} to it found\n");
2366 
2367  // We've made an edge reachable to an existing block, which may
2368  // impact predicates. Otherwise, only mark the phi nodes as touched, as
2369  // they are the only thing that depend on new edges. Anything using their
2370  // values will get propagated to if necessary.
2371  if (MemoryAccess *MemPhi = getMemoryAccess(To))
2372  TouchedInstructions.set(InstrToDFSNum(MemPhi));
2373 
2374  // FIXME: We should just add a union op on a Bitvector and
2375  // SparseBitVector. We can do it word by word faster than we are doing it
2376  // here.
2377  for (auto InstNum : RevisitOnReachabilityChange[To])
2378  TouchedInstructions.set(InstNum);
2379  }
2380  }
2381 }
2382 
2383 // Given a predicate condition (from a switch, cmp, or whatever) and a block,
2384 // see if we know some constant value for it already.
2385 Value *NewGVN::findConditionEquivalence(Value *Cond) const {
2386  auto Result = lookupOperandLeader(Cond);
2387  return isa<Constant>(Result) ? Result : nullptr;
2388 }
2389 
2390 // Process the outgoing edges of a block for reachability.
2391 void NewGVN::processOutgoingEdges(TerminatorInst *TI, BasicBlock *B) {
2392  // Evaluate reachability of terminator instruction.
2393  BranchInst *BR;
2394  if ((BR = dyn_cast<BranchInst>(TI)) && BR->isConditional()) {
2395  Value *Cond = BR->getCondition();
2396  Value *CondEvaluated = findConditionEquivalence(Cond);
2397  if (!CondEvaluated) {
2398  if (auto *I = dyn_cast<Instruction>(Cond)) {
2399  const Expression *E = createExpression(I);
2400  if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2401  CondEvaluated = CE->getConstantValue();
2402  }
2403  } else if (isa<ConstantInt>(Cond)) {
2404  CondEvaluated = Cond;
2405  }
2406  }
2407  ConstantInt *CI;
2408  BasicBlock *TrueSucc = BR->getSuccessor(0);
2409  BasicBlock *FalseSucc = BR->getSuccessor(1);
2410  if (CondEvaluated && (CI = dyn_cast<ConstantInt>(CondEvaluated))) {
2411  if (CI->isOne()) {
2412  DEBUG(dbgs() << "Condition for Terminator " << *TI
2413  << " evaluated to true\n");
2414  updateReachableEdge(B, TrueSucc);
2415  } else if (CI->isZero()) {
2416  DEBUG(dbgs() << "Condition for Terminator " << *TI
2417  << " evaluated to false\n");
2418  updateReachableEdge(B, FalseSucc);
2419  }
2420  } else {
2421  updateReachableEdge(B, TrueSucc);
2422  updateReachableEdge(B, FalseSucc);
2423  }
2424  } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
2425  // For switches, propagate the case values into the case
2426  // destinations.
2427 
2428  // Remember how many outgoing edges there are to every successor.
2430 
2431  Value *SwitchCond = SI->getCondition();
2432  Value *CondEvaluated = findConditionEquivalence(SwitchCond);
2433  // See if we were able to turn this switch statement into a constant.
2434  if (CondEvaluated && isa<ConstantInt>(CondEvaluated)) {
2435  auto *CondVal = cast<ConstantInt>(CondEvaluated);
2436  // We should be able to get case value for this.
2437  auto Case = *SI->findCaseValue(CondVal);
2438  if (Case.getCaseSuccessor() == SI->getDefaultDest()) {
2439  // We proved the value is outside of the range of the case.
2440  // We can't do anything other than mark the default dest as reachable,
2441  // and go home.
2442  updateReachableEdge(B, SI->getDefaultDest());
2443  return;
2444  }
2445  // Now get where it goes and mark it reachable.
2446  BasicBlock *TargetBlock = Case.getCaseSuccessor();
2447  updateReachableEdge(B, TargetBlock);
2448  } else {
2449  for (unsigned i = 0, e = SI->getNumSuccessors(); i != e; ++i) {
2450  BasicBlock *TargetBlock = SI->getSuccessor(i);
2451  ++SwitchEdges[TargetBlock];
2452  updateReachableEdge(B, TargetBlock);
2453  }
2454  }
2455  } else {
2456  // Otherwise this is either unconditional, or a type we have no
2457  // idea about. Just mark successors as reachable.
2458  for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) {
2459  BasicBlock *TargetBlock = TI->getSuccessor(i);
2460  updateReachableEdge(B, TargetBlock);
2461  }
2462 
2463  // This also may be a memory defining terminator, in which case, set it
2464  // equivalent only to itself.
2465  //
2466  auto *MA = getMemoryAccess(TI);
2467  if (MA && !isa<MemoryUse>(MA)) {
2468  auto *CC = ensureLeaderOfMemoryClass(MA);
2469  if (setMemoryClass(MA, CC))
2470  markMemoryUsersTouched(MA);
2471  }
2472  }
2473 }
2474 
2475 // Remove the PHI of Ops PHI for I
2476 void NewGVN::removePhiOfOps(Instruction *I, PHINode *PHITemp) {
2477  InstrDFS.erase(PHITemp);
2478  // It's still a temp instruction. We keep it in the array so it gets erased.
2479  // However, it's no longer used by I, or in the block
2480  TempToBlock.erase(PHITemp);
2481  RealToTemp.erase(I);
2482  // We don't remove the users from the phi node uses. This wastes a little
2483  // time, but such is life. We could use two sets to track which were there
2484  // are the start of NewGVN, and which were added, but right nowt he cost of
2485  // tracking is more than the cost of checking for more phi of ops.
2486 }
2487 
2488 // Add PHI Op in BB as a PHI of operations version of ExistingValue.
2489 void NewGVN::addPhiOfOps(PHINode *Op, BasicBlock *BB,
2490  Instruction *ExistingValue) {
2491  InstrDFS[Op] = InstrToDFSNum(ExistingValue);
2492  AllTempInstructions.insert(Op);
2493  TempToBlock[Op] = BB;
2494  RealToTemp[ExistingValue] = Op;
2495  // Add all users to phi node use, as they are now uses of the phi of ops phis
2496  // and may themselves be phi of ops.
2497  for (auto *U : ExistingValue->users())
2498  if (auto *UI = dyn_cast<Instruction>(U))
2499  PHINodeUses.insert(UI);
2500 }
2501 
2502 static bool okayForPHIOfOps(const Instruction *I) {
2503  if (!EnablePhiOfOps)
2504  return false;
2505  return isa<BinaryOperator>(I) || isa<SelectInst>(I) || isa<CmpInst>(I) ||
2506  isa<LoadInst>(I);
2507 }
2508 
2509 bool NewGVN::OpIsSafeForPHIOfOpsHelper(
2510  Value *V, const BasicBlock *PHIBlock,
2512  SmallVectorImpl<Instruction *> &Worklist) {
2513 
2514  if (!isa<Instruction>(V))
2515  return true;
2516  auto OISIt = OpSafeForPHIOfOps.find(V);
2517  if (OISIt != OpSafeForPHIOfOps.end())
2518  return OISIt->second;
2519 
2520  // Keep walking until we either dominate the phi block, or hit a phi, or run
2521  // out of things to check.
2522  if (DT->properlyDominates(getBlockForValue(V), PHIBlock)) {
2523  OpSafeForPHIOfOps.insert({V, true});
2524  return true;
2525  }
2526  // PHI in the same block.
2527  if (isa<PHINode>(V) && getBlockForValue(V) == PHIBlock) {
2528  OpSafeForPHIOfOps.insert({V, false});
2529  return false;
2530  }
2531 
2532  auto *OrigI = cast<Instruction>(V);
2533  for (auto *Op : OrigI->operand_values()) {
2534  if (!isa<Instruction>(Op))
2535  continue;
2536  // Stop now if we find an unsafe operand.
2537  auto OISIt = OpSafeForPHIOfOps.find(OrigI);
2538  if (OISIt != OpSafeForPHIOfOps.end()) {
2539  if (!OISIt->second) {
2540  OpSafeForPHIOfOps.insert({V, false});
2541  return false;
2542  }
2543  continue;
2544  }
2545  if (!Visited.insert(Op).second)
2546  continue;
2547  Worklist.push_back(cast<Instruction>(Op));
2548  }
2549  return true;
2550 }
2551 
2552 // Return true if this operand will be safe to use for phi of ops.
2553 //
2554 // The reason some operands are unsafe is that we are not trying to recursively
2555 // translate everything back through phi nodes. We actually expect some lookups
2556 // of expressions to fail. In particular, a lookup where the expression cannot
2557 // exist in the predecessor. This is true even if the expression, as shown, can
2558 // be determined to be constant.
2559 bool NewGVN::OpIsSafeForPHIOfOps(Value *V, const BasicBlock *PHIBlock,
2560  SmallPtrSetImpl<const Value *> &Visited) {
2562  if (!OpIsSafeForPHIOfOpsHelper(V, PHIBlock, Visited, Worklist))
2563  return false;
2564  while (!Worklist.empty()) {
2565  auto *I = Worklist.pop_back_val();
2566  if (!OpIsSafeForPHIOfOpsHelper(I, PHIBlock, Visited, Worklist))
2567  return false;
2568  }
2569  OpSafeForPHIOfOps.insert({V, true});
2570  return true;
2571 }
2572 
2573 // Try to find a leader for instruction TransInst, which is a phi translated
2574 // version of something in our original program. Visited is used to ensure we
2575 // don't infinite loop during translations of cycles. OrigInst is the
2576 // instruction in the original program, and PredBB is the predecessor we
2577 // translated it through.
2578 Value *NewGVN::findLeaderForInst(Instruction *TransInst,
2579  SmallPtrSetImpl<Value *> &Visited,
2580  MemoryAccess *MemAccess, Instruction *OrigInst,
2581  BasicBlock *PredBB) {
2582  unsigned IDFSNum = InstrToDFSNum(OrigInst);
2583  // Make sure it's marked as a temporary instruction.
2584  AllTempInstructions.insert(TransInst);
2585  // and make sure anything that tries to add it's DFS number is
2586  // redirected to the instruction we are making a phi of ops
2587  // for.
2588  TempToBlock.insert({TransInst, PredBB});
2589  InstrDFS.insert({TransInst, IDFSNum});
2590 
2591  const Expression *E = performSymbolicEvaluation(TransInst, Visited);
2592  InstrDFS.erase(TransInst);
2593  AllTempInstructions.erase(TransInst);
2594  TempToBlock.erase(TransInst);
2595  if (MemAccess)
2596  TempToMemory.erase(TransInst);
2597  if (!E)
2598  return nullptr;
2599  auto *FoundVal = findPHIOfOpsLeader(E, OrigInst, PredBB);
2600  if (!FoundVal) {
2601  ExpressionToPhiOfOps[E].insert(OrigInst);
2602  DEBUG(dbgs() << "Cannot find phi of ops operand for " << *TransInst
2603  << " in block " << getBlockName(PredBB) << "\n");
2604  return nullptr;
2605  }
2606  if (auto *SI = dyn_cast<StoreInst>(FoundVal))
2607  FoundVal = SI->getValueOperand();
2608  return FoundVal;
2609 }
2610 
2611 // When we see an instruction that is an op of phis, generate the equivalent phi
2612 // of ops form.
2613 const Expression *
2614 NewGVN::makePossiblePHIOfOps(Instruction *I,
2615  SmallPtrSetImpl<Value *> &Visited) {
2616  if (!okayForPHIOfOps(I))
2617  return nullptr;
2618 
2619  if (!Visited.insert(I).second)
2620  return nullptr;
2621  // For now, we require the instruction be cycle free because we don't
2622  // *always* create a phi of ops for instructions that could be done as phi
2623  // of ops, we only do it if we think it is useful. If we did do it all the
2624  // time, we could remove the cycle free check.
2625  if (!isCycleFree(I))
2626  return nullptr;
2627 
2628  SmallPtrSet<const Value *, 8> ProcessedPHIs;
2629  // TODO: We don't do phi translation on memory accesses because it's
2630  // complicated. For a load, we'd need to be able to simulate a new memoryuse,
2631  // which we don't have a good way of doing ATM.
2632  auto *MemAccess = getMemoryAccess(I);
2633  // If the memory operation is defined by a memory operation this block that
2634  // isn't a MemoryPhi, transforming the pointer backwards through a scalar phi
2635  // can't help, as it would still be killed by that memory operation.
2636  if (MemAccess && !isa<MemoryPhi>(MemAccess->getDefiningAccess()) &&
2637  MemAccess->getDefiningAccess()->getBlock() == I->getParent())
2638  return nullptr;
2639 
2640  SmallPtrSet<const Value *, 10> VisitedOps;
2641  // Convert op of phis to phi of ops
2642  for (auto *Op : I->operand_values()) {
2643  if (!isa<PHINode>(Op)) {
2644  auto *ValuePHI = RealToTemp.lookup(Op);
2645  if (!ValuePHI)
2646  continue;
2647  DEBUG(dbgs() << "Found possible dependent phi of ops\n");
2648  Op = ValuePHI;
2649  }
2650  auto *OpPHI = cast<PHINode>(Op);
2651  // No point in doing this for one-operand phis.
2652  if (OpPHI->getNumOperands() == 1)
2653  continue;
2654  if (!DebugCounter::shouldExecute(PHIOfOpsCounter))
2655  return nullptr;
2658  auto *PHIBlock = getBlockForValue(OpPHI);
2659  RevisitOnReachabilityChange[PHIBlock].reset(InstrToDFSNum(I));
2660  for (unsigned PredNum = 0; PredNum < OpPHI->getNumOperands(); ++PredNum) {
2661  auto *PredBB = OpPHI->getIncomingBlock(PredNum);
2662  Value *FoundVal = nullptr;
2663  // We could just skip unreachable edges entirely but it's tricky to do
2664  // with rewriting existing phi nodes.
2665  if (ReachableEdges.count({PredBB, PHIBlock})) {
2666  // Clone the instruction, create an expression from it that is
2667  // translated back into the predecessor, and see if we have a leader.
2668  Instruction *ValueOp = I->clone();
2669  if (MemAccess)
2670  TempToMemory.insert({ValueOp, MemAccess});
2671  bool SafeForPHIOfOps = true;
2672  VisitedOps.clear();
2673  for (auto &Op : ValueOp->operands()) {
2674  auto *OrigOp = &*Op;
2675  // When these operand changes, it could change whether there is a
2676  // leader for us or not, so we have to add additional users.
2677  if (isa<PHINode>(Op)) {
2678  Op = Op->DoPHITranslation(PHIBlock, PredBB);
2679  if (Op != OrigOp && Op != I)
2680  Deps.insert(Op);
2681  } else if (auto *ValuePHI = RealToTemp.lookup(Op)) {
2682  if (getBlockForValue(ValuePHI) == PHIBlock)
2683  Op = ValuePHI->getIncomingValue(PredNum);
2684  }
2685  // If we phi-translated the op, it must be safe.
2686  SafeForPHIOfOps =
2687  SafeForPHIOfOps &&
2688  (Op != OrigOp || OpIsSafeForPHIOfOps(Op, PHIBlock, VisitedOps));
2689  }
2690  // FIXME: For those things that are not safe we could generate
2691  // expressions all the way down, and see if this comes out to a
2692  // constant. For anything where that is true, and unsafe, we should
2693  // have made a phi-of-ops (or value numbered it equivalent to something)
2694  // for the pieces already.
2695  FoundVal = !SafeForPHIOfOps ? nullptr
2696  : findLeaderForInst(ValueOp, Visited,
2697  MemAccess, I, PredBB);
2698  ValueOp->deleteValue();
2699  if (!FoundVal)
2700  return nullptr;
2701  } else {
2702  DEBUG(dbgs() << "Skipping phi of ops operand for incoming block "
2703  << getBlockName(PredBB)
2704  << " because the block is unreachable\n");
2705  FoundVal = UndefValue::get(I->getType());
2706  RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I));
2707  }
2708 
2709  Ops.push_back({FoundVal, PredBB});
2710  DEBUG(dbgs() << "Found phi of ops operand " << *FoundVal << " in "
2711  << getBlockName(PredBB) << "\n");
2712  }
2713  for (auto Dep : Deps)
2714  addAdditionalUsers(Dep, I);
2715  sortPHIOps(Ops);
2716  auto *E = performSymbolicPHIEvaluation(Ops, I, PHIBlock);
2717  if (isa<ConstantExpression>(E) || isa<VariableExpression>(E)) {
2718  DEBUG(dbgs()
2719  << "Not creating real PHI of ops because it simplified to existing "
2720  "value or constant\n");
2721  return E;
2722  }
2723  auto *ValuePHI = RealToTemp.lookup(I);
2724  bool NewPHI = false;
2725  if (!ValuePHI) {
2726  ValuePHI =
2727  PHINode::Create(I->getType(), OpPHI->getNumOperands(), "phiofops");
2728  addPhiOfOps(ValuePHI, PHIBlock, I);
2729  NewPHI = true;
2730  NumGVNPHIOfOpsCreated++;
2731  }
2732  if (NewPHI) {
2733  for (auto PHIOp : Ops)
2734  ValuePHI->addIncoming(PHIOp.first, PHIOp.second);
2735  } else {
2736  unsigned int i = 0;
2737  for (auto PHIOp : Ops) {
2738  ValuePHI->setIncomingValue(i, PHIOp.first);
2739  ValuePHI->setIncomingBlock(i, PHIOp.second);
2740  ++i;
2741  }
2742  }
2743  RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I));
2744  DEBUG(dbgs() << "Created phi of ops " << *ValuePHI << " for " << *I
2745  << "\n");
2746 
2747  return E;
2748  }
2749  return nullptr;
2750 }
2751 
2752 // The algorithm initially places the values of the routine in the TOP
2753 // congruence class. The leader of TOP is the undetermined value `undef`.
2754 // When the algorithm has finished, values still in TOP are unreachable.
2755 void NewGVN::initializeCongruenceClasses(Function &F) {
2756  NextCongruenceNum = 0;
2757 
2758  // Note that even though we use the live on entry def as a representative
2759  // MemoryAccess, it is *not* the same as the actual live on entry def. We
2760  // have no real equivalemnt to undef for MemoryAccesses, and so we really
2761  // should be checking whether the MemoryAccess is top if we want to know if it
2762  // is equivalent to everything. Otherwise, what this really signifies is that
2763  // the access "it reaches all the way back to the beginning of the function"
2764 
2765  // Initialize all other instructions to be in TOP class.
2766  TOPClass = createCongruenceClass(nullptr, nullptr);
2767  TOPClass->setMemoryLeader(MSSA->getLiveOnEntryDef());
2768  // The live on entry def gets put into it's own class
2769  MemoryAccessToClass[MSSA->getLiveOnEntryDef()] =
2770  createMemoryClass(MSSA->getLiveOnEntryDef());
2771 
2772  for (auto DTN : nodes(DT)) {
2773  BasicBlock *BB = DTN->getBlock();
2774  // All MemoryAccesses are equivalent to live on entry to start. They must
2775  // be initialized to something so that initial changes are noticed. For
2776  // the maximal answer, we initialize them all to be the same as
2777  // liveOnEntry.
2778  auto *MemoryBlockDefs = MSSA->getBlockDefs(BB);
2779  if (MemoryBlockDefs)
2780  for (const auto &Def : *MemoryBlockDefs) {
2781  MemoryAccessToClass[&Def] = TOPClass;
2782  auto *MD = dyn_cast<MemoryDef>(&Def);
2783  // Insert the memory phis into the member list.
2784  if (!MD) {
2785  const MemoryPhi *MP = cast<MemoryPhi>(&Def);
2786  TOPClass->memory_insert(MP);
2787  MemoryPhiState.insert({MP, MPS_TOP});
2788  }
2789 
2790  if (MD && isa<StoreInst>(MD->getMemoryInst()))
2791  TOPClass->incStoreCount();
2792  }
2793 
2794  // FIXME: This is trying to discover which instructions are uses of phi
2795  // nodes. We should move this into one of the myriad of places that walk
2796  // all the operands already.
2797  for (auto &I : *BB) {
2798  if (isa<PHINode>(&I))
2799  for (auto *U : I.users())
2800  if (auto *UInst = dyn_cast<Instruction>(U))
2801  if (InstrToDFSNum(UInst) != 0 && okayForPHIOfOps(UInst))
2802  PHINodeUses.insert(UInst);
2803  // Don't insert void terminators into the class. We don't value number
2804  // them, and they just end up sitting in TOP.
2805  if (isa<TerminatorInst>(I) && I.getType()->isVoidTy())
2806  continue;
2807  TOPClass->insert(&I);
2808  ValueToClass[&I] = TOPClass;
2809  }
2810  }
2811 
2812  // Initialize arguments to be in their own unique congruence classes
2813  for (auto &FA : F.args())
2814  createSingletonCongruenceClass(&FA);
2815 }
2816 
2817 void NewGVN::cleanupTables() {
2818  for (unsigned i = 0, e = CongruenceClasses.size(); i != e; ++i) {
2819  DEBUG(dbgs() << "Congruence class " << CongruenceClasses[i]->getID()
2820  << " has " << CongruenceClasses[i]->size() << " members\n");
2821  // Make sure we delete the congruence class (probably worth switching to
2822  // a unique_ptr at some point.
2823  delete CongruenceClasses[i];
2824  CongruenceClasses[i] = nullptr;
2825  }
2826 
2827  // Destroy the value expressions
2828  SmallVector<Instruction *, 8> TempInst(AllTempInstructions.begin(),
2829  AllTempInstructions.end());
2830  AllTempInstructions.clear();
2831 
2832  // We have to drop all references for everything first, so there are no uses
2833  // left as we delete them.
2834  for (auto *I : TempInst) {
2835  I->dropAllReferences();
2836  }
2837 
2838  while (!TempInst.empty()) {
2839  auto *I = TempInst.back();
2840  TempInst.pop_back();
2841  I->deleteValue();
2842  }
2843 
2844  ValueToClass.clear();
2845  ArgRecycler.clear(ExpressionAllocator);
2846  ExpressionAllocator.Reset();
2847  CongruenceClasses.clear();
2848  ExpressionToClass.clear();
2849  ValueToExpression.clear();
2850  RealToTemp.clear();
2851  AdditionalUsers.clear();
2852  ExpressionToPhiOfOps.clear();
2853  TempToBlock.clear();
2854  TempToMemory.clear();
2855  PHINodeUses.clear();
2856  OpSafeForPHIOfOps.clear();
2857  ReachableBlocks.clear();
2858  ReachableEdges.clear();
2859 #ifndef NDEBUG
2860  ProcessedCount.clear();
2861 #endif
2862  InstrDFS.clear();
2863  InstructionsToErase.clear();
2864  DFSToInstr.clear();
2865  BlockInstRange.clear();
2866  TouchedInstructions.clear();
2867  MemoryAccessToClass.clear();
2868  PredicateToUsers.clear();
2869  MemoryToUsers.clear();
2870  RevisitOnReachabilityChange.clear();
2871 }
2872 
2873 // Assign local DFS number mapping to instructions, and leave space for Value
2874 // PHI's.
2875 std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,
2876  unsigned Start) {
2877  unsigned End = Start;
2878  if (MemoryAccess *MemPhi = getMemoryAccess(B)) {
2879  InstrDFS[MemPhi] = End++;
2880  DFSToInstr.emplace_back(MemPhi);
2881  }
2882 
2883  // Then the real block goes next.
2884  for (auto &I : *B) {
2885  // There's no need to call isInstructionTriviallyDead more than once on
2886  // an instruction. Therefore, once we know that an instruction is dead
2887  // we change its DFS number so that it doesn't get value numbered.
2888  if (isInstructionTriviallyDead(&I, TLI)) {
2889  InstrDFS[&I] = 0;
2890  DEBUG(dbgs() << "Skipping trivially dead instruction " << I << "\n");
2891  markInstructionForDeletion(&I);
2892  continue;
2893  }
2894  if (isa<PHINode>(&I))
2895  RevisitOnReachabilityChange[B].set(End);
2896  InstrDFS[&I] = End++;
2897  DFSToInstr.emplace_back(&I);
2898  }
2899 
2900  // All of the range functions taken half-open ranges (open on the end side).
2901  // So we do not subtract one from count, because at this point it is one
2902  // greater than the last instruction.
2903  return std::make_pair(Start, End);
2904 }
2905 
2906 void NewGVN::updateProcessedCount(const Value *V) {
2907 #ifndef NDEBUG
2908  if (ProcessedCount.count(V) == 0) {
2909  ProcessedCount.insert({V, 1});
2910  } else {
2911  ++ProcessedCount[V];
2912  assert(ProcessedCount[V] < 100 &&
2913  "Seem to have processed the same Value a lot");
2914  }
2915 #endif
2916 }
2917 // Evaluate MemoryPhi nodes symbolically, just like PHI nodes
2918 void NewGVN::valueNumberMemoryPhi(MemoryPhi *MP) {
2919  // If all the arguments are the same, the MemoryPhi has the same value as the
2920  // argument. Filter out unreachable blocks and self phis from our operands.
2921  // TODO: We could do cycle-checking on the memory phis to allow valueizing for
2922  // self-phi checking.
2923  const BasicBlock *PHIBlock = MP->getBlock();
2924  auto Filtered = make_filter_range(MP->operands(), [&](const Use &U) {
2925  return cast<MemoryAccess>(U) != MP &&
2926  !isMemoryAccessTOP(cast<MemoryAccess>(U)) &&
2927  ReachableEdges.count({MP->getIncomingBlock(U), PHIBlock});
2928  });
2929  // If all that is left is nothing, our memoryphi is undef. We keep it as
2930  // InitialClass. Note: The only case this should happen is if we have at
2931  // least one self-argument.
2932  if (Filtered.begin() == Filtered.end()) {
2933  if (setMemoryClass(MP, TOPClass))
2934  markMemoryUsersTouched(MP);
2935  return;
2936  }
2937 
2938  // Transform the remaining operands into operand leaders.
2939  // FIXME: mapped_iterator should have a range version.
2940  auto LookupFunc = [&](const Use &U) {
2941  return lookupMemoryLeader(cast<MemoryAccess>(U));
2942  };
2943  auto MappedBegin = map_iterator(Filtered.begin(), LookupFunc);
2944  auto MappedEnd = map_iterator(Filtered.end(), LookupFunc);
2945 
2946  // and now check if all the elements are equal.
2947  // Sadly, we can't use std::equals since these are random access iterators.
2948  const auto *AllSameValue = *MappedBegin;
2949  ++MappedBegin;
2950  bool AllEqual = std::all_of(
2951  MappedBegin, MappedEnd,
2952  [&AllSameValue](const MemoryAccess *V) { return V == AllSameValue; });
2953 
2954  if (AllEqual)
2955  DEBUG(dbgs() << "Memory Phi value numbered to " << *AllSameValue << "\n");
2956  else
2957  DEBUG(dbgs() << "Memory Phi value numbered to itself\n");
2958  // If it's equal to something, it's in that class. Otherwise, it has to be in
2959  // a class where it is the leader (other things may be equivalent to it, but
2960  // it needs to start off in its own class, which means it must have been the
2961  // leader, and it can't have stopped being the leader because it was never
2962  // removed).
2963  CongruenceClass *CC =
2964  AllEqual ? getMemoryClass(AllSameValue) : ensureLeaderOfMemoryClass(MP);
2965  auto OldState = MemoryPhiState.lookup(MP);
2966  assert(OldState != MPS_Invalid && "Invalid memory phi state");
2967  auto NewState = AllEqual ? MPS_Equivalent : MPS_Unique;
2968  MemoryPhiState[MP] = NewState;
2969  if (setMemoryClass(MP, CC) || OldState != NewState)
2970  markMemoryUsersTouched(MP);
2971 }
2972 
2973 // Value number a single instruction, symbolically evaluating, performing
2974 // congruence finding, and updating mappings.
2975 void NewGVN::valueNumberInstruction(Instruction *I) {
2976  DEBUG(dbgs() << "Processing instruction " << *I << "\n");
2977  if (!I->isTerminator()) {
2978  const Expression *Symbolized = nullptr;
2979  SmallPtrSet<Value *, 2> Visited;
2980  if (DebugCounter::shouldExecute(VNCounter)) {
2981  Symbolized = performSymbolicEvaluation(I, Visited);
2982  // Make a phi of ops if necessary
2983  if (Symbolized && !isa<ConstantExpression>(Symbolized) &&
2984  !isa<VariableExpression>(Symbolized) && PHINodeUses.count(I)) {
2985  auto *PHIE = makePossiblePHIOfOps(I, Visited);
2986  // If we created a phi of ops, use it.
2987  // If we couldn't create one, make sure we don't leave one lying around
2988  if (PHIE) {
2989  Symbolized = PHIE;
2990  } else if (auto *Op = RealToTemp.lookup(I)) {
2991  removePhiOfOps(I, Op);
2992  }
2993  }
2994 
2995  } else {
2996  // Mark the instruction as unused so we don't value number it again.
2997  InstrDFS[I] = 0;
2998  }
2999  // If we couldn't come up with a symbolic expression, use the unknown
3000  // expression
3001  if (Symbolized == nullptr)
3002  Symbolized = createUnknownExpression(I);
3003  performCongruenceFinding(I, Symbolized);
3004  } else {
3005  // Handle terminators that return values. All of them produce values we
3006  // don't currently understand. We don't place non-value producing
3007  // terminators in a class.
3008  if (!I->getType()->isVoidTy()) {
3009  auto *Symbolized = createUnknownExpression(I);
3010  performCongruenceFinding(I, Symbolized);
3011  }
3012  processOutgoingEdges(dyn_cast<TerminatorInst>(I), I->getParent());
3013  }
3014 }
3015 
3016 // Check if there is a path, using single or equal argument phi nodes, from
3017 // First to Second.
3018 bool NewGVN::singleReachablePHIPath(
3019  SmallPtrSet<const MemoryAccess *, 8> &Visited, const MemoryAccess *First,
3020  const MemoryAccess *Second) const {
3021  if (First == Second)
3022  return true;
3023  if (MSSA->isLiveOnEntryDef(First))
3024  return false;
3025 
3026  // This is not perfect, but as we're just verifying here, we can live with
3027  // the loss of precision. The real solution would be that of doing strongly
3028  // connected component finding in this routine, and it's probably not worth
3029  // the complexity for the time being. So, we just keep a set of visited
3030  // MemoryAccess and return true when we hit a cycle.
3031  if (Visited.count(First))
3032  return true;
3033  Visited.insert(First);
3034 
3035  const auto *EndDef = First;
3036  for (auto *ChainDef : optimized_def_chain(First)) {
3037  if (ChainDef == Second)
3038  return true;
3039  if (MSSA->isLiveOnEntryDef(ChainDef))
3040  return false;
3041  EndDef = ChainDef;
3042  }
3043  auto *MP = cast<MemoryPhi>(EndDef);
3044  auto ReachableOperandPred = [&](const Use &U) {
3045  return ReachableEdges.count({MP->getIncomingBlock(U), MP->getBlock()});
3046  };
3047  auto FilteredPhiArgs =
3048  make_filter_range(MP->operands(), ReachableOperandPred);
3049  SmallVector<const Value *, 32> OperandList;
3050  std::copy(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
3051  std::back_inserter(OperandList));
3052  bool Okay = OperandList.size() == 1;
3053  if (!Okay)
3054  Okay =
3055  std::equal(OperandList.begin(), OperandList.end(), OperandList.begin());
3056  if (Okay)
3057  return singleReachablePHIPath(Visited, cast<MemoryAccess>(OperandList[0]),
3058  Second);
3059  return false;
3060 }
3061 
3062 // Verify the that the memory equivalence table makes sense relative to the
3063 // congruence classes. Note that this checking is not perfect, and is currently
3064 // subject to very rare false negatives. It is only useful for
3065 // testing/debugging.
3066 void NewGVN::verifyMemoryCongruency() const {
3067 #ifndef NDEBUG
3068  // Verify that the memory table equivalence and memory member set match
3069  for (const auto *CC : CongruenceClasses) {
3070  if (CC == TOPClass || CC->isDead())
3071  continue;
3072  if (CC->getStoreCount() != 0) {
3073  assert((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) &&
3074  "Any class with a store as a leader should have a "
3075  "representative stored value");
3076  assert(CC->getMemoryLeader() &&
3077  "Any congruence class with a store should have a "
3078  "representative access");
3079  }
3080 
3081  if (CC->getMemoryLeader())
3082  assert(MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC &&
3083  "Representative MemoryAccess does not appear to be reverse "
3084  "mapped properly");
3085  for (auto M : CC->memory())
3086  assert(MemoryAccessToClass.lookup(M) == CC &&
3087  "Memory member does not appear to be reverse mapped properly");
3088  }
3089 
3090  // Anything equivalent in the MemoryAccess table should be in the same
3091  // congruence class.
3092 
3093  // Filter out the unreachable and trivially dead entries, because they may
3094  // never have been updated if the instructions were not processed.
3095  auto ReachableAccessPred =
3096  [&](const std::pair<const MemoryAccess *, CongruenceClass *> Pair) {
3097  bool Result = ReachableBlocks.count(Pair.first->getBlock());
3098  if (!Result || MSSA->isLiveOnEntryDef(Pair.first) ||
3099  MemoryToDFSNum(Pair.first) == 0)
3100  return false;
3101  if (auto *MemDef = dyn_cast<MemoryDef>(Pair.first))
3102  return !isInstructionTriviallyDead(MemDef->getMemoryInst());
3103 
3104  // We could have phi nodes which operands are all trivially dead,
3105  // so we don't process them.
3106  if (auto *MemPHI = dyn_cast<MemoryPhi>(Pair.first)) {
3107  for (auto &U : MemPHI->incoming_values()) {
3108  if (auto *I = dyn_cast<Instruction>(&*U)) {
3110  return true;
3111  }
3112  }
3113  return false;
3114  }
3115 
3116  return true;
3117  };
3118 
3119  auto Filtered = make_filter_range(MemoryAccessToClass, ReachableAccessPred);
3120  for (auto KV : Filtered) {
3121  if (auto *FirstMUD = dyn_cast<MemoryUseOrDef>(KV.first)) {
3122  auto *SecondMUD = dyn_cast<MemoryUseOrDef>(KV.second->getMemoryLeader());
3123  if (FirstMUD && SecondMUD) {
3125  assert((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) ||
3126  ValueToClass.lookup(FirstMUD->getMemoryInst()) ==
3127  ValueToClass.lookup(SecondMUD->getMemoryInst())) &&
3128  "The instructions for these memory operations should have "
3129  "been in the same congruence class or reachable through"
3130  "a single argument phi");
3131  }
3132  } else if (auto *FirstMP = dyn_cast<MemoryPhi>(KV.first)) {
3133  // We can only sanely verify that MemoryDefs in the operand list all have
3134  // the same class.
3135  auto ReachableOperandPred = [&](const Use &U) {
3136  return ReachableEdges.count(
3137  {FirstMP->getIncomingBlock(U), FirstMP->getBlock()}) &&
3138  isa<MemoryDef>(U);
3139 
3140  };
3141  // All arguments should in the same class, ignoring unreachable arguments
3142  auto FilteredPhiArgs =
3143  make_filter_range(FirstMP->operands(), ReachableOperandPred);
3145  std::transform(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
3146  std::back_inserter(PhiOpClasses), [&](const Use &U) {
3147  const MemoryDef *MD = cast<MemoryDef>(U);
3148  return ValueToClass.lookup(MD->getMemoryInst());
3149  });
3150  assert(std::equal(PhiOpClasses.begin(), PhiOpClasses.end(),
3151  PhiOpClasses.begin()) &&
3152  "All MemoryPhi arguments should be in the same class");
3153  }
3154  }
3155 #endif
3156 }
3157 
3158 // Verify that the sparse propagation we did actually found the maximal fixpoint
3159 // We do this by storing the value to class mapping, touching all instructions,
3160 // and redoing the iteration to see if anything changed.
3161 void NewGVN::verifyIterationSettled(Function &F) {
3162 #ifndef NDEBUG
3163  DEBUG(dbgs() << "Beginning iteration verification\n");
3164  if (DebugCounter::isCounterSet(VNCounter))
3165  DebugCounter::setCounterValue(VNCounter, StartingVNCounter);
3166 
3167  // Note that we have to store the actual classes, as we may change existing
3168  // classes during iteration. This is because our memory iteration propagation
3169  // is not perfect, and so may waste a little work. But it should generate
3170  // exactly the same congruence classes we have now, with different IDs.
3171  std::map<const Value *, CongruenceClass> BeforeIteration;
3172 
3173  for (auto &KV : ValueToClass) {
3174  if (auto *I = dyn_cast<Instruction>(KV.first))
3175  // Skip unused/dead instructions.
3176  if (InstrToDFSNum(I) == 0)
3177  continue;
3178  BeforeIteration.insert({KV.first, *KV.second});
3179  }
3180 
3181  TouchedInstructions.set();
3182  TouchedInstructions.reset(0);
3183  iterateTouchedInstructions();
3185  EqualClasses;
3186  for (const auto &KV : ValueToClass) {
3187  if (auto *I = dyn_cast<Instruction>(KV.first))
3188  // Skip unused/dead instructions.
3189  if (InstrToDFSNum(I) == 0)
3190  continue;
3191  // We could sink these uses, but i think this adds a bit of clarity here as
3192  // to what we are comparing.
3193  auto *BeforeCC = &BeforeIteration.find(KV.first)->second;
3194  auto *AfterCC = KV.second;
3195  // Note that the classes can't change at this point, so we memoize the set
3196  // that are equal.
3197  if (!EqualClasses.count({BeforeCC, AfterCC})) {
3198  assert(BeforeCC->isEquivalentTo(AfterCC) &&
3199  "Value number changed after main loop completed!");
3200  EqualClasses.insert({BeforeCC, AfterCC});
3201  }
3202  }
3203 #endif
3204 }
3205 
3206 // Verify that for each store expression in the expression to class mapping,
3207 // only the latest appears, and multiple ones do not appear.
3208 // Because loads do not use the stored value when doing equality with stores,
3209 // if we don't erase the old store expressions from the table, a load can find
3210 // a no-longer valid StoreExpression.
3211 void NewGVN::verifyStoreExpressions() const {
3212 #ifndef NDEBUG
3213  // This is the only use of this, and it's not worth defining a complicated
3214  // densemapinfo hash/equality function for it.
3215  std::set<
3216  std::pair<const Value *,
3217  std::tuple<const Value *, const CongruenceClass *, Value *>>>
3218  StoreExpressionSet;
3219  for (const auto &KV : ExpressionToClass) {
3220  if (auto *SE = dyn_cast<StoreExpression>(KV.first)) {
3221  // Make sure a version that will conflict with loads is not already there
3222  auto Res = StoreExpressionSet.insert(
3223  {SE->getOperand(0), std::make_tuple(SE->getMemoryLeader(), KV.second,
3224  SE->getStoredValue())});
3225  bool Okay = Res.second;
3226  // It's okay to have the same expression already in there if it is
3227  // identical in nature.
3228  // This can happen when the leader of the stored value changes over time.
3229  if (!Okay)
3230  Okay = (std::get<1>(Res.first->second) == KV.second) &&
3231  (lookupOperandLeader(std::get<2>(Res.first->second)) ==
3232  lookupOperandLeader(SE->getStoredValue()));
3233  assert(Okay && "Stored expression conflict exists in expression table");
3234  auto *ValueExpr = ValueToExpression.lookup(SE->getStoreInst());
3235  assert(ValueExpr && ValueExpr->equals(*SE) &&
3236  "StoreExpression in ExpressionToClass is not latest "
3237  "StoreExpression for value");
3238  }
3239  }
3240 #endif
3241 }
3242 
3243 // This is the main value numbering loop, it iterates over the initial touched
3244 // instruction set, propagating value numbers, marking things touched, etc,
3245 // until the set of touched instructions is completely empty.
3246 void NewGVN::iterateTouchedInstructions() {
3247  unsigned int Iterations = 0;
3248  // Figure out where touchedinstructions starts
3249  int FirstInstr = TouchedInstructions.find_first();
3250  // Nothing set, nothing to iterate, just return.
3251  if (FirstInstr == -1)
3252  return;
3253  const BasicBlock *LastBlock = getBlockForValue(InstrFromDFSNum(FirstInstr));
3254  while (TouchedInstructions.any()) {
3255  ++Iterations;
3256  // Walk through all the instructions in all the blocks in RPO.
3257  // TODO: As we hit a new block, we should push and pop equalities into a
3258  // table lookupOperandLeader can use, to catch things PredicateInfo
3259  // might miss, like edge-only equivalences.
3260  for (unsigned InstrNum : TouchedInstructions.set_bits()) {
3261 
3262  // This instruction was found to be dead. We don't bother looking
3263  // at it again.
3264  if (InstrNum == 0) {
3265  TouchedInstructions.reset(InstrNum);
3266  continue;
3267  }
3268 
3269  Value *V = InstrFromDFSNum(InstrNum);
3270  const BasicBlock *CurrBlock = getBlockForValue(V);
3271 
3272  // If we hit a new block, do reachability processing.
3273  if (CurrBlock != LastBlock) {
3274  LastBlock = CurrBlock;
3275  bool BlockReachable = ReachableBlocks.count(CurrBlock);
3276  const auto &CurrInstRange = BlockInstRange.lookup(CurrBlock);
3277 
3278  // If it's not reachable, erase any touched instructions and move on.
3279  if (!BlockReachable) {
3280  TouchedInstructions.reset(CurrInstRange.first, CurrInstRange.second);
3281  DEBUG(dbgs() << "Skipping instructions in block "
3282  << getBlockName(CurrBlock)
3283  << " because it is unreachable\n");
3284  continue;
3285  }
3286  updateProcessedCount(CurrBlock);
3287  }
3288  // Reset after processing (because we may mark ourselves as touched when
3289  // we propagate equalities).
3290  TouchedInstructions.reset(InstrNum);
3291 
3292  if (auto *MP = dyn_cast<MemoryPhi>(V)) {
3293  DEBUG(dbgs() << "Processing MemoryPhi " << *MP << "\n");
3294  valueNumberMemoryPhi(MP);
3295  } else if (auto *I = dyn_cast<Instruction>(V)) {
3296  valueNumberInstruction(I);
3297  } else {
3298  llvm_unreachable("Should have been a MemoryPhi or Instruction");
3299  }
3300  updateProcessedCount(V);
3301  }
3302  }
3303  NumGVNMaxIterations = std::max(NumGVNMaxIterations.getValue(), Iterations);
3304 }
3305 
3306 // This is the main transformation entry point.
3307 bool NewGVN::runGVN() {
3308  if (DebugCounter::isCounterSet(VNCounter))
3309  StartingVNCounter = DebugCounter::getCounterValue(VNCounter);
3310  bool Changed = false;
3311  NumFuncArgs = F.arg_size();
3312  MSSAWalker = MSSA->getWalker();
3313  SingletonDeadExpression = new (ExpressionAllocator) DeadExpression();
3314 
3315  // Count number of instructions for sizing of hash tables, and come
3316  // up with a global dfs numbering for instructions.
3317  unsigned ICount = 1;
3318  // Add an empty instruction to account for the fact that we start at 1
3319  DFSToInstr.emplace_back(nullptr);
3320  // Note: We want ideal RPO traversal of the blocks, which is not quite the
3321  // same as dominator tree order, particularly with regard whether backedges
3322  // get visited first or second, given a block with multiple successors.
3323  // If we visit in the wrong order, we will end up performing N times as many
3324  // iterations.
3325  // The dominator tree does guarantee that, for a given dom tree node, it's
3326  // parent must occur before it in the RPO ordering. Thus, we only need to sort
3327  // the siblings.
3329  unsigned Counter = 0;
3330  for (auto &B : RPOT) {
3331  auto *Node = DT->getNode(B);
3332  assert(Node && "RPO and Dominator tree should have same reachability");
3333  RPOOrdering[Node] = ++Counter;
3334  }
3335  // Sort dominator tree children arrays into RPO.
3336  for (auto &B : RPOT) {
3337  auto *Node = DT->getNode(B);
3338  if (Node->getChildren().size() > 1)
3339  std::sort(Node->begin(), Node->end(),
3340  [&](const DomTreeNode *A, const DomTreeNode *B) {
3341  return RPOOrdering[A] < RPOOrdering[B];
3342  });
3343  }
3344 
3345  // Now a standard depth first ordering of the domtree is equivalent to RPO.
3346  for (auto DTN : depth_first(DT->getRootNode())) {
3347  BasicBlock *B = DTN->getBlock();
3348  const auto &BlockRange = assignDFSNumbers(B, ICount);
3349  BlockInstRange.insert({B, BlockRange});
3350  ICount += BlockRange.second - BlockRange.first;
3351  }
3352  initializeCongruenceClasses(F);
3353 
3354  TouchedInstructions.resize(ICount);
3355  // Ensure we don't end up resizing the expressionToClass map, as
3356  // that can be quite expensive. At most, we have one expression per
3357  // instruction.
3358  ExpressionToClass.reserve(ICount);
3359 
3360  // Initialize the touched instructions to include the entry block.
3361  const auto &InstRange = BlockInstRange.lookup(&F.getEntryBlock());
3362  TouchedInstructions.set(InstRange.first, InstRange.second);
3363  DEBUG(dbgs() << "Block " << getBlockName(&F.getEntryBlock())
3364  << " marked reachable\n");
3365  ReachableBlocks.insert(&F.getEntryBlock());
3366 
3367  iterateTouchedInstructions();
3368  verifyMemoryCongruency();
3369  verifyIterationSettled(F);
3370  verifyStoreExpressions();
3371 
3372  Changed |= eliminateInstructions(F);
3373 
3374  // Delete all instructions marked for deletion.
3375  for (Instruction *ToErase : InstructionsToErase) {
3376  if (!ToErase->use_empty())
3377  ToErase->replaceAllUsesWith(UndefValue::get(ToErase->getType()));
3378 
3379  if (ToErase->getParent())
3380  ToErase->eraseFromParent();
3381  }
3382 
3383  // Delete all unreachable blocks.
3384  auto UnreachableBlockPred = [&](const BasicBlock &BB) {
3385  return !ReachableBlocks.count(&BB);
3386  };
3387 
3388  for (auto &BB : make_filter_range(F, UnreachableBlockPred)) {
3389  DEBUG(dbgs() << "We believe block " << getBlockName(&BB)
3390  << " is unreachable\n");
3391  deleteInstructionsInBlock(&BB);
3392  Changed = true;
3393  }
3394 
3395  cleanupTables();
3396  return Changed;
3397 }
3398 
3400  int DFSIn = 0;
3401  int DFSOut = 0;
3402  int LocalNum = 0;
3403  // Only one of Def and U will be set.
3404  // The bool in the Def tells us whether the Def is the stored value of a
3405  // store.
3407  Use *U = nullptr;
3408  bool operator<(const ValueDFS &Other) const {
3409  // It's not enough that any given field be less than - we have sets
3410  // of fields that need to be evaluated together to give a proper ordering.
3411  // For example, if you have;
3412  // DFS (1, 3)
3413  // Val 0
3414  // DFS (1, 2)
3415  // Val 50
3416  // We want the second to be less than the first, but if we just go field
3417  // by field, we will get to Val 0 < Val 50 and say the first is less than
3418  // the second. We only want it to be less than if the DFS orders are equal.
3419  //
3420  // Each LLVM instruction only produces one value, and thus the lowest-level
3421  // differentiator that really matters for the stack (and what we use as as a
3422  // replacement) is the local dfs number.
3423  // Everything else in the structure is instruction level, and only affects
3424  // the order in which we will replace operands of a given instruction.
3425  //
3426  // For a given instruction (IE things with equal dfsin, dfsout, localnum),
3427  // the order of replacement of uses does not matter.
3428  // IE given,
3429  // a = 5
3430  // b = a + a
3431  // When you hit b, you will have two valuedfs with the same dfsin, out, and
3432  // localnum.
3433  // The .val will be the same as well.
3434  // The .u's will be different.
3435  // You will replace both, and it does not matter what order you replace them
3436  // in (IE whether you replace operand 2, then operand 1, or operand 1, then
3437  // operand 2).
3438  // Similarly for the case of same dfsin, dfsout, localnum, but different
3439  // .val's
3440  // a = 5
3441  // b = 6
3442  // c = a + b
3443  // in c, we will a valuedfs for a, and one for b,with everything the same
3444  // but .val and .u.
3445  // It does not matter what order we replace these operands in.
3446  // You will always end up with the same IR, and this is guaranteed.
3447  return std::tie(DFSIn, DFSOut, LocalNum, Def, U) <
3448  std::tie(Other.DFSIn, Other.DFSOut, Other.LocalNum, Other.Def,
3449  Other.U);
3450  }
3451 };
3452 
3453 // This function converts the set of members for a congruence class from values,
3454 // to sets of defs and uses with associated DFS info. The total number of
3455 // reachable uses for each value is stored in UseCount, and instructions that
3456 // seem
3457 // dead (have no non-dead uses) are stored in ProbablyDead.
3458 void NewGVN::convertClassToDFSOrdered(
3459  const CongruenceClass &Dense, SmallVectorImpl<ValueDFS> &DFSOrderedSet,
3461  SmallPtrSetImpl<Instruction *> &ProbablyDead) const {
3462  for (auto D : Dense) {
3463  // First add the value.
3464  BasicBlock *BB = getBlockForValue(D);
3465  // Constants are handled prior to ever calling this function, so
3466  // we should only be left with instructions as members.
3467  assert(BB && "Should have figured out a basic block for value");
3468  ValueDFS VDDef;
3469  DomTreeNode *DomNode = DT->getNode(BB);
3470  VDDef.DFSIn = DomNode->getDFSNumIn();
3471  VDDef.DFSOut = DomNode->getDFSNumOut();
3472  // If it's a store, use the leader of the value operand, if it's always
3473  // available, or the value operand. TODO: We could do dominance checks to
3474  // find a dominating leader, but not worth it ATM.
3475  if (auto *SI = dyn_cast<StoreInst>(D)) {
3476  auto Leader = lookupOperandLeader(SI->getValueOperand());
3477  if (alwaysAvailable(Leader)) {
3478  VDDef.Def.setPointer(Leader);
3479  } else {
3480  VDDef.Def.setPointer(SI->getValueOperand());
3481  VDDef.Def.setInt(true);
3482  }
3483  } else {
3484  VDDef.Def.setPointer(D);
3485  }
3486  assert(isa<Instruction>(D) &&
3487  "The dense set member should always be an instruction");
3488  Instruction *Def = cast<Instruction>(D);
3489  VDDef.LocalNum = InstrToDFSNum(D);
3490  DFSOrderedSet.push_back(VDDef);
3491  // If there is a phi node equivalent, add it
3492  if (auto *PN = RealToTemp.lookup(Def)) {
3493  auto *PHIE =
3494  dyn_cast_or_null<PHIExpression>(ValueToExpression.lookup(Def));
3495  if (PHIE) {
3496  VDDef.Def.setInt(false);
3497  VDDef.Def.setPointer(PN);
3498  VDDef.LocalNum = 0;
3499  DFSOrderedSet.push_back(VDDef);
3500  }
3501  }
3502 
3503  unsigned int UseCount = 0;
3504  // Now add the uses.
3505  for (auto &U : Def->uses()) {
3506  if (auto *I = dyn_cast<Instruction>(U.getUser())) {
3507  // Don't try to replace into dead uses
3508  if (InstructionsToErase.count(I))
3509  continue;
3510  ValueDFS VDUse;
3511  // Put the phi node uses in the incoming block.
3512  BasicBlock *IBlock;
3513  if (auto *P = dyn_cast<PHINode>(I)) {
3514  IBlock = P->getIncomingBlock(U);
3515  // Make phi node users appear last in the incoming block
3516  // they are from.
3517  VDUse.LocalNum = InstrDFS.size() + 1;
3518  } else {
3519  IBlock = getBlockForValue(I);
3520  VDUse.LocalNum = InstrToDFSNum(I);
3521  }
3522 
3523  // Skip uses in unreachable blocks, as we're going
3524  // to delete them.
3525  if (ReachableBlocks.count(IBlock) == 0)
3526  continue;
3527 
3528  DomTreeNode *DomNode = DT->getNode(IBlock);
3529  VDUse.DFSIn = DomNode->getDFSNumIn();
3530  VDUse.DFSOut = DomNode->getDFSNumOut();
3531  VDUse.U = &U;
3532  ++UseCount;
3533  DFSOrderedSet.emplace_back(VDUse);
3534  }
3535  }
3536 
3537  // If there are no uses, it's probably dead (but it may have side-effects,
3538  // so not definitely dead. Otherwise, store the number of uses so we can
3539  // track if it becomes dead later).
3540  if (UseCount == 0)
3541  ProbablyDead.insert(Def);
3542  else
3543  UseCounts[Def] = UseCount;
3544  }
3545 }
3546 
3547 // This function converts the set of members for a congruence class from values,
3548 // to the set of defs for loads and stores, with associated DFS info.
3549 void NewGVN::convertClassToLoadsAndStores(
3550  const CongruenceClass &Dense,
3551  SmallVectorImpl<ValueDFS> &LoadsAndStores) const {
3552  for (auto D : Dense) {
3553  if (!isa<LoadInst>(D) && !isa<StoreInst>(D))
3554  continue;
3555 
3556  BasicBlock *BB = getBlockForValue(D);
3557  ValueDFS VD;
3558  DomTreeNode *DomNode = DT->getNode(BB);
3559  VD.DFSIn = DomNode->getDFSNumIn();
3560  VD.DFSOut = DomNode->getDFSNumOut();
3561  VD.Def.setPointer(D);
3562 
3563  // If it's an instruction, use the real local dfs number.
3564  if (auto *I = dyn_cast<Instruction>(D))
3565  VD.LocalNum = InstrToDFSNum(I);
3566  else
3567  llvm_unreachable("Should have been an instruction");
3568 
3569  LoadsAndStores.emplace_back(VD);
3570  }
3571 }
3572 
3574  auto *ReplInst = dyn_cast<Instruction>(Repl);
3575  if (!ReplInst)
3576  return;
3577 
3578  // Patch the replacement so that it is not more restrictive than the value
3579  // being replaced.
3580  // Note that if 'I' is a load being replaced by some operation,
3581  // for example, by an arithmetic operation, then andIRFlags()
3582  // would just erase all math flags from the original arithmetic
3583  // operation, which is clearly not wanted and not needed.
3584  if (!isa<LoadInst>(I))
3585  ReplInst->andIRFlags(I);
3586 
3587  // FIXME: If both the original and replacement value are part of the
3588  // same control-flow region (meaning that the execution of one
3589  // guarantees the execution of the other), then we can combine the
3590  // noalias scopes here and do better than the general conservative
3591  // answer used in combineMetadata().
3592 
3593  // In general, GVN unifies expressions over different control-flow
3594  // regions, and so we need a conservative combination of the noalias
3595  // scopes.
3596  static const unsigned KnownIDs[] = {
3601  combineMetadata(ReplInst, I, KnownIDs);
3602 }
3603 
3605  patchReplacementInstruction(I, Repl);
3606  I->replaceAllUsesWith(Repl);
3607 }
3608 
3609 void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) {
3610  DEBUG(dbgs() << " BasicBlock Dead:" << *BB);
3611  ++NumGVNBlocksDeleted;
3612 
3613  // Delete the instructions backwards, as it has a reduced likelihood of having
3614  // to update as many def-use and use-def chains. Start after the terminator.
3615  auto StartPoint = BB->rbegin();
3616  ++StartPoint;
3617  // Note that we explicitly recalculate BB->rend() on each iteration,
3618  // as it may change when we remove the first instruction.
3619  for (BasicBlock::reverse_iterator I(StartPoint); I != BB->rend();) {
3620  Instruction &Inst = *I++;
3621  if (!Inst.use_empty())
3623  if (isa<LandingPadInst>(Inst))
3624  continue;
3625 
3626  Inst.eraseFromParent();
3627  ++NumGVNInstrDeleted;
3628  }
3629  // Now insert something that simplifycfg will turn into an unreachable.
3630  Type *Int8Ty = Type::getInt8Ty(BB->getContext());
3631  new StoreInst(UndefValue::get(Int8Ty),
3633  BB->getTerminator());
3634 }
3635 
3636 void NewGVN::markInstructionForDeletion(Instruction *I) {
3637  DEBUG(dbgs() << "Marking " << *I << " for deletion\n");
3638  InstructionsToErase.insert(I);
3639 }
3640 
3641 void NewGVN::replaceInstruction(Instruction *I, Value *V) {
3642 
3643  DEBUG(dbgs() << "Replacing " << *I << " with " << *V << "\n");
3645  // We save the actual erasing to avoid invalidating memory
3646  // dependencies until we are done with everything.
3647  markInstructionForDeletion(I);
3648 }
3649 
3650 namespace {
3651 
3652 // This is a stack that contains both the value and dfs info of where
3653 // that value is valid.
3654 class ValueDFSStack {
3655 public:
3656  Value *back() const { return ValueStack.back(); }
3657  std::pair<int, int> dfs_back() const { return DFSStack.back(); }
3658 
3659  void push_back(Value *V, int DFSIn, int DFSOut) {
3660  ValueStack.emplace_back(V);
3661  DFSStack.emplace_back(DFSIn, DFSOut);
3662  }
3663  bool empty() const { return DFSStack.empty(); }
3664  bool isInScope(int DFSIn, int DFSOut) const {
3665  if (empty())
3666  return false;
3667  return DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second;
3668  }
3669 
3670  void popUntilDFSScope(int DFSIn, int DFSOut) {
3671 
3672  // These two should always be in sync at this point.
3673  assert(ValueStack.size() == DFSStack.size() &&
3674  "Mismatch between ValueStack and DFSStack");
3675  while (
3676  !DFSStack.empty() &&
3677  !(DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second)) {
3678  DFSStack.pop_back();
3679  ValueStack.pop_back();
3680  }
3681  }
3682 
3683 private:
3684  SmallVector<Value *, 8> ValueStack;
3685  SmallVector<std::pair<int, int>, 8> DFSStack;
3686 };
3687 }
3688 
3689 // Given an expression, get the congruence class for it.
3690 CongruenceClass *NewGVN::getClassForExpression(const Expression *E) const {
3691  if (auto *VE = dyn_cast<VariableExpression>(E))
3692  return ValueToClass.lookup(VE->getVariableValue());
3693  else if (isa<DeadExpression>(E))
3694  return TOPClass;
3695  return ExpressionToClass.lookup(E);
3696 }
3697 
3698 // Given a value and a basic block we are trying to see if it is available in,
3699 // see if the value has a leader available in that block.
3700 Value *NewGVN::findPHIOfOpsLeader(const Expression *E,
3701  const Instruction *OrigInst,
3702  const BasicBlock *BB) const {
3703  // It would already be constant if we could make it constant
3704  if (auto *CE = dyn_cast<ConstantExpression>(E))
3705  return CE->getConstantValue();
3706  if (auto *VE = dyn_cast<VariableExpression>(E)) {
3707  auto *V = VE->getVariableValue();
3708  if (alwaysAvailable(V) || DT->dominates(getBlockForValue(V), BB))
3709  return VE->getVariableValue();
3710  }
3711 
3712  auto *CC = getClassForExpression(E);
3713  if (!CC)
3714  return nullptr;
3715  if (alwaysAvailable(CC->getLeader()))
3716  return CC->getLeader();
3717 
3718  for (auto Member : *CC) {
3719  auto *MemberInst = dyn_cast<Instruction>(Member);
3720  if (MemberInst == OrigInst)
3721  continue;
3722  // Anything that isn't an instruction is always available.
3723  if (!MemberInst)
3724  return Member;
3725  if (DT->dominates(getBlockForValue(MemberInst), BB))
3726  return Member;
3727  }
3728  return nullptr;
3729 }
3730 
3731 bool NewGVN::eliminateInstructions(Function &F) {
3732  // This is a non-standard eliminator. The normal way to eliminate is
3733  // to walk the dominator tree in order, keeping track of available
3734  // values, and eliminating them. However, this is mildly
3735  // pointless. It requires doing lookups on every instruction,
3736  // regardless of whether we will ever eliminate it. For
3737  // instructions part of most singleton congruence classes, we know we
3738  // will never eliminate them.
3739 
3740  // Instead, this eliminator looks at the congruence classes directly, sorts
3741  // them into a DFS ordering of the dominator tree, and then we just
3742  // perform elimination straight on the sets by walking the congruence
3743  // class member uses in order, and eliminate the ones dominated by the
3744  // last member. This is worst case O(E log E) where E = number of
3745  // instructions in a single congruence class. In theory, this is all
3746  // instructions. In practice, it is much faster, as most instructions are
3747  // either in singleton congruence classes or can't possibly be eliminated
3748  // anyway (if there are no overlapping DFS ranges in class).
3749  // When we find something not dominated, it becomes the new leader
3750  // for elimination purposes.
3751  // TODO: If we wanted to be faster, We could remove any members with no
3752  // overlapping ranges while sorting, as we will never eliminate anything
3753  // with those members, as they don't dominate anything else in our set.
3754 
3755  bool AnythingReplaced = false;
3756 
3757  // Since we are going to walk the domtree anyway, and we can't guarantee the
3758  // DFS numbers are updated, we compute some ourselves.
3759  DT->updateDFSNumbers();
3760 
3761  // Go through all of our phi nodes, and kill the arguments associated with
3762  // unreachable edges.
3763  auto ReplaceUnreachablePHIArgs = [&](PHINode *PHI, BasicBlock *BB) {
3764  for (auto &Operand : PHI->incoming_values())
3765  if (!ReachableEdges.count({PHI->getIncomingBlock(Operand), BB})) {
3766  DEBUG(dbgs() << "Replacing incoming value of " << PHI << " for block "
3767  << getBlockName(PHI->getIncomingBlock(Operand))
3768  << " with undef due to it being unreachable\n");
3769  Operand.set(UndefValue::get(PHI->getType()));
3770  }
3771  };
3772  // Replace unreachable phi arguments.
3773  // At this point, RevisitOnReachabilityChange only contains:
3774  //
3775  // 1. PHIs
3776  // 2. Temporaries that will convert to PHIs
3777  // 3. Operations that are affected by an unreachable edge but do not fit into
3778  // 1 or 2 (rare).
3779  // So it is a slight overshoot of what we want. We could make it exact by
3780  // using two SparseBitVectors per block.
3781  DenseMap<const BasicBlock *, unsigned> ReachablePredCount;
3782  for (auto &KV : ReachableEdges)
3783  ReachablePredCount[KV.getEnd()]++;
3784  for (auto &BBPair : RevisitOnReachabilityChange) {
3785  for (auto InstNum : BBPair.second) {
3786  auto *Inst = InstrFromDFSNum(InstNum);
3787  auto *PHI = dyn_cast<PHINode>(Inst);
3788  PHI = PHI ? PHI : dyn_cast_or_null<PHINode>(RealToTemp.lookup(Inst));
3789  if (!PHI)
3790  continue;
3791  auto *BB = BBPair.first;
3792  if (ReachablePredCount.lookup(BB) != PHI->getNumIncomingValues())
3793  ReplaceUnreachablePHIArgs(PHI, BB);
3794  }
3795  }
3796 
3797  // Map to store the use counts
3799  for (auto *CC : reverse(CongruenceClasses)) {
3800  DEBUG(dbgs() << "Eliminating in congruence class " << CC->getID() << "\n");
3801  // Track the equivalent store info so we can decide whether to try
3802  // dead store elimination.
3803  SmallVector<ValueDFS, 8> PossibleDeadStores;
3804  SmallPtrSet<Instruction *, 8> ProbablyDead;
3805  if (CC->isDead() || CC->empty())
3806  continue;
3807  // Everything still in the TOP class is unreachable or dead.
3808  if (CC == TOPClass) {
3809  for (auto M : *CC) {
3810  auto *VTE = ValueToExpression.lookup(M);
3811  if (VTE && isa<DeadExpression>(VTE))
3812  markInstructionForDeletion(cast<Instruction>(M));
3813  assert((!ReachableBlocks.count(cast<Instruction>(M)->getParent()) ||
3814  InstructionsToErase.count(cast<Instruction>(M))) &&
3815  "Everything in TOP should be unreachable or dead at this "
3816  "point");
3817  }
3818  continue;
3819  }
3820 
3821  assert(CC->getLeader() && "We should have had a leader");
3822  // If this is a leader that is always available, and it's a
3823  // constant or has no equivalences, just replace everything with
3824  // it. We then update the congruence class with whatever members
3825  // are left.
3826  Value *Leader =
3827  CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
3828  if (alwaysAvailable(Leader)) {
3829  CongruenceClass::MemberSet MembersLeft;
3830  for (auto M : *CC) {
3831  Value *Member = M;
3832  // Void things have no uses we can replace.
3833  if (Member == Leader || !isa<Instruction>(Member) ||
3834  Member->getType()->isVoidTy()) {
3835  MembersLeft.insert(Member);
3836  continue;
3837  }
3838  DEBUG(dbgs() << "Found replacement " << *(Leader) << " for " << *Member
3839  << "\n");
3840  auto *I = cast<Instruction>(Member);
3841  assert(Leader != I && "About to accidentally remove our leader");
3842  replaceInstruction(I, Leader);
3843  AnythingReplaced = true;
3844  }
3845  CC->swap(MembersLeft);
3846  } else {
3847  // If this is a singleton, we can skip it.
3848  if (CC->size() != 1 || RealToTemp.count(Leader)) {
3849  // This is a stack because equality replacement/etc may place
3850  // constants in the middle of the member list, and we want to use
3851  // those constant values in preference to the current leader, over
3852  // the scope of those constants.
3853  ValueDFSStack EliminationStack;
3854 
3855  // Convert the members to DFS ordered sets and then merge them.
3856  SmallVector<ValueDFS, 8> DFSOrderedSet;
3857  convertClassToDFSOrdered(*CC, DFSOrderedSet, UseCounts, ProbablyDead);
3858 
3859  // Sort the whole thing.
3860  std::sort(DFSOrderedSet.begin(), DFSOrderedSet.end());
3861  for (auto &VD : DFSOrderedSet) {
3862  int MemberDFSIn = VD.DFSIn;
3863  int MemberDFSOut = VD.DFSOut;
3864  Value *Def = VD.Def.getPointer();
3865  bool FromStore = VD.Def.getInt();
3866  Use *U = VD.U;
3867  // We ignore void things because we can't get a value from them.
3868  if (Def && Def->getType()->isVoidTy())
3869  continue;
3870  auto *DefInst = dyn_cast_or_null<Instruction>(Def);
3871  if (DefInst && AllTempInstructions.count(DefInst)) {
3872  auto *PN = cast<PHINode>(DefInst);
3873 
3874  // If this is a value phi and that's the expression we used, insert
3875  // it into the program
3876  // remove from temp instruction list.
3877  AllTempInstructions.erase(PN);
3878  auto *DefBlock = getBlockForValue(Def);
3879  DEBUG(dbgs() << "Inserting fully real phi of ops" << *Def
3880  << " into block "
3881  << getBlockName(getBlockForValue(Def)) << "\n");
3882  PN->insertBefore(&DefBlock->front());
3883  Def = PN;
3884  NumGVNPHIOfOpsEliminations++;
3885  }
3886 
3887  if (EliminationStack.empty()) {
3888  DEBUG(dbgs() << "Elimination Stack is empty\n");
3889  } else {
3890  DEBUG(dbgs() << "Elimination Stack Top DFS numbers are ("
3891  << EliminationStack.dfs_back().first << ","
3892  << EliminationStack.dfs_back().second << ")\n");
3893  }
3894 
3895  DEBUG(dbgs() << "Current DFS numbers are (" << MemberDFSIn << ","
3896  << MemberDFSOut << ")\n");
3897  // First, we see if we are out of scope or empty. If so,
3898  // and there equivalences, we try to replace the top of
3899  // stack with equivalences (if it's on the stack, it must
3900  // not have been eliminated yet).
3901  // Then we synchronize to our current scope, by
3902  // popping until we are back within a DFS scope that
3903  // dominates the current member.
3904  // Then, what happens depends on a few factors
3905  // If the stack is now empty, we need to push
3906  // If we have a constant or a local equivalence we want to
3907  // start using, we also push.
3908  // Otherwise, we walk along, processing members who are
3909  // dominated by this scope, and eliminate them.
3910  bool ShouldPush = Def && EliminationStack.empty();
3911  bool OutOfScope =
3912  !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut);
3913 
3914  if (OutOfScope || ShouldPush) {
3915  // Sync to our current scope.
3916  EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
3917  bool ShouldPush = Def && EliminationStack.empty();
3918  if (ShouldPush) {
3919  EliminationStack.push_back(Def, MemberDFSIn, MemberDFSOut);
3920  }
3921  }
3922 
3923  // Skip the Def's, we only want to eliminate on their uses. But mark
3924  // dominated defs as dead.
3925  if (Def) {
3926  // For anything in this case, what and how we value number
3927  // guarantees that any side-effets that would have occurred (ie
3928  // throwing, etc) can be proven to either still occur (because it's
3929  // dominated by something that has the same side-effects), or never
3930  // occur. Otherwise, we would not have been able to prove it value
3931  // equivalent to something else. For these things, we can just mark
3932  // it all dead. Note that this is different from the "ProbablyDead"
3933  // set, which may not be dominated by anything, and thus, are only
3934  // easy to prove dead if they are also side-effect free. Note that
3935  // because stores are put in terms of the stored value, we skip
3936  // stored values here. If the stored value is really dead, it will
3937  // still be marked for deletion when we process it in its own class.
3938  if (!EliminationStack.empty() && Def != EliminationStack.back() &&
3939  isa<Instruction>(Def) && !FromStore)
3940  markInstructionForDeletion(cast<Instruction>(Def));
3941  continue;
3942  }
3943  // At this point, we know it is a Use we are trying to possibly
3944  // replace.
3945 
3946  assert(isa<Instruction>(U->get()) &&
3947  "Current def should have been an instruction");
3948  assert(isa<Instruction>(U->getUser()) &&
3949  "Current user should have been an instruction");
3950 
3951  // If the thing we are replacing into is already marked to be dead,
3952  // this use is dead. Note that this is true regardless of whether
3953  // we have anything dominating the use or not. We do this here
3954  // because we are already walking all the uses anyway.
3955  Instruction *InstUse = cast<Instruction>(U->getUser());
3956  if (InstructionsToErase.count(InstUse)) {
3957  auto &UseCount = UseCounts[U->get()];
3958  if (--UseCount == 0) {
3959  ProbablyDead.insert(cast<Instruction>(U->get()));
3960  }
3961  }
3962 
3963  // If we get to this point, and the stack is empty we must have a use
3964  // with nothing we can use to eliminate this use, so just skip it.
3965  if (EliminationStack.empty())
3966  continue;
3967 
3968  Value *DominatingLeader = EliminationStack.back();
3969 
3970  auto *II = dyn_cast<IntrinsicInst>(DominatingLeader);
3971  if (II && II->getIntrinsicID() == Intrinsic::ssa_copy)
3972  DominatingLeader = II->getOperand(0);
3973 
3974  // Don't replace our existing users with ourselves.
3975  if (U->get() == DominatingLeader)
3976  continue;
3977  DEBUG(dbgs() << "Found replacement " << *DominatingLeader << " for "
3978  << *U->get() << " in " << *(U->getUser()) << "\n");
3979 
3980  // If we replaced something in an instruction, handle the patching of
3981  // metadata. Skip this if we are replacing predicateinfo with its
3982  // original operand, as we already know we can just drop it.
3983  auto *ReplacedInst = cast<Instruction>(U->get());
3984  auto *PI = PredInfo->getPredicateInfoFor(ReplacedInst);
3985  if (!PI || DominatingLeader != PI->OriginalOp)
3986  patchReplacementInstruction(ReplacedInst, DominatingLeader);
3987  U->set(DominatingLeader);
3988  // This is now a use of the dominating leader, which means if the
3989  // dominating leader was dead, it's now live!
3990  auto &LeaderUseCount = UseCounts[DominatingLeader];
3991  // It's about to be alive again.
3992  if (LeaderUseCount == 0 && isa<Instruction>(DominatingLeader))
3993  ProbablyDead.erase(cast<Instruction>(DominatingLeader));
3994  if (LeaderUseCount == 0 && II)
3995  ProbablyDead.insert(II);
3996  ++LeaderUseCount;
3997  AnythingReplaced = true;
3998  }
3999  }
4000  }
4001 
4002  // At this point, anything still in the ProbablyDead set is actually dead if
4003  // would be trivially dead.
4004  for (auto *I : ProbablyDead)
4006  markInstructionForDeletion(I);
4007 
4008  // Cleanup the congruence class.
4009  CongruenceClass::MemberSet MembersLeft;
4010  for (auto *Member : *CC)
4011  if (!isa<Instruction>(Member) ||
4012  !InstructionsToErase.count(cast<Instruction>(Member)))
4013  MembersLeft.insert(Member);
4014  CC->swap(MembersLeft);
4015 
4016  // If we have possible dead stores to look at, try to eliminate them.
4017  if (CC->getStoreCount() > 0) {
4018  convertClassToLoadsAndStores(*CC, PossibleDeadStores);
4019  std::sort(PossibleDeadStores.begin(), PossibleDeadStores.end());
4020  ValueDFSStack EliminationStack;
4021  for (auto &VD : PossibleDeadStores) {
4022  int MemberDFSIn = VD.DFSIn;
4023  int MemberDFSOut = VD.DFSOut;
4024  Instruction *Member = cast<Instruction>(VD.Def.getPointer());
4025  if (EliminationStack.empty() ||
4026  !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut)) {
4027  // Sync to our current scope.
4028  EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
4029  if (EliminationStack.empty()) {
4030  EliminationStack.push_back(Member, MemberDFSIn, MemberDFSOut);
4031  continue;
4032  }
4033  }
4034  // We already did load elimination, so nothing to do here.
4035  if (isa<LoadInst>(Member))
4036  continue;
4037  assert(!EliminationStack.empty());
4038  Instruction *Leader = cast<Instruction>(EliminationStack.back());
4039  (void)Leader;
4040  assert(DT->dominates(Leader->getParent(), Member->getParent()));
4041  // Member is dominater by Leader, and thus dead
4042  DEBUG(dbgs() << "Marking dead store " << *Member
4043  << " that is dominated by " << *Leader << "\n");
4044  markInstructionForDeletion(Member);
4045  CC->erase(Member);
4046  ++NumGVNDeadStores;
4047  }
4048  }
4049  }
4050  return AnythingReplaced;
4051 }
4052 
4053 // This function provides global ranking of operations so that we can place them
4054 // in a canonical order. Note that rank alone is not necessarily enough for a
4055 // complete ordering, as constants all have the same rank. However, generally,
4056 // we will simplify an operation with all constants so that it doesn't matter
4057 // what order they appear in.
4058 unsigned int NewGVN::getRank(const Value *V) const {
4059  // Prefer constants to undef to anything else
4060  // Undef is a constant, have to check it first.
4061  // Prefer smaller constants to constantexprs
4062  if (isa<ConstantExpr>(V))
4063  return 2;
4064  if (isa<UndefValue>(V))
4065  return 1;
4066  if (isa<Constant>(V))
4067  return 0;
4068  else if (auto *A = dyn_cast<Argument>(V))
4069  return 3 + A->getArgNo();
4070 
4071  // Need to shift the instruction DFS by number of arguments + 3 to account for
4072  // the constant and argument ranking above.
4073  unsigned Result = InstrToDFSNum(V);
4074  if (Result > 0)
4075  return 4 + NumFuncArgs + Result;
4076  // Unreachable or something else, just return a really large number.
4077  return ~0;
4078 }
4079 
4080 // This is a function that says whether two commutative operations should
4081 // have their order swapped when canonicalizing.
4082 bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {
4083  // Because we only care about a total ordering, and don't rewrite expressions
4084  // in this order, we order by rank, which will give a strict weak ordering to
4085  // everything but constants, and then we order by pointer address.
4086  return std::make_pair(getRank(A), A) > std::make_pair(getRank(B), B);
4087 }
4088 
4089 namespace {
4090 class NewGVNLegacyPass : public FunctionPass {
4091 public:
4092  static char ID; // Pass identification, replacement for typeid.
4093  NewGVNLegacyPass() : FunctionPass(ID) {
4095  }
4096  bool runOnFunction(Function &F) override;
4097 
4098 private:
4099  void getAnalysisUsage(AnalysisUsage &AU) const override {
4107  }
4108 };
4109 } // namespace
4110 
4111 bool NewGVNLegacyPass::runOnFunction(Function &F) {
4112  if (skipFunction(F))
4113  return false;
4114  return NewGVN(F, &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
4115  &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
4116  &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(),
4117  &getAnalysis<AAResultsWrapperPass>().getAAResults(),
4118  &getAnalysis<MemorySSAWrapperPass>().getMSSA(),
4119  F.getParent()->getDataLayout())
4120  .runGVN();
4121 }
4122 
4123 INITIALIZE_PASS_BEGIN(NewGVNLegacyPass, "newgvn", "Global Value Numbering",
4124  false, false)
4131 INITIALIZE_PASS_END(NewGVNLegacyPass, "newgvn", "Global Value Numbering", false,
4132  false)
4133 
4134 char NewGVNLegacyPass::ID = 0;
4135 
4136 // createGVNPass - The public interface to this file.
4137 FunctionPass *llvm::createNewGVNPass() { return new NewGVNLegacyPass(); }
4138 
4140  // Apparently the order in which we get these results matter for
4141  // the old GVN (see Chandler's comment in GVN.cpp). I'll keep
4142  // the same order here, just in case.
4143  auto &AC = AM.getResult<AssumptionAnalysis>(F);
4144  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
4145  auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
4146  auto &AA = AM.getResult<AAManager>(F);
4147  auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
4148  bool Changed =
4149  NewGVN(F, &DT, &AC, &TLI, &AA, &MSSA, F.getParent()->getDataLayout())
4150  .runGVN();
4151  if (!Changed)
4152  return PreservedAnalyses::all();
4153  PreservedAnalyses PA;
4155  PA.preserve<GlobalsAA>();
4156  return PA;
4157 }
Legacy wrapper pass to provide the GlobalsAAResult object.
Global Value Numbering
Definition: NewGVN.cpp:4131
void initializeNewGVNLegacyPassPass(PassRegistry &)
uint64_t CallInst * C
Value * getValueOperand()
Definition: Instructions.h:395
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:69
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:109
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:243
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:523
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:843
void setInt(IntType IntVal)
bool isSimple() const
Definition: Instructions.h:262
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
iterator_range< use_iterator > uses()
Definition: Value.h:350
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
int analyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr, LoadInst *DepLI, const DataLayout &DL)
This function determines whether a value for the pointer LoadPtr can be extracted from the load at De...
Definition: VNCoercion.cpp:219
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
unsigned getValueID() const
Return an ID for the concrete type of this object.
Definition: Value.h:459
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:234
void dropAllReferences()
Drop all references to operands.
Definition: User.h:279
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:687
static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS)
Definition: NewGVN.cpp:780
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
bool isAtomic() const
Return true if this instruction has an AtomicOrdering of unordered or higher.
static bool isEqual(const ExactEqualsExpression &LHS, const Expression *RHS)
Definition: NewGVN.cpp:393
This is the interface for a simple mod/ref and alias analysis over globals.
BasicBlock * getSuccessor(unsigned idx) const
Return the specified successor.
iterator begin() const
Definition: ArrayRef.h:137
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:136
static bool okayForPHIOfOps(const Instruction *I)
Definition: NewGVN.cpp:2502
Implements a dense probed hash-table based set.
Definition: DenseSet.h:221
This class represents a function call, abstracting a target machine&#39;s calling convention.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of .assume calls within a function.
Represents a read-write access to memory, whether it is a must-alias, or a may-alias.
Definition: MemorySSA.h:350
bool isTerminator() const
Definition: Instruction.h:128
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:869
Recycle small arrays allocated from a BumpPtrAllocator.
Definition: ArrayRecycler.h:29
void deleteValue()
Delete a pointer to a generic Value.
Definition: Value.cpp:93
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:816
BasicBlock * getSuccessor(unsigned i) const
STATISTIC(NumFunctions, "Total number of functions")
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:238
F(f)
reverse_iterator rend()
Definition: BasicBlock.h:259
An instruction for reading from memory.
Definition: Instructions.h:164
reverse_iterator rbegin()
Definition: BasicBlock.h:257
Hexagon Common GEP
Value * getCondition() const
Constant * getConstantMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset, Type *LoadTy, const DataLayout &DL)
Definition: VNCoercion.cpp:483
const Value * DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) const
Translate PHI node to its predecessor from the given basic block.
Definition: Value.cpp:689
ExactEqualsExpression(const Expression &E)
Definition: NewGVN.cpp:369
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:33
op_iterator op_begin()
Definition: User.h:214
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:207
static bool isImpliedFalseByMatchingCmp(Predicate Pred1, Predicate Pred2)
Determine if Pred1 implies Pred2 is false when two compares have matching operands.
hash_code getComputedHash() const
Definition: GVNExpression.h:92
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:51
Legacy analysis pass which computes MemorySSA.
Definition: MemorySSA.h:850
void setPointer(PointerTy PtrVal)
static cl::opt< bool > EnablePhiOfOps("enable-phi-of-ops", cl::init(true), cl::Hidden)
Currently, the generation "phi of ops" can result in correctness issues.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
Definition: InstrTypes.h:951
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:361
Value * SimplifyGEPInst(Type *SrcTy, ArrayRef< Value *> Ops, const SimplifyQuery &Q)
Given operands for a GetElementPtrInst, fold the result or return null.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:639
hash_code getComputedHash() const
Definition: NewGVN.cpp:370
int analyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr, MemIntrinsic *DepMI, const DataLayout &DL)
This function determines whether a value for the pointer LoadPtr can be extracted from the memory int...
Definition: VNCoercion.cpp:251
Constant * getConstantStoreValueForLoad(Constant *SrcVal, unsigned Offset, Type *LoadTy, const DataLayout &DL)
Definition: VNCoercion.cpp:351
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
Encapsulates MemorySSA, including all data associated with memory accesses.
Definition: MemorySSA.h:612
static bool isImpliedTrueByMatchingCmp(Predicate Pred1, Predicate Pred2)
Determine if Pred1 implies Pred2 is true when two compares have matching operands.
PointerIntPair< Value *, 1, bool > Def
Definition: NewGVN.cpp:3406
friend const_iterator begin(StringRef path, Style style)
Get begin iterator over path.
Definition: Path.cpp:234
This file provides an implementation of debug counters.
mapped_iterator< ItTy, FuncTy > map_iterator(const ItTy &I, FuncTy F)
Definition: STLExtras.h:221
static unsigned getHashValue(const ExactEqualsExpression &E)
Definition: NewGVN.cpp:390
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:248
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:201
Instruction * clone() const
Create a copy of &#39;this&#39; instruction that is identical in all ways except the following: ...
Key
PAL metadata keys.
Value * SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a CmpInst, fold the result or return null.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
static int getID(struct InternalInstruction *insn, const void *miiArg)
void andIRFlags(const Value *V)
Logical &#39;and&#39; of any supported wrapping, exact, and fast-math flags of V and this instruction...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
static bool isCounterSet(unsigned ID)
Definition: DebugCounter.h:106
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:125
An instruction for storing to memory.
Definition: Instructions.h:306
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:428
This is the generic walker interface for walkers of MemorySSA.
Definition: MemorySSA.h:881
FunctionPass * createNewGVNPass()
Definition: NewGVN.cpp:4137
const Expression & E
Definition: NewGVN.cpp:368
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:140
static const Expression * getEmptyKey()
Definition: NewGVN.cpp:377
Value * getOperand(unsigned i) const
Definition: User.h:154
int analyzeLoadFromClobberingStore(Type *LoadTy, Value *LoadPtr, StoreInst *DepSI, const DataLayout &DL)
This function determines whether a value for the pointer LoadPtr can be extracted from the store at D...
Definition: VNCoercion.cpp:202
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:141
const BasicBlock & getEntryBlock() const
Definition: Function.h:572
unsigned getDFSNumIn() const
getDFSNumIn/getDFSNumOut - These return the DFS visitation order for nodes in the dominator tree...
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:406
friend const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:243
Value * SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal, const SimplifyQuery &Q)
Given operands for a SelectInst, fold the result or return null.
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:596
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:54
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:153
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:291
LLVM Basic Block Representation.
Definition: BasicBlock.h:59
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:138
Conditional or Unconditional Branch instruction.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:149
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:42
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:116
Constant * getConstantLoadValueForLoad(Constant *SrcVal, unsigned Offset, Type *LoadTy, const DataLayout &DL)
Definition: VNCoercion.cpp:409
A manager for alias analyses.
static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl)
Definition: NewGVN.cpp:3604
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:371
static void setCounterValue(unsigned ID, const std::pair< int, int > &Val)
Definition: DebugCounter.h:119
Value * SimplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, const SimplifyQuery &Q)
Given operands for a CastInst, fold the result or return null.
Represent the analysis usage information of a pass.
op_iterator op_end()
Definition: User.h:216
iterator_range< def_chain_iterator< T, true > > optimized_def_chain(T MA)
Definition: MemorySSA.h:1195
bool any_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:823
Analysis pass providing a never-invalidated alias analysis result.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:853
bool isBinaryOp() const
Definition: Instruction.h:129
static const unsigned End
This file provides the interface for LLVM&#39;s Global Value Numbering pass.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:285
size_t arg_size() const
Definition: Function.h:630
op_range operands()
Definition: User.h:222
Value * getPointerOperand()
Definition: Instructions.h:270
unsigned getDFSNumOut() const
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:382
static bool shouldExecute(unsigned CounterName)
Definition: DebugCounter.h:72
static UndefValue * get(Type *T)
Static factory methods - Return an &#39;undef&#39; object of the specified type.
Definition: Constants.cpp:1320
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:159
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
static const Expression * getTombstoneKey()
Definition: NewGVN.cpp:382
This file implements the PredicateInfo analysis, which creates an Extended SSA form for operations us...
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static bool isCopyOfAPHI(const Value *V)
Definition: NewGVN.cpp:863
static std::pair< int, int > getCounterValue(unsigned ID)
Definition: DebugCounter.h:111
DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to &#39;dot...
A function analysis which provides an AssumptionCache.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Iterator for intrusive lists based on ilist_node.
unsigned getNumOperands() const
Definition: User.h:176
This is the shared class of boolean and integer constants.
Definition: Constants.h:84
void setOpcode(unsigned opcode)
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false...
Definition: SmallPtrSet.h:378
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:864
Provides information about what library functions are available for the current target.
An analysis that produces MemorySSA for a function.
Definition: MemorySSA.h:814
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:385
PreservedAnalyses run(Function &F, AnalysisManager< Function > &AM)
Run the pass over the function.
Definition: NewGVN.cpp:4139
BasicBlock * getBlock() const
Definition: MemorySSA.h:156
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
bool isConditional() const
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:516
bool isCommutative() const
Return true if the instruction is commutative:
Definition: Instruction.h:435
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:923
bool isTrueWhenEqual() const
This is just a convenience.
Definition: InstrTypes.h:1019
A range adaptor for a pair of iterators.
static bool alwaysAvailable(Value *V)
Definition: NewGVN.cpp:881
Class that has the common methods + fields of memory uses/defs.
Definition: MemorySSA.h:236
iterator_range< user_iterator > users()
Definition: Value.h:395
BasicBlock * getIncomingBlock(unsigned I) const
Return incoming basic block number i.
Definition: MemorySSA.h:493
iterator begin() const
Definition: ArrayRef.h:331
An opaque object representing a hash code.
Definition: Hashing.h:72
bool isMallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that allocates uninitialized memory (such ...
Instruction * getMemoryInst() const
Get the instruction that this MemoryUse represents.
Definition: MemorySSA.h:243
bool operator<(const ValueDFS &Other) const
Definition: NewGVN.cpp:3408
amdgpu Simplify well known AMD library false Value Value * Arg
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:927
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:120
iterator_range< typename GraphTraits< GraphType >::nodes_iterator > nodes(const GraphType &G)
Definition: GraphTraits.h:89
static Value * getCopyOf(const Value *V)
Definition: NewGVN.cpp:851
unsigned getAlignment() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:226
void emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:656
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:61
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
bool isCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that allocates zero-filled memory (such as...
#define I(x, y, z)
Definition: MD5.cpp:58
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
iterator find(const_arg_type_t< ValueT > V)
Definition: DenseSet.h:165
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:193
iterator_range< value_op_iterator > operand_values()
Definition: User.h:246
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
Constant * ConstantFoldInstOperands(Instruction *I, ArrayRef< Constant *> Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands...
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:174
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:91
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
Definition: STLExtras.h:348
static std::string getBlockName(const BasicBlock *B)
Definition: NewGVN.cpp:808
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:181
OutputIt transform(R &&Range, OutputIt d_first, UnaryPredicate P)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere...
Definition: STLExtras.h:894
Analysis pass providing the TargetLibraryInfo.
iterator_range< df_iterator< T > > depth_first(const T &G)
Value * SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a BinaryOperator, fold the result or return null.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
static bool isEqual(const Expression *LHS, const Expression *RHS)
Definition: NewGVN.cpp:399
static bool isCopyOfPHI(const Value *V, const PHINode *PN)
Definition: NewGVN.cpp:859
aarch64 promote const
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:856
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:545
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction has no side ef...
Definition: Local.cpp:324
LLVM Value Representation.
Definition: Value.h:73
Value * getOperand(unsigned N) const
The header file for the GVN pass that contains expression handling classes.
iterator end() const
Definition: ArrayRef.h:332
bool wouldInstructionBeTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used...
Definition: Local.cpp:331
#define DEBUG(X)
Definition: Debug.h:118
static cl::opt< bool > EnableStoreRefinement("enable-store-refinement", cl::init(false), cl::Hidden)
This file exposes an interface to building/using memory SSA to walk memory instructions using a use/d...
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:967
A container for analyses that lazily runs them and caches their results.
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:267
void sort(Policy policy, RandomAccessIterator Start, RandomAccessIterator End, const Comparator &Comp=Comparator())
Definition: Parallel.h:199
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
bool isSimple() const
Definition: Instructions.h:387
Represents phi nodes for memory accesses.
Definition: MemorySSA.h:431
const TerminatorInst * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:120
static unsigned getHashValue(const Expression *E)
Definition: NewGVN.cpp:387
virtual bool exactlyEquals(const Expression &Other) const
op_range incoming_values()
Value * getPointerOperand()
Definition: Instructions.h:398
DEBUG_COUNTER(VNCounter, "newgvn-vn", "Controls which instructions are value numbered")
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:174
void combineMetadata(Instruction *K, const Instruction *J, ArrayRef< unsigned > KnownIDs)
Combine the metadata of two instructions so that K can replace J.
Definition: Local.cpp:1741
INITIALIZE_PASS_BEGIN(NewGVNLegacyPass, "newgvn", "Global Value Numbering", false, false) INITIALIZE_PASS_END(NewGVNLegacyPass
static void patchReplacementInstruction(Instruction *I, Value *Repl)
Definition: NewGVN.cpp:3573
bool use_empty() const
Definition: Value.h:322
void allocateOperands(RecyclerType &Recycler, BumpPtrAllocator &Allocator)
iterator_range< arg_iterator > args()
Definition: Function.h:621
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
const BasicBlock * getParent() const
Definition: Instruction.h:66
newgvn
Definition: NewGVN.cpp:4131
bool operator==(const Expression &Other) const
Definition: NewGVN.cpp:371