File: | lib/Transforms/Scalar/NewGVN.cpp |
Warning: | line 1226, column 3 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- NewGVN.cpp - Global Value Numbering Pass ---------------------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | /// \file | |||
10 | /// This file implements the new LLVM's Global Value Numbering pass. | |||
11 | /// GVN partitions values computed by a function into congruence classes. | |||
12 | /// Values ending up in the same congruence class are guaranteed to be the same | |||
13 | /// for every execution of the program. In that respect, congruency is a | |||
14 | /// compile-time approximation of equivalence of values at runtime. | |||
15 | /// The algorithm implemented here uses a sparse formulation and it's based | |||
16 | /// on the ideas described in the paper: | |||
17 | /// "A Sparse Algorithm for Predicated Global Value Numbering" from | |||
18 | /// Karthik Gargi. | |||
19 | /// | |||
20 | /// A brief overview of the algorithm: The algorithm is essentially the same as | |||
21 | /// the standard RPO value numbering algorithm (a good reference is the paper | |||
22 | /// "SCC based value numbering" by L. Taylor Simpson) with one major difference: | |||
23 | /// The RPO algorithm proceeds, on every iteration, to process every reachable | |||
24 | /// block and every instruction in that block. This is because the standard RPO | |||
25 | /// algorithm does not track what things have the same value number, it only | |||
26 | /// tracks what the value number of a given operation is (the mapping is | |||
27 | /// operation -> value number). Thus, when a value number of an operation | |||
28 | /// changes, it must reprocess everything to ensure all uses of a value number | |||
29 | /// get updated properly. In constrast, the sparse algorithm we use *also* | |||
30 | /// tracks what operations have a given value number (IE it also tracks the | |||
31 | /// reverse mapping from value number -> operations with that value number), so | |||
32 | /// that it only needs to reprocess the instructions that are affected when | |||
33 | /// something's value number changes. The vast majority of complexity and code | |||
34 | /// in this file is devoted to tracking what value numbers could change for what | |||
35 | /// instructions when various things happen. The rest of the algorithm is | |||
36 | /// devoted to performing symbolic evaluation, forward propagation, and | |||
37 | /// simplification of operations based on the value numbers deduced so far | |||
38 | /// | |||
39 | /// In order to make the GVN mostly-complete, we use a technique derived from | |||
40 | /// "Detection of Redundant Expressions: A Complete and Polynomial-time | |||
41 | /// Algorithm in SSA" by R.R. Pai. The source of incompleteness in most SSA | |||
42 | /// based GVN algorithms is related to their inability to detect equivalence | |||
43 | /// between phi of ops (IE phi(a+b, c+d)) and op of phis (phi(a,c) + phi(b, d)). | |||
44 | /// We resolve this issue by generating the equivalent "phi of ops" form for | |||
45 | /// each op of phis we see, in a way that only takes polynomial time to resolve. | |||
46 | /// | |||
47 | /// We also do not perform elimination by using any published algorithm. All | |||
48 | /// published algorithms are O(Instructions). Instead, we use a technique that | |||
49 | /// is O(number of operations with the same value number), enabling us to skip | |||
50 | /// trying to eliminate things that have unique value numbers. | |||
51 | // | |||
52 | //===----------------------------------------------------------------------===// | |||
53 | ||||
54 | #include "llvm/Transforms/Scalar/NewGVN.h" | |||
55 | #include "llvm/ADT/ArrayRef.h" | |||
56 | #include "llvm/ADT/BitVector.h" | |||
57 | #include "llvm/ADT/DenseMap.h" | |||
58 | #include "llvm/ADT/DenseMapInfo.h" | |||
59 | #include "llvm/ADT/DenseSet.h" | |||
60 | #include "llvm/ADT/DepthFirstIterator.h" | |||
61 | #include "llvm/ADT/GraphTraits.h" | |||
62 | #include "llvm/ADT/Hashing.h" | |||
63 | #include "llvm/ADT/PointerIntPair.h" | |||
64 | #include "llvm/ADT/PostOrderIterator.h" | |||
65 | #include "llvm/ADT/SmallPtrSet.h" | |||
66 | #include "llvm/ADT/SmallVector.h" | |||
67 | #include "llvm/ADT/SparseBitVector.h" | |||
68 | #include "llvm/ADT/Statistic.h" | |||
69 | #include "llvm/ADT/iterator_range.h" | |||
70 | #include "llvm/Analysis/AliasAnalysis.h" | |||
71 | #include "llvm/Analysis/AssumptionCache.h" | |||
72 | #include "llvm/Analysis/CFGPrinter.h" | |||
73 | #include "llvm/Analysis/ConstantFolding.h" | |||
74 | #include "llvm/Analysis/GlobalsModRef.h" | |||
75 | #include "llvm/Analysis/InstructionSimplify.h" | |||
76 | #include "llvm/Analysis/MemoryBuiltins.h" | |||
77 | #include "llvm/Analysis/MemorySSA.h" | |||
78 | #include "llvm/Analysis/TargetLibraryInfo.h" | |||
79 | #include "llvm/Transforms/Utils/Local.h" | |||
80 | #include "llvm/IR/Argument.h" | |||
81 | #include "llvm/IR/BasicBlock.h" | |||
82 | #include "llvm/IR/Constant.h" | |||
83 | #include "llvm/IR/Constants.h" | |||
84 | #include "llvm/IR/Dominators.h" | |||
85 | #include "llvm/IR/Function.h" | |||
86 | #include "llvm/IR/InstrTypes.h" | |||
87 | #include "llvm/IR/Instruction.h" | |||
88 | #include "llvm/IR/Instructions.h" | |||
89 | #include "llvm/IR/IntrinsicInst.h" | |||
90 | #include "llvm/IR/Intrinsics.h" | |||
91 | #include "llvm/IR/LLVMContext.h" | |||
92 | #include "llvm/IR/Type.h" | |||
93 | #include "llvm/IR/Use.h" | |||
94 | #include "llvm/IR/User.h" | |||
95 | #include "llvm/IR/Value.h" | |||
96 | #include "llvm/Pass.h" | |||
97 | #include "llvm/Support/Allocator.h" | |||
98 | #include "llvm/Support/ArrayRecycler.h" | |||
99 | #include "llvm/Support/Casting.h" | |||
100 | #include "llvm/Support/CommandLine.h" | |||
101 | #include "llvm/Support/Debug.h" | |||
102 | #include "llvm/Support/DebugCounter.h" | |||
103 | #include "llvm/Support/ErrorHandling.h" | |||
104 | #include "llvm/Support/PointerLikeTypeTraits.h" | |||
105 | #include "llvm/Support/raw_ostream.h" | |||
106 | #include "llvm/Transforms/Scalar.h" | |||
107 | #include "llvm/Transforms/Scalar/GVNExpression.h" | |||
108 | #include "llvm/Transforms/Utils/PredicateInfo.h" | |||
109 | #include "llvm/Transforms/Utils/VNCoercion.h" | |||
110 | #include <algorithm> | |||
111 | #include <cassert> | |||
112 | #include <cstdint> | |||
113 | #include <iterator> | |||
114 | #include <map> | |||
115 | #include <memory> | |||
116 | #include <set> | |||
117 | #include <string> | |||
118 | #include <tuple> | |||
119 | #include <utility> | |||
120 | #include <vector> | |||
121 | ||||
122 | using namespace llvm; | |||
123 | using namespace llvm::GVNExpression; | |||
124 | using namespace llvm::VNCoercion; | |||
125 | ||||
126 | #define DEBUG_TYPE"newgvn" "newgvn" | |||
127 | ||||
128 | STATISTIC(NumGVNInstrDeleted, "Number of instructions deleted")static llvm::Statistic NumGVNInstrDeleted = {"newgvn", "NumGVNInstrDeleted" , "Number of instructions deleted", {0}, {false}}; | |||
129 | STATISTIC(NumGVNBlocksDeleted, "Number of blocks deleted")static llvm::Statistic NumGVNBlocksDeleted = {"newgvn", "NumGVNBlocksDeleted" , "Number of blocks deleted", {0}, {false}}; | |||
130 | STATISTIC(NumGVNOpsSimplified, "Number of Expressions simplified")static llvm::Statistic NumGVNOpsSimplified = {"newgvn", "NumGVNOpsSimplified" , "Number of Expressions simplified", {0}, {false}}; | |||
131 | STATISTIC(NumGVNPhisAllSame, "Number of PHIs whos arguments are all the same")static llvm::Statistic NumGVNPhisAllSame = {"newgvn", "NumGVNPhisAllSame" , "Number of PHIs whos arguments are all the same", {0}, {false }}; | |||
132 | STATISTIC(NumGVNMaxIterations,static llvm::Statistic NumGVNMaxIterations = {"newgvn", "NumGVNMaxIterations" , "Maximum Number of iterations it took to converge GVN", {0} , {false}} | |||
133 | "Maximum Number of iterations it took to converge GVN")static llvm::Statistic NumGVNMaxIterations = {"newgvn", "NumGVNMaxIterations" , "Maximum Number of iterations it took to converge GVN", {0} , {false}}; | |||
134 | STATISTIC(NumGVNLeaderChanges, "Number of leader changes")static llvm::Statistic NumGVNLeaderChanges = {"newgvn", "NumGVNLeaderChanges" , "Number of leader changes", {0}, {false}}; | |||
135 | STATISTIC(NumGVNSortedLeaderChanges, "Number of sorted leader changes")static llvm::Statistic NumGVNSortedLeaderChanges = {"newgvn", "NumGVNSortedLeaderChanges", "Number of sorted leader changes" , {0}, {false}}; | |||
136 | STATISTIC(NumGVNAvoidedSortedLeaderChanges,static llvm::Statistic NumGVNAvoidedSortedLeaderChanges = {"newgvn" , "NumGVNAvoidedSortedLeaderChanges", "Number of avoided sorted leader changes" , {0}, {false}} | |||
137 | "Number of avoided sorted leader changes")static llvm::Statistic NumGVNAvoidedSortedLeaderChanges = {"newgvn" , "NumGVNAvoidedSortedLeaderChanges", "Number of avoided sorted leader changes" , {0}, {false}}; | |||
138 | STATISTIC(NumGVNDeadStores, "Number of redundant/dead stores eliminated")static llvm::Statistic NumGVNDeadStores = {"newgvn", "NumGVNDeadStores" , "Number of redundant/dead stores eliminated", {0}, {false}}; | |||
139 | STATISTIC(NumGVNPHIOfOpsCreated, "Number of PHI of ops created")static llvm::Statistic NumGVNPHIOfOpsCreated = {"newgvn", "NumGVNPHIOfOpsCreated" , "Number of PHI of ops created", {0}, {false}}; | |||
140 | STATISTIC(NumGVNPHIOfOpsEliminations,static llvm::Statistic NumGVNPHIOfOpsEliminations = {"newgvn" , "NumGVNPHIOfOpsEliminations", "Number of things eliminated using PHI of ops" , {0}, {false}} | |||
141 | "Number of things eliminated using PHI of ops")static llvm::Statistic NumGVNPHIOfOpsEliminations = {"newgvn" , "NumGVNPHIOfOpsEliminations", "Number of things eliminated using PHI of ops" , {0}, {false}}; | |||
142 | DEBUG_COUNTER(VNCounter, "newgvn-vn",static const unsigned VNCounter = DebugCounter::registerCounter ("newgvn-vn", "Controls which instructions are value numbered" ) | |||
143 | "Controls which instructions are value numbered")static const unsigned VNCounter = DebugCounter::registerCounter ("newgvn-vn", "Controls which instructions are value numbered" ); | |||
144 | DEBUG_COUNTER(PHIOfOpsCounter, "newgvn-phi",static const unsigned PHIOfOpsCounter = DebugCounter::registerCounter ("newgvn-phi", "Controls which instructions we create phi of ops for" ) | |||
145 | "Controls which instructions we create phi of ops for")static const unsigned PHIOfOpsCounter = DebugCounter::registerCounter ("newgvn-phi", "Controls which instructions we create phi of ops for" ); | |||
146 | // Currently store defining access refinement is too slow due to basicaa being | |||
147 | // egregiously slow. This flag lets us keep it working while we work on this | |||
148 | // issue. | |||
149 | static cl::opt<bool> EnableStoreRefinement("enable-store-refinement", | |||
150 | cl::init(false), cl::Hidden); | |||
151 | ||||
152 | /// Currently, the generation "phi of ops" can result in correctness issues. | |||
153 | static cl::opt<bool> EnablePhiOfOps("enable-phi-of-ops", cl::init(true), | |||
154 | cl::Hidden); | |||
155 | ||||
156 | //===----------------------------------------------------------------------===// | |||
157 | // GVN Pass | |||
158 | //===----------------------------------------------------------------------===// | |||
159 | ||||
160 | // Anchor methods. | |||
161 | namespace llvm { | |||
162 | namespace GVNExpression { | |||
163 | ||||
164 | Expression::~Expression() = default; | |||
165 | BasicExpression::~BasicExpression() = default; | |||
166 | CallExpression::~CallExpression() = default; | |||
167 | LoadExpression::~LoadExpression() = default; | |||
168 | StoreExpression::~StoreExpression() = default; | |||
169 | AggregateValueExpression::~AggregateValueExpression() = default; | |||
170 | PHIExpression::~PHIExpression() = default; | |||
171 | ||||
172 | } // end namespace GVNExpression | |||
173 | } // end namespace llvm | |||
174 | ||||
175 | namespace { | |||
176 | ||||
177 | // Tarjan's SCC finding algorithm with Nuutila's improvements | |||
178 | // SCCIterator is actually fairly complex for the simple thing we want. | |||
179 | // It also wants to hand us SCC's that are unrelated to the phi node we ask | |||
180 | // about, and have us process them there or risk redoing work. | |||
181 | // Graph traits over a filter iterator also doesn't work that well here. | |||
182 | // This SCC finder is specialized to walk use-def chains, and only follows | |||
183 | // instructions, | |||
184 | // not generic values (arguments, etc). | |||
185 | struct TarjanSCC { | |||
186 | TarjanSCC() : Components(1) {} | |||
187 | ||||
188 | void Start(const Instruction *Start) { | |||
189 | if (Root.lookup(Start) == 0) | |||
190 | FindSCC(Start); | |||
191 | } | |||
192 | ||||
193 | const SmallPtrSetImpl<const Value *> &getComponentFor(const Value *V) const { | |||
194 | unsigned ComponentID = ValueToComponent.lookup(V); | |||
195 | ||||
196 | assert(ComponentID > 0 &&((ComponentID > 0 && "Asking for a component for a value we never processed" ) ? static_cast<void> (0) : __assert_fail ("ComponentID > 0 && \"Asking for a component for a value we never processed\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 197, __PRETTY_FUNCTION__)) | |||
197 | "Asking for a component for a value we never processed")((ComponentID > 0 && "Asking for a component for a value we never processed" ) ? static_cast<void> (0) : __assert_fail ("ComponentID > 0 && \"Asking for a component for a value we never processed\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 197, __PRETTY_FUNCTION__)); | |||
198 | return Components[ComponentID]; | |||
199 | } | |||
200 | ||||
201 | private: | |||
202 | void FindSCC(const Instruction *I) { | |||
203 | Root[I] = ++DFSNum; | |||
204 | // Store the DFS Number we had before it possibly gets incremented. | |||
205 | unsigned int OurDFS = DFSNum; | |||
206 | for (auto &Op : I->operands()) { | |||
207 | if (auto *InstOp = dyn_cast<Instruction>(Op)) { | |||
208 | if (Root.lookup(Op) == 0) | |||
209 | FindSCC(InstOp); | |||
210 | if (!InComponent.count(Op)) | |||
211 | Root[I] = std::min(Root.lookup(I), Root.lookup(Op)); | |||
212 | } | |||
213 | } | |||
214 | // See if we really were the root of a component, by seeing if we still have | |||
215 | // our DFSNumber. If we do, we are the root of the component, and we have | |||
216 | // completed a component. If we do not, we are not the root of a component, | |||
217 | // and belong on the component stack. | |||
218 | if (Root.lookup(I) == OurDFS) { | |||
219 | unsigned ComponentID = Components.size(); | |||
220 | Components.resize(Components.size() + 1); | |||
221 | auto &Component = Components.back(); | |||
222 | Component.insert(I); | |||
223 | LLVM_DEBUG(dbgs() << "Component root is " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Component root is " << * I << "\n"; } } while (false); | |||
224 | InComponent.insert(I); | |||
225 | ValueToComponent[I] = ComponentID; | |||
226 | // Pop a component off the stack and label it. | |||
227 | while (!Stack.empty() && Root.lookup(Stack.back()) >= OurDFS) { | |||
228 | auto *Member = Stack.back(); | |||
229 | LLVM_DEBUG(dbgs() << "Component member is " << *Member << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Component member is " << *Member << "\n"; } } while (false); | |||
230 | Component.insert(Member); | |||
231 | InComponent.insert(Member); | |||
232 | ValueToComponent[Member] = ComponentID; | |||
233 | Stack.pop_back(); | |||
234 | } | |||
235 | } else { | |||
236 | // Part of a component, push to stack | |||
237 | Stack.push_back(I); | |||
238 | } | |||
239 | } | |||
240 | ||||
241 | unsigned int DFSNum = 1; | |||
242 | SmallPtrSet<const Value *, 8> InComponent; | |||
243 | DenseMap<const Value *, unsigned int> Root; | |||
244 | SmallVector<const Value *, 8> Stack; | |||
245 | ||||
246 | // Store the components as vector of ptr sets, because we need the topo order | |||
247 | // of SCC's, but not individual member order | |||
248 | SmallVector<SmallPtrSet<const Value *, 8>, 8> Components; | |||
249 | ||||
250 | DenseMap<const Value *, unsigned> ValueToComponent; | |||
251 | }; | |||
252 | ||||
253 | // Congruence classes represent the set of expressions/instructions | |||
254 | // that are all the same *during some scope in the function*. | |||
255 | // That is, because of the way we perform equality propagation, and | |||
256 | // because of memory value numbering, it is not correct to assume | |||
257 | // you can willy-nilly replace any member with any other at any | |||
258 | // point in the function. | |||
259 | // | |||
260 | // For any Value in the Member set, it is valid to replace any dominated member | |||
261 | // with that Value. | |||
262 | // | |||
263 | // Every congruence class has a leader, and the leader is used to symbolize | |||
264 | // instructions in a canonical way (IE every operand of an instruction that is a | |||
265 | // member of the same congruence class will always be replaced with leader | |||
266 | // during symbolization). To simplify symbolization, we keep the leader as a | |||
267 | // constant if class can be proved to be a constant value. Otherwise, the | |||
268 | // leader is the member of the value set with the smallest DFS number. Each | |||
269 | // congruence class also has a defining expression, though the expression may be | |||
270 | // null. If it exists, it can be used for forward propagation and reassociation | |||
271 | // of values. | |||
272 | ||||
273 | // For memory, we also track a representative MemoryAccess, and a set of memory | |||
274 | // members for MemoryPhis (which have no real instructions). Note that for | |||
275 | // memory, it seems tempting to try to split the memory members into a | |||
276 | // MemoryCongruenceClass or something. Unfortunately, this does not work | |||
277 | // easily. The value numbering of a given memory expression depends on the | |||
278 | // leader of the memory congruence class, and the leader of memory congruence | |||
279 | // class depends on the value numbering of a given memory expression. This | |||
280 | // leads to wasted propagation, and in some cases, missed optimization. For | |||
281 | // example: If we had value numbered two stores together before, but now do not, | |||
282 | // we move them to a new value congruence class. This in turn will move at one | |||
283 | // of the memorydefs to a new memory congruence class. Which in turn, affects | |||
284 | // the value numbering of the stores we just value numbered (because the memory | |||
285 | // congruence class is part of the value number). So while theoretically | |||
286 | // possible to split them up, it turns out to be *incredibly* complicated to get | |||
287 | // it to work right, because of the interdependency. While structurally | |||
288 | // slightly messier, it is algorithmically much simpler and faster to do what we | |||
289 | // do here, and track them both at once in the same class. | |||
290 | // Note: The default iterators for this class iterate over values | |||
291 | class CongruenceClass { | |||
292 | public: | |||
293 | using MemberType = Value; | |||
294 | using MemberSet = SmallPtrSet<MemberType *, 4>; | |||
295 | using MemoryMemberType = MemoryPhi; | |||
296 | using MemoryMemberSet = SmallPtrSet<const MemoryMemberType *, 2>; | |||
297 | ||||
298 | explicit CongruenceClass(unsigned ID) : ID(ID) {} | |||
299 | CongruenceClass(unsigned ID, Value *Leader, const Expression *E) | |||
300 | : ID(ID), RepLeader(Leader), DefiningExpr(E) {} | |||
301 | ||||
302 | unsigned getID() const { return ID; } | |||
303 | ||||
304 | // True if this class has no members left. This is mainly used for assertion | |||
305 | // purposes, and for skipping empty classes. | |||
306 | bool isDead() const { | |||
307 | // If it's both dead from a value perspective, and dead from a memory | |||
308 | // perspective, it's really dead. | |||
309 | return empty() && memory_empty(); | |||
310 | } | |||
311 | ||||
312 | // Leader functions | |||
313 | Value *getLeader() const { return RepLeader; } | |||
314 | void setLeader(Value *Leader) { RepLeader = Leader; } | |||
315 | const std::pair<Value *, unsigned int> &getNextLeader() const { | |||
316 | return NextLeader; | |||
317 | } | |||
318 | void resetNextLeader() { NextLeader = {nullptr, ~0}; } | |||
319 | void addPossibleNextLeader(std::pair<Value *, unsigned int> LeaderPair) { | |||
320 | if (LeaderPair.second < NextLeader.second) | |||
321 | NextLeader = LeaderPair; | |||
322 | } | |||
323 | ||||
324 | Value *getStoredValue() const { return RepStoredValue; } | |||
325 | void setStoredValue(Value *Leader) { RepStoredValue = Leader; } | |||
326 | const MemoryAccess *getMemoryLeader() const { return RepMemoryAccess; } | |||
327 | void setMemoryLeader(const MemoryAccess *Leader) { RepMemoryAccess = Leader; } | |||
328 | ||||
329 | // Forward propagation info | |||
330 | const Expression *getDefiningExpr() const { return DefiningExpr; } | |||
331 | ||||
332 | // Value member set | |||
333 | bool empty() const { return Members.empty(); } | |||
334 | unsigned size() const { return Members.size(); } | |||
335 | MemberSet::const_iterator begin() const { return Members.begin(); } | |||
336 | MemberSet::const_iterator end() const { return Members.end(); } | |||
337 | void insert(MemberType *M) { Members.insert(M); } | |||
338 | void erase(MemberType *M) { Members.erase(M); } | |||
339 | void swap(MemberSet &Other) { Members.swap(Other); } | |||
340 | ||||
341 | // Memory member set | |||
342 | bool memory_empty() const { return MemoryMembers.empty(); } | |||
343 | unsigned memory_size() const { return MemoryMembers.size(); } | |||
344 | MemoryMemberSet::const_iterator memory_begin() const { | |||
345 | return MemoryMembers.begin(); | |||
346 | } | |||
347 | MemoryMemberSet::const_iterator memory_end() const { | |||
348 | return MemoryMembers.end(); | |||
349 | } | |||
350 | iterator_range<MemoryMemberSet::const_iterator> memory() const { | |||
351 | return make_range(memory_begin(), memory_end()); | |||
352 | } | |||
353 | ||||
354 | void memory_insert(const MemoryMemberType *M) { MemoryMembers.insert(M); } | |||
355 | void memory_erase(const MemoryMemberType *M) { MemoryMembers.erase(M); } | |||
356 | ||||
357 | // Store count | |||
358 | unsigned getStoreCount() const { return StoreCount; } | |||
359 | void incStoreCount() { ++StoreCount; } | |||
360 | void decStoreCount() { | |||
361 | assert(StoreCount != 0 && "Store count went negative")((StoreCount != 0 && "Store count went negative") ? static_cast <void> (0) : __assert_fail ("StoreCount != 0 && \"Store count went negative\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 361, __PRETTY_FUNCTION__)); | |||
362 | --StoreCount; | |||
363 | } | |||
364 | ||||
365 | // True if this class has no memory members. | |||
366 | bool definesNoMemory() const { return StoreCount == 0 && memory_empty(); } | |||
367 | ||||
368 | // Return true if two congruence classes are equivalent to each other. This | |||
369 | // means that every field but the ID number and the dead field are equivalent. | |||
370 | bool isEquivalentTo(const CongruenceClass *Other) const { | |||
371 | if (!Other) | |||
372 | return false; | |||
373 | if (this == Other) | |||
374 | return true; | |||
375 | ||||
376 | if (std::tie(StoreCount, RepLeader, RepStoredValue, RepMemoryAccess) != | |||
377 | std::tie(Other->StoreCount, Other->RepLeader, Other->RepStoredValue, | |||
378 | Other->RepMemoryAccess)) | |||
379 | return false; | |||
380 | if (DefiningExpr != Other->DefiningExpr) | |||
381 | if (!DefiningExpr || !Other->DefiningExpr || | |||
382 | *DefiningExpr != *Other->DefiningExpr) | |||
383 | return false; | |||
384 | ||||
385 | if (Members.size() != Other->Members.size()) | |||
386 | return false; | |||
387 | ||||
388 | return all_of(Members, | |||
389 | [&](const Value *V) { return Other->Members.count(V); }); | |||
390 | } | |||
391 | ||||
392 | private: | |||
393 | unsigned ID; | |||
394 | ||||
395 | // Representative leader. | |||
396 | Value *RepLeader = nullptr; | |||
397 | ||||
398 | // The most dominating leader after our current leader, because the member set | |||
399 | // is not sorted and is expensive to keep sorted all the time. | |||
400 | std::pair<Value *, unsigned int> NextLeader = {nullptr, ~0U}; | |||
401 | ||||
402 | // If this is represented by a store, the value of the store. | |||
403 | Value *RepStoredValue = nullptr; | |||
404 | ||||
405 | // If this class contains MemoryDefs or MemoryPhis, this is the leading memory | |||
406 | // access. | |||
407 | const MemoryAccess *RepMemoryAccess = nullptr; | |||
408 | ||||
409 | // Defining Expression. | |||
410 | const Expression *DefiningExpr = nullptr; | |||
411 | ||||
412 | // Actual members of this class. | |||
413 | MemberSet Members; | |||
414 | ||||
415 | // This is the set of MemoryPhis that exist in the class. MemoryDefs and | |||
416 | // MemoryUses have real instructions representing them, so we only need to | |||
417 | // track MemoryPhis here. | |||
418 | MemoryMemberSet MemoryMembers; | |||
419 | ||||
420 | // Number of stores in this congruence class. | |||
421 | // This is used so we can detect store equivalence changes properly. | |||
422 | int StoreCount = 0; | |||
423 | }; | |||
424 | ||||
425 | } // end anonymous namespace | |||
426 | ||||
427 | namespace llvm { | |||
428 | ||||
429 | struct ExactEqualsExpression { | |||
430 | const Expression &E; | |||
431 | ||||
432 | explicit ExactEqualsExpression(const Expression &E) : E(E) {} | |||
433 | ||||
434 | hash_code getComputedHash() const { return E.getComputedHash(); } | |||
435 | ||||
436 | bool operator==(const Expression &Other) const { | |||
437 | return E.exactlyEquals(Other); | |||
438 | } | |||
439 | }; | |||
440 | ||||
441 | template <> struct DenseMapInfo<const Expression *> { | |||
442 | static const Expression *getEmptyKey() { | |||
443 | auto Val = static_cast<uintptr_t>(-1); | |||
444 | Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable; | |||
445 | return reinterpret_cast<const Expression *>(Val); | |||
446 | } | |||
447 | ||||
448 | static const Expression *getTombstoneKey() { | |||
449 | auto Val = static_cast<uintptr_t>(~1U); | |||
450 | Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable; | |||
451 | return reinterpret_cast<const Expression *>(Val); | |||
452 | } | |||
453 | ||||
454 | static unsigned getHashValue(const Expression *E) { | |||
455 | return E->getComputedHash(); | |||
456 | } | |||
457 | ||||
458 | static unsigned getHashValue(const ExactEqualsExpression &E) { | |||
459 | return E.getComputedHash(); | |||
460 | } | |||
461 | ||||
462 | static bool isEqual(const ExactEqualsExpression &LHS, const Expression *RHS) { | |||
463 | if (RHS == getTombstoneKey() || RHS == getEmptyKey()) | |||
464 | return false; | |||
465 | return LHS == *RHS; | |||
466 | } | |||
467 | ||||
468 | static bool isEqual(const Expression *LHS, const Expression *RHS) { | |||
469 | if (LHS == RHS) | |||
470 | return true; | |||
471 | if (LHS == getTombstoneKey() || RHS == getTombstoneKey() || | |||
472 | LHS == getEmptyKey() || RHS == getEmptyKey()) | |||
473 | return false; | |||
474 | // Compare hashes before equality. This is *not* what the hashtable does, | |||
475 | // since it is computing it modulo the number of buckets, whereas we are | |||
476 | // using the full hash keyspace. Since the hashes are precomputed, this | |||
477 | // check is *much* faster than equality. | |||
478 | if (LHS->getComputedHash() != RHS->getComputedHash()) | |||
479 | return false; | |||
480 | return *LHS == *RHS; | |||
481 | } | |||
482 | }; | |||
483 | ||||
484 | } // end namespace llvm | |||
485 | ||||
486 | namespace { | |||
487 | ||||
488 | class NewGVN { | |||
489 | Function &F; | |||
490 | DominatorTree *DT; | |||
491 | const TargetLibraryInfo *TLI; | |||
492 | AliasAnalysis *AA; | |||
493 | MemorySSA *MSSA; | |||
494 | MemorySSAWalker *MSSAWalker; | |||
495 | const DataLayout &DL; | |||
496 | std::unique_ptr<PredicateInfo> PredInfo; | |||
497 | ||||
498 | // These are the only two things the create* functions should have | |||
499 | // side-effects on due to allocating memory. | |||
500 | mutable BumpPtrAllocator ExpressionAllocator; | |||
501 | mutable ArrayRecycler<Value *> ArgRecycler; | |||
502 | mutable TarjanSCC SCCFinder; | |||
503 | const SimplifyQuery SQ; | |||
504 | ||||
505 | // Number of function arguments, used by ranking | |||
506 | unsigned int NumFuncArgs; | |||
507 | ||||
508 | // RPOOrdering of basic blocks | |||
509 | DenseMap<const DomTreeNode *, unsigned> RPOOrdering; | |||
510 | ||||
511 | // Congruence class info. | |||
512 | ||||
513 | // This class is called INITIAL in the paper. It is the class everything | |||
514 | // startsout in, and represents any value. Being an optimistic analysis, | |||
515 | // anything in the TOP class has the value TOP, which is indeterminate and | |||
516 | // equivalent to everything. | |||
517 | CongruenceClass *TOPClass; | |||
518 | std::vector<CongruenceClass *> CongruenceClasses; | |||
519 | unsigned NextCongruenceNum; | |||
520 | ||||
521 | // Value Mappings. | |||
522 | DenseMap<Value *, CongruenceClass *> ValueToClass; | |||
523 | DenseMap<Value *, const Expression *> ValueToExpression; | |||
524 | ||||
525 | // Value PHI handling, used to make equivalence between phi(op, op) and | |||
526 | // op(phi, phi). | |||
527 | // These mappings just store various data that would normally be part of the | |||
528 | // IR. | |||
529 | SmallPtrSet<const Instruction *, 8> PHINodeUses; | |||
530 | ||||
531 | DenseMap<const Value *, bool> OpSafeForPHIOfOps; | |||
532 | ||||
533 | // Map a temporary instruction we created to a parent block. | |||
534 | DenseMap<const Value *, BasicBlock *> TempToBlock; | |||
535 | ||||
536 | // Map between the already in-program instructions and the temporary phis we | |||
537 | // created that they are known equivalent to. | |||
538 | DenseMap<const Value *, PHINode *> RealToTemp; | |||
539 | ||||
540 | // In order to know when we should re-process instructions that have | |||
541 | // phi-of-ops, we track the set of expressions that they needed as | |||
542 | // leaders. When we discover new leaders for those expressions, we process the | |||
543 | // associated phi-of-op instructions again in case they have changed. The | |||
544 | // other way they may change is if they had leaders, and those leaders | |||
545 | // disappear. However, at the point they have leaders, there are uses of the | |||
546 | // relevant operands in the created phi node, and so they will get reprocessed | |||
547 | // through the normal user marking we perform. | |||
548 | mutable DenseMap<const Value *, SmallPtrSet<Value *, 2>> AdditionalUsers; | |||
549 | DenseMap<const Expression *, SmallPtrSet<Instruction *, 2>> | |||
550 | ExpressionToPhiOfOps; | |||
551 | ||||
552 | // Map from temporary operation to MemoryAccess. | |||
553 | DenseMap<const Instruction *, MemoryUseOrDef *> TempToMemory; | |||
554 | ||||
555 | // Set of all temporary instructions we created. | |||
556 | // Note: This will include instructions that were just created during value | |||
557 | // numbering. The way to test if something is using them is to check | |||
558 | // RealToTemp. | |||
559 | DenseSet<Instruction *> AllTempInstructions; | |||
560 | ||||
561 | // This is the set of instructions to revisit on a reachability change. At | |||
562 | // the end of the main iteration loop it will contain at least all the phi of | |||
563 | // ops instructions that will be changed to phis, as well as regular phis. | |||
564 | // During the iteration loop, it may contain other things, such as phi of ops | |||
565 | // instructions that used edge reachability to reach a result, and so need to | |||
566 | // be revisited when the edge changes, independent of whether the phi they | |||
567 | // depended on changes. | |||
568 | DenseMap<BasicBlock *, SparseBitVector<>> RevisitOnReachabilityChange; | |||
569 | ||||
570 | // Mapping from predicate info we used to the instructions we used it with. | |||
571 | // In order to correctly ensure propagation, we must keep track of what | |||
572 | // comparisons we used, so that when the values of the comparisons change, we | |||
573 | // propagate the information to the places we used the comparison. | |||
574 | mutable DenseMap<const Value *, SmallPtrSet<Instruction *, 2>> | |||
575 | PredicateToUsers; | |||
576 | ||||
577 | // the same reasoning as PredicateToUsers. When we skip MemoryAccesses for | |||
578 | // stores, we no longer can rely solely on the def-use chains of MemorySSA. | |||
579 | mutable DenseMap<const MemoryAccess *, SmallPtrSet<MemoryAccess *, 2>> | |||
580 | MemoryToUsers; | |||
581 | ||||
582 | // A table storing which memorydefs/phis represent a memory state provably | |||
583 | // equivalent to another memory state. | |||
584 | // We could use the congruence class machinery, but the MemoryAccess's are | |||
585 | // abstract memory states, so they can only ever be equivalent to each other, | |||
586 | // and not to constants, etc. | |||
587 | DenseMap<const MemoryAccess *, CongruenceClass *> MemoryAccessToClass; | |||
588 | ||||
589 | // We could, if we wanted, build MemoryPhiExpressions and | |||
590 | // MemoryVariableExpressions, etc, and value number them the same way we value | |||
591 | // number phi expressions. For the moment, this seems like overkill. They | |||
592 | // can only exist in one of three states: they can be TOP (equal to | |||
593 | // everything), Equivalent to something else, or unique. Because we do not | |||
594 | // create expressions for them, we need to simulate leader change not just | |||
595 | // when they change class, but when they change state. Note: We can do the | |||
596 | // same thing for phis, and avoid having phi expressions if we wanted, We | |||
597 | // should eventually unify in one direction or the other, so this is a little | |||
598 | // bit of an experiment in which turns out easier to maintain. | |||
599 | enum MemoryPhiState { MPS_Invalid, MPS_TOP, MPS_Equivalent, MPS_Unique }; | |||
600 | DenseMap<const MemoryPhi *, MemoryPhiState> MemoryPhiState; | |||
601 | ||||
602 | enum InstCycleState { ICS_Unknown, ICS_CycleFree, ICS_Cycle }; | |||
603 | mutable DenseMap<const Instruction *, InstCycleState> InstCycleState; | |||
604 | ||||
605 | // Expression to class mapping. | |||
606 | using ExpressionClassMap = DenseMap<const Expression *, CongruenceClass *>; | |||
607 | ExpressionClassMap ExpressionToClass; | |||
608 | ||||
609 | // We have a single expression that represents currently DeadExpressions. | |||
610 | // For dead expressions we can prove will stay dead, we mark them with | |||
611 | // DFS number zero. However, it's possible in the case of phi nodes | |||
612 | // for us to assume/prove all arguments are dead during fixpointing. | |||
613 | // We use DeadExpression for that case. | |||
614 | DeadExpression *SingletonDeadExpression = nullptr; | |||
615 | ||||
616 | // Which values have changed as a result of leader changes. | |||
617 | SmallPtrSet<Value *, 8> LeaderChanges; | |||
618 | ||||
619 | // Reachability info. | |||
620 | using BlockEdge = BasicBlockEdge; | |||
621 | DenseSet<BlockEdge> ReachableEdges; | |||
622 | SmallPtrSet<const BasicBlock *, 8> ReachableBlocks; | |||
623 | ||||
624 | // This is a bitvector because, on larger functions, we may have | |||
625 | // thousands of touched instructions at once (entire blocks, | |||
626 | // instructions with hundreds of uses, etc). Even with optimization | |||
627 | // for when we mark whole blocks as touched, when this was a | |||
628 | // SmallPtrSet or DenseSet, for some functions, we spent >20% of all | |||
629 | // the time in GVN just managing this list. The bitvector, on the | |||
630 | // other hand, efficiently supports test/set/clear of both | |||
631 | // individual and ranges, as well as "find next element" This | |||
632 | // enables us to use it as a worklist with essentially 0 cost. | |||
633 | BitVector TouchedInstructions; | |||
634 | ||||
635 | DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange; | |||
636 | ||||
637 | #ifndef NDEBUG | |||
638 | // Debugging for how many times each block and instruction got processed. | |||
639 | DenseMap<const Value *, unsigned> ProcessedCount; | |||
640 | #endif | |||
641 | ||||
642 | // DFS info. | |||
643 | // This contains a mapping from Instructions to DFS numbers. | |||
644 | // The numbering starts at 1. An instruction with DFS number zero | |||
645 | // means that the instruction is dead. | |||
646 | DenseMap<const Value *, unsigned> InstrDFS; | |||
647 | ||||
648 | // This contains the mapping DFS numbers to instructions. | |||
649 | SmallVector<Value *, 32> DFSToInstr; | |||
650 | ||||
651 | // Deletion info. | |||
652 | SmallPtrSet<Instruction *, 8> InstructionsToErase; | |||
653 | ||||
654 | public: | |||
655 | NewGVN(Function &F, DominatorTree *DT, AssumptionCache *AC, | |||
656 | TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA, | |||
657 | const DataLayout &DL) | |||
658 | : F(F), DT(DT), TLI(TLI), AA(AA), MSSA(MSSA), DL(DL), | |||
659 | PredInfo(make_unique<PredicateInfo>(F, *DT, *AC)), | |||
660 | SQ(DL, TLI, DT, AC, /*CtxI=*/nullptr, /*UseInstrInfo=*/false) {} | |||
661 | ||||
662 | bool runGVN(); | |||
663 | ||||
664 | private: | |||
665 | // Expression handling. | |||
666 | const Expression *createExpression(Instruction *) const; | |||
667 | const Expression *createBinaryExpression(unsigned, Type *, Value *, Value *, | |||
668 | Instruction *) const; | |||
669 | ||||
670 | // Our canonical form for phi arguments is a pair of incoming value, incoming | |||
671 | // basic block. | |||
672 | using ValPair = std::pair<Value *, BasicBlock *>; | |||
673 | ||||
674 | PHIExpression *createPHIExpression(ArrayRef<ValPair>, const Instruction *, | |||
675 | BasicBlock *, bool &HasBackEdge, | |||
676 | bool &OriginalOpsConstant) const; | |||
677 | const DeadExpression *createDeadExpression() const; | |||
678 | const VariableExpression *createVariableExpression(Value *) const; | |||
679 | const ConstantExpression *createConstantExpression(Constant *) const; | |||
680 | const Expression *createVariableOrConstant(Value *V) const; | |||
681 | const UnknownExpression *createUnknownExpression(Instruction *) const; | |||
682 | const StoreExpression *createStoreExpression(StoreInst *, | |||
683 | const MemoryAccess *) const; | |||
684 | LoadExpression *createLoadExpression(Type *, Value *, LoadInst *, | |||
685 | const MemoryAccess *) const; | |||
686 | const CallExpression *createCallExpression(CallInst *, | |||
687 | const MemoryAccess *) const; | |||
688 | const AggregateValueExpression * | |||
689 | createAggregateValueExpression(Instruction *) const; | |||
690 | bool setBasicExpressionInfo(Instruction *, BasicExpression *) const; | |||
691 | ||||
692 | // Congruence class handling. | |||
693 | CongruenceClass *createCongruenceClass(Value *Leader, const Expression *E) { | |||
694 | auto *result = new CongruenceClass(NextCongruenceNum++, Leader, E); | |||
695 | CongruenceClasses.emplace_back(result); | |||
696 | return result; | |||
697 | } | |||
698 | ||||
699 | CongruenceClass *createMemoryClass(MemoryAccess *MA) { | |||
700 | auto *CC = createCongruenceClass(nullptr, nullptr); | |||
701 | CC->setMemoryLeader(MA); | |||
702 | return CC; | |||
703 | } | |||
704 | ||||
705 | CongruenceClass *ensureLeaderOfMemoryClass(MemoryAccess *MA) { | |||
706 | auto *CC = getMemoryClass(MA); | |||
707 | if (CC->getMemoryLeader() != MA) | |||
708 | CC = createMemoryClass(MA); | |||
709 | return CC; | |||
710 | } | |||
711 | ||||
712 | CongruenceClass *createSingletonCongruenceClass(Value *Member) { | |||
713 | CongruenceClass *CClass = createCongruenceClass(Member, nullptr); | |||
714 | CClass->insert(Member); | |||
715 | ValueToClass[Member] = CClass; | |||
716 | return CClass; | |||
717 | } | |||
718 | ||||
719 | void initializeCongruenceClasses(Function &F); | |||
720 | const Expression *makePossiblePHIOfOps(Instruction *, | |||
721 | SmallPtrSetImpl<Value *> &); | |||
722 | Value *findLeaderForInst(Instruction *ValueOp, | |||
723 | SmallPtrSetImpl<Value *> &Visited, | |||
724 | MemoryAccess *MemAccess, Instruction *OrigInst, | |||
725 | BasicBlock *PredBB); | |||
726 | bool OpIsSafeForPHIOfOpsHelper(Value *V, const BasicBlock *PHIBlock, | |||
727 | SmallPtrSetImpl<const Value *> &Visited, | |||
728 | SmallVectorImpl<Instruction *> &Worklist); | |||
729 | bool OpIsSafeForPHIOfOps(Value *Op, const BasicBlock *PHIBlock, | |||
730 | SmallPtrSetImpl<const Value *> &); | |||
731 | void addPhiOfOps(PHINode *Op, BasicBlock *BB, Instruction *ExistingValue); | |||
732 | void removePhiOfOps(Instruction *I, PHINode *PHITemp); | |||
733 | ||||
734 | // Value number an Instruction or MemoryPhi. | |||
735 | void valueNumberMemoryPhi(MemoryPhi *); | |||
736 | void valueNumberInstruction(Instruction *); | |||
737 | ||||
738 | // Symbolic evaluation. | |||
739 | const Expression *checkSimplificationResults(Expression *, Instruction *, | |||
740 | Value *) const; | |||
741 | const Expression *performSymbolicEvaluation(Value *, | |||
742 | SmallPtrSetImpl<Value *> &) const; | |||
743 | const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *, | |||
744 | Instruction *, | |||
745 | MemoryAccess *) const; | |||
746 | const Expression *performSymbolicLoadEvaluation(Instruction *) const; | |||
747 | const Expression *performSymbolicStoreEvaluation(Instruction *) const; | |||
748 | const Expression *performSymbolicCallEvaluation(Instruction *) const; | |||
749 | void sortPHIOps(MutableArrayRef<ValPair> Ops) const; | |||
750 | const Expression *performSymbolicPHIEvaluation(ArrayRef<ValPair>, | |||
751 | Instruction *I, | |||
752 | BasicBlock *PHIBlock) const; | |||
753 | const Expression *performSymbolicAggrValueEvaluation(Instruction *) const; | |||
754 | const Expression *performSymbolicCmpEvaluation(Instruction *) const; | |||
755 | const Expression *performSymbolicPredicateInfoEvaluation(Instruction *) const; | |||
756 | ||||
757 | // Congruence finding. | |||
758 | bool someEquivalentDominates(const Instruction *, const Instruction *) const; | |||
759 | Value *lookupOperandLeader(Value *) const; | |||
760 | CongruenceClass *getClassForExpression(const Expression *E) const; | |||
761 | void performCongruenceFinding(Instruction *, const Expression *); | |||
762 | void moveValueToNewCongruenceClass(Instruction *, const Expression *, | |||
763 | CongruenceClass *, CongruenceClass *); | |||
764 | void moveMemoryToNewCongruenceClass(Instruction *, MemoryAccess *, | |||
765 | CongruenceClass *, CongruenceClass *); | |||
766 | Value *getNextValueLeader(CongruenceClass *) const; | |||
767 | const MemoryAccess *getNextMemoryLeader(CongruenceClass *) const; | |||
768 | bool setMemoryClass(const MemoryAccess *From, CongruenceClass *To); | |||
769 | CongruenceClass *getMemoryClass(const MemoryAccess *MA) const; | |||
770 | const MemoryAccess *lookupMemoryLeader(const MemoryAccess *) const; | |||
771 | bool isMemoryAccessTOP(const MemoryAccess *) const; | |||
772 | ||||
773 | // Ranking | |||
774 | unsigned int getRank(const Value *) const; | |||
775 | bool shouldSwapOperands(const Value *, const Value *) const; | |||
776 | ||||
777 | // Reachability handling. | |||
778 | void updateReachableEdge(BasicBlock *, BasicBlock *); | |||
779 | void processOutgoingEdges(Instruction *, BasicBlock *); | |||
780 | Value *findConditionEquivalence(Value *) const; | |||
781 | ||||
782 | // Elimination. | |||
783 | struct ValueDFS; | |||
784 | void convertClassToDFSOrdered(const CongruenceClass &, | |||
785 | SmallVectorImpl<ValueDFS> &, | |||
786 | DenseMap<const Value *, unsigned int> &, | |||
787 | SmallPtrSetImpl<Instruction *> &) const; | |||
788 | void convertClassToLoadsAndStores(const CongruenceClass &, | |||
789 | SmallVectorImpl<ValueDFS> &) const; | |||
790 | ||||
791 | bool eliminateInstructions(Function &); | |||
792 | void replaceInstruction(Instruction *, Value *); | |||
793 | void markInstructionForDeletion(Instruction *); | |||
794 | void deleteInstructionsInBlock(BasicBlock *); | |||
795 | Value *findPHIOfOpsLeader(const Expression *, const Instruction *, | |||
796 | const BasicBlock *) const; | |||
797 | ||||
798 | // New instruction creation. | |||
799 | void handleNewInstruction(Instruction *) {} | |||
800 | ||||
801 | // Various instruction touch utilities | |||
802 | template <typename Map, typename KeyType, typename Func> | |||
803 | void for_each_found(Map &, const KeyType &, Func); | |||
804 | template <typename Map, typename KeyType> | |||
805 | void touchAndErase(Map &, const KeyType &); | |||
806 | void markUsersTouched(Value *); | |||
807 | void markMemoryUsersTouched(const MemoryAccess *); | |||
808 | void markMemoryDefTouched(const MemoryAccess *); | |||
809 | void markPredicateUsersTouched(Instruction *); | |||
810 | void markValueLeaderChangeTouched(CongruenceClass *CC); | |||
811 | void markMemoryLeaderChangeTouched(CongruenceClass *CC); | |||
812 | void markPhiOfOpsChanged(const Expression *E); | |||
813 | void addPredicateUsers(const PredicateBase *, Instruction *) const; | |||
814 | void addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const; | |||
815 | void addAdditionalUsers(Value *To, Value *User) const; | |||
816 | ||||
817 | // Main loop of value numbering | |||
818 | void iterateTouchedInstructions(); | |||
819 | ||||
820 | // Utilities. | |||
821 | void cleanupTables(); | |||
822 | std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned); | |||
823 | void updateProcessedCount(const Value *V); | |||
824 | void verifyMemoryCongruency() const; | |||
825 | void verifyIterationSettled(Function &F); | |||
826 | void verifyStoreExpressions() const; | |||
827 | bool singleReachablePHIPath(SmallPtrSet<const MemoryAccess *, 8> &, | |||
828 | const MemoryAccess *, const MemoryAccess *) const; | |||
829 | BasicBlock *getBlockForValue(Value *V) const; | |||
830 | void deleteExpression(const Expression *E) const; | |||
831 | MemoryUseOrDef *getMemoryAccess(const Instruction *) const; | |||
832 | MemoryAccess *getDefiningAccess(const MemoryAccess *) const; | |||
833 | MemoryPhi *getMemoryAccess(const BasicBlock *) const; | |||
834 | template <class T, class Range> T *getMinDFSOfRange(const Range &) const; | |||
835 | ||||
836 | unsigned InstrToDFSNum(const Value *V) const { | |||
837 | assert(isa<Instruction>(V) && "This should not be used for MemoryAccesses")((isa<Instruction>(V) && "This should not be used for MemoryAccesses" ) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(V) && \"This should not be used for MemoryAccesses\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 837, __PRETTY_FUNCTION__)); | |||
838 | return InstrDFS.lookup(V); | |||
839 | } | |||
840 | ||||
841 | unsigned InstrToDFSNum(const MemoryAccess *MA) const { | |||
842 | return MemoryToDFSNum(MA); | |||
843 | } | |||
844 | ||||
845 | Value *InstrFromDFSNum(unsigned DFSNum) { return DFSToInstr[DFSNum]; } | |||
846 | ||||
847 | // Given a MemoryAccess, return the relevant instruction DFS number. Note: | |||
848 | // This deliberately takes a value so it can be used with Use's, which will | |||
849 | // auto-convert to Value's but not to MemoryAccess's. | |||
850 | unsigned MemoryToDFSNum(const Value *MA) const { | |||
851 | assert(isa<MemoryAccess>(MA) &&((isa<MemoryAccess>(MA) && "This should not be used with instructions" ) ? static_cast<void> (0) : __assert_fail ("isa<MemoryAccess>(MA) && \"This should not be used with instructions\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 852, __PRETTY_FUNCTION__)) | |||
852 | "This should not be used with instructions")((isa<MemoryAccess>(MA) && "This should not be used with instructions" ) ? static_cast<void> (0) : __assert_fail ("isa<MemoryAccess>(MA) && \"This should not be used with instructions\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 852, __PRETTY_FUNCTION__)); | |||
853 | return isa<MemoryUseOrDef>(MA) | |||
854 | ? InstrToDFSNum(cast<MemoryUseOrDef>(MA)->getMemoryInst()) | |||
855 | : InstrDFS.lookup(MA); | |||
856 | } | |||
857 | ||||
858 | bool isCycleFree(const Instruction *) const; | |||
859 | bool isBackedge(BasicBlock *From, BasicBlock *To) const; | |||
860 | ||||
861 | // Debug counter info. When verifying, we have to reset the value numbering | |||
862 | // debug counter to the same state it started in to get the same results. | |||
863 | int64_t StartingVNCounter; | |||
864 | }; | |||
865 | ||||
866 | } // end anonymous namespace | |||
867 | ||||
868 | template <typename T> | |||
869 | static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) { | |||
870 | if (!isa<LoadExpression>(RHS) && !isa<StoreExpression>(RHS)) | |||
871 | return false; | |||
872 | return LHS.MemoryExpression::equals(RHS); | |||
873 | } | |||
874 | ||||
875 | bool LoadExpression::equals(const Expression &Other) const { | |||
876 | return equalsLoadStoreHelper(*this, Other); | |||
877 | } | |||
878 | ||||
879 | bool StoreExpression::equals(const Expression &Other) const { | |||
880 | if (!equalsLoadStoreHelper(*this, Other)) | |||
881 | return false; | |||
882 | // Make sure that store vs store includes the value operand. | |||
883 | if (const auto *S = dyn_cast<StoreExpression>(&Other)) | |||
884 | if (getStoredValue() != S->getStoredValue()) | |||
885 | return false; | |||
886 | return true; | |||
887 | } | |||
888 | ||||
889 | // Determine if the edge From->To is a backedge | |||
890 | bool NewGVN::isBackedge(BasicBlock *From, BasicBlock *To) const { | |||
891 | return From == To || | |||
892 | RPOOrdering.lookup(DT->getNode(From)) >= | |||
893 | RPOOrdering.lookup(DT->getNode(To)); | |||
894 | } | |||
895 | ||||
896 | #ifndef NDEBUG | |||
897 | static std::string getBlockName(const BasicBlock *B) { | |||
898 | return DOTGraphTraits<const Function *>::getSimpleNodeLabel(B, nullptr); | |||
899 | } | |||
900 | #endif | |||
901 | ||||
902 | // Get a MemoryAccess for an instruction, fake or real. | |||
903 | MemoryUseOrDef *NewGVN::getMemoryAccess(const Instruction *I) const { | |||
904 | auto *Result = MSSA->getMemoryAccess(I); | |||
905 | return Result ? Result : TempToMemory.lookup(I); | |||
906 | } | |||
907 | ||||
908 | // Get a MemoryPhi for a basic block. These are all real. | |||
909 | MemoryPhi *NewGVN::getMemoryAccess(const BasicBlock *BB) const { | |||
910 | return MSSA->getMemoryAccess(BB); | |||
911 | } | |||
912 | ||||
913 | // Get the basic block from an instruction/memory value. | |||
914 | BasicBlock *NewGVN::getBlockForValue(Value *V) const { | |||
915 | if (auto *I = dyn_cast<Instruction>(V)) { | |||
916 | auto *Parent = I->getParent(); | |||
917 | if (Parent) | |||
918 | return Parent; | |||
919 | Parent = TempToBlock.lookup(V); | |||
920 | assert(Parent && "Every fake instruction should have a block")((Parent && "Every fake instruction should have a block" ) ? static_cast<void> (0) : __assert_fail ("Parent && \"Every fake instruction should have a block\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 920, __PRETTY_FUNCTION__)); | |||
921 | return Parent; | |||
922 | } | |||
923 | ||||
924 | auto *MP = dyn_cast<MemoryPhi>(V); | |||
925 | assert(MP && "Should have been an instruction or a MemoryPhi")((MP && "Should have been an instruction or a MemoryPhi" ) ? static_cast<void> (0) : __assert_fail ("MP && \"Should have been an instruction or a MemoryPhi\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 925, __PRETTY_FUNCTION__)); | |||
926 | return MP->getBlock(); | |||
927 | } | |||
928 | ||||
929 | // Delete a definitely dead expression, so it can be reused by the expression | |||
930 | // allocator. Some of these are not in creation functions, so we have to accept | |||
931 | // const versions. | |||
932 | void NewGVN::deleteExpression(const Expression *E) const { | |||
933 | assert(isa<BasicExpression>(E))((isa<BasicExpression>(E)) ? static_cast<void> (0 ) : __assert_fail ("isa<BasicExpression>(E)", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 933, __PRETTY_FUNCTION__)); | |||
934 | auto *BE = cast<BasicExpression>(E); | |||
935 | const_cast<BasicExpression *>(BE)->deallocateOperands(ArgRecycler); | |||
936 | ExpressionAllocator.Deallocate(E); | |||
937 | } | |||
938 | ||||
939 | // If V is a predicateinfo copy, get the thing it is a copy of. | |||
940 | static Value *getCopyOf(const Value *V) { | |||
941 | if (auto *II = dyn_cast<IntrinsicInst>(V)) | |||
942 | if (II->getIntrinsicID() == Intrinsic::ssa_copy) | |||
943 | return II->getOperand(0); | |||
944 | return nullptr; | |||
945 | } | |||
946 | ||||
947 | // Return true if V is really PN, even accounting for predicateinfo copies. | |||
948 | static bool isCopyOfPHI(const Value *V, const PHINode *PN) { | |||
949 | return V == PN || getCopyOf(V) == PN; | |||
950 | } | |||
951 | ||||
952 | static bool isCopyOfAPHI(const Value *V) { | |||
953 | auto *CO = getCopyOf(V); | |||
954 | return CO && isa<PHINode>(CO); | |||
955 | } | |||
956 | ||||
957 | // Sort PHI Operands into a canonical order. What we use here is an RPO | |||
958 | // order. The BlockInstRange numbers are generated in an RPO walk of the basic | |||
959 | // blocks. | |||
960 | void NewGVN::sortPHIOps(MutableArrayRef<ValPair> Ops) const { | |||
961 | llvm::sort(Ops, [&](const ValPair &P1, const ValPair &P2) { | |||
962 | return BlockInstRange.lookup(P1.second).first < | |||
963 | BlockInstRange.lookup(P2.second).first; | |||
964 | }); | |||
965 | } | |||
966 | ||||
967 | // Return true if V is a value that will always be available (IE can | |||
968 | // be placed anywhere) in the function. We don't do globals here | |||
969 | // because they are often worse to put in place. | |||
970 | static bool alwaysAvailable(Value *V) { | |||
971 | return isa<Constant>(V) || isa<Argument>(V); | |||
972 | } | |||
973 | ||||
974 | // Create a PHIExpression from an array of {incoming edge, value} pairs. I is | |||
975 | // the original instruction we are creating a PHIExpression for (but may not be | |||
976 | // a phi node). We require, as an invariant, that all the PHIOperands in the | |||
977 | // same block are sorted the same way. sortPHIOps will sort them into a | |||
978 | // canonical order. | |||
979 | PHIExpression *NewGVN::createPHIExpression(ArrayRef<ValPair> PHIOperands, | |||
980 | const Instruction *I, | |||
981 | BasicBlock *PHIBlock, | |||
982 | bool &HasBackedge, | |||
983 | bool &OriginalOpsConstant) const { | |||
984 | unsigned NumOps = PHIOperands.size(); | |||
985 | auto *E = new (ExpressionAllocator) PHIExpression(NumOps, PHIBlock); | |||
986 | ||||
987 | E->allocateOperands(ArgRecycler, ExpressionAllocator); | |||
988 | E->setType(PHIOperands.begin()->first->getType()); | |||
989 | E->setOpcode(Instruction::PHI); | |||
990 | ||||
991 | // Filter out unreachable phi operands. | |||
992 | auto Filtered = make_filter_range(PHIOperands, [&](const ValPair &P) { | |||
993 | auto *BB = P.second; | |||
994 | if (auto *PHIOp = dyn_cast<PHINode>(I)) | |||
995 | if (isCopyOfPHI(P.first, PHIOp)) | |||
996 | return false; | |||
997 | if (!ReachableEdges.count({BB, PHIBlock})) | |||
998 | return false; | |||
999 | // Things in TOPClass are equivalent to everything. | |||
1000 | if (ValueToClass.lookup(P.first) == TOPClass) | |||
1001 | return false; | |||
1002 | OriginalOpsConstant = OriginalOpsConstant && isa<Constant>(P.first); | |||
1003 | HasBackedge = HasBackedge || isBackedge(BB, PHIBlock); | |||
1004 | return lookupOperandLeader(P.first) != I; | |||
1005 | }); | |||
1006 | std::transform(Filtered.begin(), Filtered.end(), op_inserter(E), | |||
1007 | [&](const ValPair &P) -> Value * { | |||
1008 | return lookupOperandLeader(P.first); | |||
1009 | }); | |||
1010 | return E; | |||
1011 | } | |||
1012 | ||||
1013 | // Set basic expression info (Arguments, type, opcode) for Expression | |||
1014 | // E from Instruction I in block B. | |||
1015 | bool NewGVN::setBasicExpressionInfo(Instruction *I, BasicExpression *E) const { | |||
1016 | bool AllConstant = true; | |||
1017 | if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) | |||
1018 | E->setType(GEP->getSourceElementType()); | |||
1019 | else | |||
1020 | E->setType(I->getType()); | |||
1021 | E->setOpcode(I->getOpcode()); | |||
1022 | E->allocateOperands(ArgRecycler, ExpressionAllocator); | |||
1023 | ||||
1024 | // Transform the operand array into an operand leader array, and keep track of | |||
1025 | // whether all members are constant. | |||
1026 | std::transform(I->op_begin(), I->op_end(), op_inserter(E), [&](Value *O) { | |||
1027 | auto Operand = lookupOperandLeader(O); | |||
1028 | AllConstant = AllConstant && isa<Constant>(Operand); | |||
1029 | return Operand; | |||
1030 | }); | |||
1031 | ||||
1032 | return AllConstant; | |||
1033 | } | |||
1034 | ||||
1035 | const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T, | |||
1036 | Value *Arg1, Value *Arg2, | |||
1037 | Instruction *I) const { | |||
1038 | auto *E = new (ExpressionAllocator) BasicExpression(2); | |||
1039 | ||||
1040 | E->setType(T); | |||
1041 | E->setOpcode(Opcode); | |||
1042 | E->allocateOperands(ArgRecycler, ExpressionAllocator); | |||
1043 | if (Instruction::isCommutative(Opcode)) { | |||
1044 | // Ensure that commutative instructions that only differ by a permutation | |||
1045 | // of their operands get the same value number by sorting the operand value | |||
1046 | // numbers. Since all commutative instructions have two operands it is more | |||
1047 | // efficient to sort by hand rather than using, say, std::sort. | |||
1048 | if (shouldSwapOperands(Arg1, Arg2)) | |||
1049 | std::swap(Arg1, Arg2); | |||
1050 | } | |||
1051 | E->op_push_back(lookupOperandLeader(Arg1)); | |||
1052 | E->op_push_back(lookupOperandLeader(Arg2)); | |||
1053 | ||||
1054 | Value *V = SimplifyBinOp(Opcode, E->getOperand(0), E->getOperand(1), SQ); | |||
1055 | if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V)) | |||
1056 | return SimplifiedE; | |||
1057 | return E; | |||
1058 | } | |||
1059 | ||||
1060 | // Take a Value returned by simplification of Expression E/Instruction | |||
1061 | // I, and see if it resulted in a simpler expression. If so, return | |||
1062 | // that expression. | |||
1063 | const Expression *NewGVN::checkSimplificationResults(Expression *E, | |||
1064 | Instruction *I, | |||
1065 | Value *V) const { | |||
1066 | if (!V) | |||
1067 | return nullptr; | |||
1068 | if (auto *C = dyn_cast<Constant>(V)) { | |||
1069 | if (I) | |||
1070 | LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Simplified " << *I << " to " << " constant " << *C << "\n"; } } while (false) | |||
1071 | << " constant " << *C << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Simplified " << *I << " to " << " constant " << *C << "\n"; } } while (false); | |||
1072 | NumGVNOpsSimplified++; | |||
1073 | assert(isa<BasicExpression>(E) &&((isa<BasicExpression>(E) && "We should always have had a basic expression here" ) ? static_cast<void> (0) : __assert_fail ("isa<BasicExpression>(E) && \"We should always have had a basic expression here\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1074, __PRETTY_FUNCTION__)) | |||
1074 | "We should always have had a basic expression here")((isa<BasicExpression>(E) && "We should always have had a basic expression here" ) ? static_cast<void> (0) : __assert_fail ("isa<BasicExpression>(E) && \"We should always have had a basic expression here\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1074, __PRETTY_FUNCTION__)); | |||
1075 | deleteExpression(E); | |||
1076 | return createConstantExpression(C); | |||
1077 | } else if (isa<Argument>(V) || isa<GlobalVariable>(V)) { | |||
1078 | if (I) | |||
1079 | LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Simplified " << *I << " to " << " variable " << *V << "\n"; } } while (false) | |||
1080 | << " variable " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Simplified " << *I << " to " << " variable " << *V << "\n"; } } while (false); | |||
1081 | deleteExpression(E); | |||
1082 | return createVariableExpression(V); | |||
1083 | } | |||
1084 | ||||
1085 | CongruenceClass *CC = ValueToClass.lookup(V); | |||
1086 | if (CC) { | |||
1087 | if (CC->getLeader() && CC->getLeader() != I) { | |||
1088 | // If we simplified to something else, we need to communicate | |||
1089 | // that we're users of the value we simplified to. | |||
1090 | if (I != V) { | |||
1091 | // Don't add temporary instructions to the user lists. | |||
1092 | if (!AllTempInstructions.count(I)) | |||
1093 | addAdditionalUsers(V, I); | |||
1094 | } | |||
1095 | return createVariableOrConstant(CC->getLeader()); | |||
1096 | } | |||
1097 | if (CC->getDefiningExpr()) { | |||
1098 | // If we simplified to something else, we need to communicate | |||
1099 | // that we're users of the value we simplified to. | |||
1100 | if (I != V) { | |||
1101 | // Don't add temporary instructions to the user lists. | |||
1102 | if (!AllTempInstructions.count(I)) | |||
1103 | addAdditionalUsers(V, I); | |||
1104 | } | |||
1105 | ||||
1106 | if (I) | |||
1107 | LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Simplified " << *I << " to " << " expression " << *CC->getDefiningExpr () << "\n"; } } while (false) | |||
1108 | << " expression " << *CC->getDefiningExpr() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Simplified " << *I << " to " << " expression " << *CC->getDefiningExpr () << "\n"; } } while (false); | |||
1109 | NumGVNOpsSimplified++; | |||
1110 | deleteExpression(E); | |||
1111 | return CC->getDefiningExpr(); | |||
1112 | } | |||
1113 | } | |||
1114 | ||||
1115 | return nullptr; | |||
1116 | } | |||
1117 | ||||
1118 | // Create a value expression from the instruction I, replacing operands with | |||
1119 | // their leaders. | |||
1120 | ||||
1121 | const Expression *NewGVN::createExpression(Instruction *I) const { | |||
1122 | auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands()); | |||
1123 | ||||
1124 | bool AllConstant = setBasicExpressionInfo(I, E); | |||
1125 | ||||
1126 | if (I->isCommutative()) { | |||
1127 | // Ensure that commutative instructions that only differ by a permutation | |||
1128 | // of their operands get the same value number by sorting the operand value | |||
1129 | // numbers. Since all commutative instructions have two operands it is more | |||
1130 | // efficient to sort by hand rather than using, say, std::sort. | |||
1131 | assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!")((I->getNumOperands() == 2 && "Unsupported commutative instruction!" ) ? static_cast<void> (0) : __assert_fail ("I->getNumOperands() == 2 && \"Unsupported commutative instruction!\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1131, __PRETTY_FUNCTION__)); | |||
1132 | if (shouldSwapOperands(E->getOperand(0), E->getOperand(1))) | |||
1133 | E->swapOperands(0, 1); | |||
1134 | } | |||
1135 | // Perform simplification. | |||
1136 | if (auto *CI = dyn_cast<CmpInst>(I)) { | |||
1137 | // Sort the operand value numbers so x<y and y>x get the same value | |||
1138 | // number. | |||
1139 | CmpInst::Predicate Predicate = CI->getPredicate(); | |||
1140 | if (shouldSwapOperands(E->getOperand(0), E->getOperand(1))) { | |||
1141 | E->swapOperands(0, 1); | |||
1142 | Predicate = CmpInst::getSwappedPredicate(Predicate); | |||
1143 | } | |||
1144 | E->setOpcode((CI->getOpcode() << 8) | Predicate); | |||
1145 | // TODO: 25% of our time is spent in SimplifyCmpInst with pointer operands | |||
1146 | assert(I->getOperand(0)->getType() == I->getOperand(1)->getType() &&((I->getOperand(0)->getType() == I->getOperand(1)-> getType() && "Wrong types on cmp instruction") ? static_cast <void> (0) : __assert_fail ("I->getOperand(0)->getType() == I->getOperand(1)->getType() && \"Wrong types on cmp instruction\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1147, __PRETTY_FUNCTION__)) | |||
1147 | "Wrong types on cmp instruction")((I->getOperand(0)->getType() == I->getOperand(1)-> getType() && "Wrong types on cmp instruction") ? static_cast <void> (0) : __assert_fail ("I->getOperand(0)->getType() == I->getOperand(1)->getType() && \"Wrong types on cmp instruction\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1147, __PRETTY_FUNCTION__)); | |||
1148 | assert((E->getOperand(0)->getType() == I->getOperand(0)->getType() &&(((E->getOperand(0)->getType() == I->getOperand(0)-> getType() && E->getOperand(1)->getType() == I-> getOperand(1)->getType())) ? static_cast<void> (0) : __assert_fail ("(E->getOperand(0)->getType() == I->getOperand(0)->getType() && E->getOperand(1)->getType() == I->getOperand(1)->getType())" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1149, __PRETTY_FUNCTION__)) | |||
1149 | E->getOperand(1)->getType() == I->getOperand(1)->getType()))(((E->getOperand(0)->getType() == I->getOperand(0)-> getType() && E->getOperand(1)->getType() == I-> getOperand(1)->getType())) ? static_cast<void> (0) : __assert_fail ("(E->getOperand(0)->getType() == I->getOperand(0)->getType() && E->getOperand(1)->getType() == I->getOperand(1)->getType())" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1149, __PRETTY_FUNCTION__)); | |||
1150 | Value *V = | |||
1151 | SimplifyCmpInst(Predicate, E->getOperand(0), E->getOperand(1), SQ); | |||
1152 | if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V)) | |||
1153 | return SimplifiedE; | |||
1154 | } else if (isa<SelectInst>(I)) { | |||
1155 | if (isa<Constant>(E->getOperand(0)) || | |||
1156 | E->getOperand(1) == E->getOperand(2)) { | |||
1157 | assert(E->getOperand(1)->getType() == I->getOperand(1)->getType() &&((E->getOperand(1)->getType() == I->getOperand(1)-> getType() && E->getOperand(2)->getType() == I-> getOperand(2)->getType()) ? static_cast<void> (0) : __assert_fail ("E->getOperand(1)->getType() == I->getOperand(1)->getType() && E->getOperand(2)->getType() == I->getOperand(2)->getType()" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1158, __PRETTY_FUNCTION__)) | |||
1158 | E->getOperand(2)->getType() == I->getOperand(2)->getType())((E->getOperand(1)->getType() == I->getOperand(1)-> getType() && E->getOperand(2)->getType() == I-> getOperand(2)->getType()) ? static_cast<void> (0) : __assert_fail ("E->getOperand(1)->getType() == I->getOperand(1)->getType() && E->getOperand(2)->getType() == I->getOperand(2)->getType()" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1158, __PRETTY_FUNCTION__)); | |||
1159 | Value *V = SimplifySelectInst(E->getOperand(0), E->getOperand(1), | |||
1160 | E->getOperand(2), SQ); | |||
1161 | if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V)) | |||
1162 | return SimplifiedE; | |||
1163 | } | |||
1164 | } else if (I->isBinaryOp()) { | |||
1165 | Value *V = | |||
1166 | SimplifyBinOp(E->getOpcode(), E->getOperand(0), E->getOperand(1), SQ); | |||
1167 | if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V)) | |||
1168 | return SimplifiedE; | |||
1169 | } else if (auto *BI = dyn_cast<BitCastInst>(I)) { | |||
1170 | Value *V = | |||
1171 | SimplifyCastInst(BI->getOpcode(), BI->getOperand(0), BI->getType(), SQ); | |||
1172 | if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V)) | |||
1173 | return SimplifiedE; | |||
1174 | } else if (isa<GetElementPtrInst>(I)) { | |||
1175 | Value *V = SimplifyGEPInst( | |||
1176 | E->getType(), ArrayRef<Value *>(E->op_begin(), E->op_end()), SQ); | |||
1177 | if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V)) | |||
1178 | return SimplifiedE; | |||
1179 | } else if (AllConstant) { | |||
1180 | // We don't bother trying to simplify unless all of the operands | |||
1181 | // were constant. | |||
1182 | // TODO: There are a lot of Simplify*'s we could call here, if we | |||
1183 | // wanted to. The original motivating case for this code was a | |||
1184 | // zext i1 false to i8, which we don't have an interface to | |||
1185 | // simplify (IE there is no SimplifyZExt). | |||
1186 | ||||
1187 | SmallVector<Constant *, 8> C; | |||
1188 | for (Value *Arg : E->operands()) | |||
1189 | C.emplace_back(cast<Constant>(Arg)); | |||
1190 | ||||
1191 | if (Value *V = ConstantFoldInstOperands(I, C, DL, TLI)) | |||
1192 | if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V)) | |||
1193 | return SimplifiedE; | |||
1194 | } | |||
1195 | return E; | |||
1196 | } | |||
1197 | ||||
1198 | const AggregateValueExpression * | |||
1199 | NewGVN::createAggregateValueExpression(Instruction *I) const { | |||
1200 | if (auto *II = dyn_cast<InsertValueInst>(I)) { | |||
1201 | auto *E = new (ExpressionAllocator) | |||
1202 | AggregateValueExpression(I->getNumOperands(), II->getNumIndices()); | |||
1203 | setBasicExpressionInfo(I, E); | |||
1204 | E->allocateIntOperands(ExpressionAllocator); | |||
1205 | std::copy(II->idx_begin(), II->idx_end(), int_op_inserter(E)); | |||
1206 | return E; | |||
1207 | } else if (auto *EI = dyn_cast<ExtractValueInst>(I)) { | |||
1208 | auto *E = new (ExpressionAllocator) | |||
1209 | AggregateValueExpression(I->getNumOperands(), EI->getNumIndices()); | |||
1210 | setBasicExpressionInfo(EI, E); | |||
1211 | E->allocateIntOperands(ExpressionAllocator); | |||
1212 | std::copy(EI->idx_begin(), EI->idx_end(), int_op_inserter(E)); | |||
1213 | return E; | |||
1214 | } | |||
1215 | llvm_unreachable("Unhandled type of aggregate value operation")::llvm::llvm_unreachable_internal("Unhandled type of aggregate value operation" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1215); | |||
1216 | } | |||
1217 | ||||
1218 | const DeadExpression *NewGVN::createDeadExpression() const { | |||
1219 | // DeadExpression has no arguments and all DeadExpression's are the same, | |||
1220 | // so we only need one of them. | |||
1221 | return SingletonDeadExpression; | |||
1222 | } | |||
1223 | ||||
1224 | const VariableExpression *NewGVN::createVariableExpression(Value *V) const { | |||
1225 | auto *E = new (ExpressionAllocator) VariableExpression(V); | |||
1226 | E->setOpcode(V->getValueID()); | |||
| ||||
1227 | return E; | |||
1228 | } | |||
1229 | ||||
1230 | const Expression *NewGVN::createVariableOrConstant(Value *V) const { | |||
1231 | if (auto *C = dyn_cast<Constant>(V)) | |||
1232 | return createConstantExpression(C); | |||
1233 | return createVariableExpression(V); | |||
1234 | } | |||
1235 | ||||
1236 | const ConstantExpression *NewGVN::createConstantExpression(Constant *C) const { | |||
1237 | auto *E = new (ExpressionAllocator) ConstantExpression(C); | |||
1238 | E->setOpcode(C->getValueID()); | |||
1239 | return E; | |||
1240 | } | |||
1241 | ||||
1242 | const UnknownExpression *NewGVN::createUnknownExpression(Instruction *I) const { | |||
1243 | auto *E = new (ExpressionAllocator) UnknownExpression(I); | |||
1244 | E->setOpcode(I->getOpcode()); | |||
1245 | return E; | |||
1246 | } | |||
1247 | ||||
1248 | const CallExpression * | |||
1249 | NewGVN::createCallExpression(CallInst *CI, const MemoryAccess *MA) const { | |||
1250 | // FIXME: Add operand bundles for calls. | |||
1251 | auto *E = | |||
1252 | new (ExpressionAllocator) CallExpression(CI->getNumOperands(), CI, MA); | |||
1253 | setBasicExpressionInfo(CI, E); | |||
1254 | return E; | |||
1255 | } | |||
1256 | ||||
1257 | // Return true if some equivalent of instruction Inst dominates instruction U. | |||
1258 | bool NewGVN::someEquivalentDominates(const Instruction *Inst, | |||
1259 | const Instruction *U) const { | |||
1260 | auto *CC = ValueToClass.lookup(Inst); | |||
1261 | // This must be an instruction because we are only called from phi nodes | |||
1262 | // in the case that the value it needs to check against is an instruction. | |||
1263 | ||||
1264 | // The most likely candidates for dominance are the leader and the next leader. | |||
1265 | // The leader or nextleader will dominate in all cases where there is an | |||
1266 | // equivalent that is higher up in the dom tree. | |||
1267 | // We can't *only* check them, however, because the | |||
1268 | // dominator tree could have an infinite number of non-dominating siblings | |||
1269 | // with instructions that are in the right congruence class. | |||
1270 | // A | |||
1271 | // B C D E F G | |||
1272 | // | | |||
1273 | // H | |||
1274 | // Instruction U could be in H, with equivalents in every other sibling. | |||
1275 | // Depending on the rpo order picked, the leader could be the equivalent in | |||
1276 | // any of these siblings. | |||
1277 | if (!CC) | |||
1278 | return false; | |||
1279 | if (alwaysAvailable(CC->getLeader())) | |||
1280 | return true; | |||
1281 | if (DT->dominates(cast<Instruction>(CC->getLeader()), U)) | |||
1282 | return true; | |||
1283 | if (CC->getNextLeader().first && | |||
1284 | DT->dominates(cast<Instruction>(CC->getNextLeader().first), U)) | |||
1285 | return true; | |||
1286 | return llvm::any_of(*CC, [&](const Value *Member) { | |||
1287 | return Member != CC->getLeader() && | |||
1288 | DT->dominates(cast<Instruction>(Member), U); | |||
1289 | }); | |||
1290 | } | |||
1291 | ||||
1292 | // See if we have a congruence class and leader for this operand, and if so, | |||
1293 | // return it. Otherwise, return the operand itself. | |||
1294 | Value *NewGVN::lookupOperandLeader(Value *V) const { | |||
1295 | CongruenceClass *CC = ValueToClass.lookup(V); | |||
1296 | if (CC) { | |||
1297 | // Everything in TOP is represented by undef, as it can be any value. | |||
1298 | // We do have to make sure we get the type right though, so we can't set the | |||
1299 | // RepLeader to undef. | |||
1300 | if (CC == TOPClass) | |||
1301 | return UndefValue::get(V->getType()); | |||
1302 | return CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader(); | |||
1303 | } | |||
1304 | ||||
1305 | return V; | |||
1306 | } | |||
1307 | ||||
1308 | const MemoryAccess *NewGVN::lookupMemoryLeader(const MemoryAccess *MA) const { | |||
1309 | auto *CC = getMemoryClass(MA); | |||
1310 | assert(CC->getMemoryLeader() &&((CC->getMemoryLeader() && "Every MemoryAccess should be mapped to a congruence class with a " "representative memory access") ? static_cast<void> (0 ) : __assert_fail ("CC->getMemoryLeader() && \"Every MemoryAccess should be mapped to a congruence class with a \" \"representative memory access\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1312, __PRETTY_FUNCTION__)) | |||
1311 | "Every MemoryAccess should be mapped to a congruence class with a "((CC->getMemoryLeader() && "Every MemoryAccess should be mapped to a congruence class with a " "representative memory access") ? static_cast<void> (0 ) : __assert_fail ("CC->getMemoryLeader() && \"Every MemoryAccess should be mapped to a congruence class with a \" \"representative memory access\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1312, __PRETTY_FUNCTION__)) | |||
1312 | "representative memory access")((CC->getMemoryLeader() && "Every MemoryAccess should be mapped to a congruence class with a " "representative memory access") ? static_cast<void> (0 ) : __assert_fail ("CC->getMemoryLeader() && \"Every MemoryAccess should be mapped to a congruence class with a \" \"representative memory access\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1312, __PRETTY_FUNCTION__)); | |||
1313 | return CC->getMemoryLeader(); | |||
1314 | } | |||
1315 | ||||
1316 | // Return true if the MemoryAccess is really equivalent to everything. This is | |||
1317 | // equivalent to the lattice value "TOP" in most lattices. This is the initial | |||
1318 | // state of all MemoryAccesses. | |||
1319 | bool NewGVN::isMemoryAccessTOP(const MemoryAccess *MA) const { | |||
1320 | return getMemoryClass(MA) == TOPClass; | |||
1321 | } | |||
1322 | ||||
1323 | LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp, | |||
1324 | LoadInst *LI, | |||
1325 | const MemoryAccess *MA) const { | |||
1326 | auto *E = | |||
1327 | new (ExpressionAllocator) LoadExpression(1, LI, lookupMemoryLeader(MA)); | |||
1328 | E->allocateOperands(ArgRecycler, ExpressionAllocator); | |||
1329 | E->setType(LoadType); | |||
1330 | ||||
1331 | // Give store and loads same opcode so they value number together. | |||
1332 | E->setOpcode(0); | |||
1333 | E->op_push_back(PointerOp); | |||
1334 | if (LI) | |||
1335 | E->setAlignment(LI->getAlignment()); | |||
1336 | ||||
1337 | // TODO: Value number heap versions. We may be able to discover | |||
1338 | // things alias analysis can't on it's own (IE that a store and a | |||
1339 | // load have the same value, and thus, it isn't clobbering the load). | |||
1340 | return E; | |||
1341 | } | |||
1342 | ||||
1343 | const StoreExpression * | |||
1344 | NewGVN::createStoreExpression(StoreInst *SI, const MemoryAccess *MA) const { | |||
1345 | auto *StoredValueLeader = lookupOperandLeader(SI->getValueOperand()); | |||
1346 | auto *E = new (ExpressionAllocator) | |||
1347 | StoreExpression(SI->getNumOperands(), SI, StoredValueLeader, MA); | |||
1348 | E->allocateOperands(ArgRecycler, ExpressionAllocator); | |||
1349 | E->setType(SI->getValueOperand()->getType()); | |||
1350 | ||||
1351 | // Give store and loads same opcode so they value number together. | |||
1352 | E->setOpcode(0); | |||
1353 | E->op_push_back(lookupOperandLeader(SI->getPointerOperand())); | |||
1354 | ||||
1355 | // TODO: Value number heap versions. We may be able to discover | |||
1356 | // things alias analysis can't on it's own (IE that a store and a | |||
1357 | // load have the same value, and thus, it isn't clobbering the load). | |||
1358 | return E; | |||
1359 | } | |||
1360 | ||||
1361 | const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I) const { | |||
1362 | // Unlike loads, we never try to eliminate stores, so we do not check if they | |||
1363 | // are simple and avoid value numbering them. | |||
1364 | auto *SI = cast<StoreInst>(I); | |||
1365 | auto *StoreAccess = getMemoryAccess(SI); | |||
1366 | // Get the expression, if any, for the RHS of the MemoryDef. | |||
1367 | const MemoryAccess *StoreRHS = StoreAccess->getDefiningAccess(); | |||
1368 | if (EnableStoreRefinement) | |||
1369 | StoreRHS = MSSAWalker->getClobberingMemoryAccess(StoreAccess); | |||
1370 | // If we bypassed the use-def chains, make sure we add a use. | |||
1371 | StoreRHS = lookupMemoryLeader(StoreRHS); | |||
1372 | if (StoreRHS != StoreAccess->getDefiningAccess()) | |||
1373 | addMemoryUsers(StoreRHS, StoreAccess); | |||
1374 | // If we are defined by ourselves, use the live on entry def. | |||
1375 | if (StoreRHS == StoreAccess) | |||
1376 | StoreRHS = MSSA->getLiveOnEntryDef(); | |||
1377 | ||||
1378 | if (SI->isSimple()) { | |||
1379 | // See if we are defined by a previous store expression, it already has a | |||
1380 | // value, and it's the same value as our current store. FIXME: Right now, we | |||
1381 | // only do this for simple stores, we should expand to cover memcpys, etc. | |||
1382 | const auto *LastStore = createStoreExpression(SI, StoreRHS); | |||
1383 | const auto *LastCC = ExpressionToClass.lookup(LastStore); | |||
1384 | // We really want to check whether the expression we matched was a store. No | |||
1385 | // easy way to do that. However, we can check that the class we found has a | |||
1386 | // store, which, assuming the value numbering state is not corrupt, is | |||
1387 | // sufficient, because we must also be equivalent to that store's expression | |||
1388 | // for it to be in the same class as the load. | |||
1389 | if (LastCC && LastCC->getStoredValue() == LastStore->getStoredValue()) | |||
1390 | return LastStore; | |||
1391 | // Also check if our value operand is defined by a load of the same memory | |||
1392 | // location, and the memory state is the same as it was then (otherwise, it | |||
1393 | // could have been overwritten later. See test32 in | |||
1394 | // transforms/DeadStoreElimination/simple.ll). | |||
1395 | if (auto *LI = dyn_cast<LoadInst>(LastStore->getStoredValue())) | |||
1396 | if ((lookupOperandLeader(LI->getPointerOperand()) == | |||
1397 | LastStore->getOperand(0)) && | |||
1398 | (lookupMemoryLeader(getMemoryAccess(LI)->getDefiningAccess()) == | |||
1399 | StoreRHS)) | |||
1400 | return LastStore; | |||
1401 | deleteExpression(LastStore); | |||
1402 | } | |||
1403 | ||||
1404 | // If the store is not equivalent to anything, value number it as a store that | |||
1405 | // produces a unique memory state (instead of using it's MemoryUse, we use | |||
1406 | // it's MemoryDef). | |||
1407 | return createStoreExpression(SI, StoreAccess); | |||
1408 | } | |||
1409 | ||||
1410 | // See if we can extract the value of a loaded pointer from a load, a store, or | |||
1411 | // a memory instruction. | |||
1412 | const Expression * | |||
1413 | NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr, | |||
1414 | LoadInst *LI, Instruction *DepInst, | |||
1415 | MemoryAccess *DefiningAccess) const { | |||
1416 | assert((!LI || LI->isSimple()) && "Not a simple load")(((!LI || LI->isSimple()) && "Not a simple load") ? static_cast<void> (0) : __assert_fail ("(!LI || LI->isSimple()) && \"Not a simple load\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1416, __PRETTY_FUNCTION__)); | |||
1417 | if (auto *DepSI = dyn_cast<StoreInst>(DepInst)) { | |||
1418 | // Can't forward from non-atomic to atomic without violating memory model. | |||
1419 | // Also don't need to coerce if they are the same type, we will just | |||
1420 | // propagate. | |||
1421 | if (LI->isAtomic() > DepSI->isAtomic() || | |||
1422 | LoadType == DepSI->getValueOperand()->getType()) | |||
1423 | return nullptr; | |||
1424 | int Offset = analyzeLoadFromClobberingStore(LoadType, LoadPtr, DepSI, DL); | |||
1425 | if (Offset >= 0) { | |||
1426 | if (auto *C = dyn_cast<Constant>( | |||
1427 | lookupOperandLeader(DepSI->getValueOperand()))) { | |||
1428 | LLVM_DEBUG(dbgs() << "Coercing load from store " << *DepSIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Coercing load from store " << *DepSI << " to constant " << *C << "\n"; } } while (false) | |||
1429 | << " to constant " << *C << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Coercing load from store " << *DepSI << " to constant " << *C << "\n"; } } while (false); | |||
1430 | return createConstantExpression( | |||
1431 | getConstantStoreValueForLoad(C, Offset, LoadType, DL)); | |||
1432 | } | |||
1433 | } | |||
1434 | } else if (auto *DepLI = dyn_cast<LoadInst>(DepInst)) { | |||
1435 | // Can't forward from non-atomic to atomic without violating memory model. | |||
1436 | if (LI->isAtomic() > DepLI->isAtomic()) | |||
1437 | return nullptr; | |||
1438 | int Offset = analyzeLoadFromClobberingLoad(LoadType, LoadPtr, DepLI, DL); | |||
1439 | if (Offset >= 0) { | |||
1440 | // We can coerce a constant load into a load. | |||
1441 | if (auto *C = dyn_cast<Constant>(lookupOperandLeader(DepLI))) | |||
1442 | if (auto *PossibleConstant = | |||
1443 | getConstantLoadValueForLoad(C, Offset, LoadType, DL)) { | |||
1444 | LLVM_DEBUG(dbgs() << "Coercing load from load " << *LIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Coercing load from load " << *LI << " to constant " << *PossibleConstant << "\n"; } } while (false) | |||
1445 | << " to constant " << *PossibleConstant << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Coercing load from load " << *LI << " to constant " << *PossibleConstant << "\n"; } } while (false); | |||
1446 | return createConstantExpression(PossibleConstant); | |||
1447 | } | |||
1448 | } | |||
1449 | } else if (auto *DepMI = dyn_cast<MemIntrinsic>(DepInst)) { | |||
1450 | int Offset = analyzeLoadFromClobberingMemInst(LoadType, LoadPtr, DepMI, DL); | |||
1451 | if (Offset >= 0) { | |||
1452 | if (auto *PossibleConstant = | |||
1453 | getConstantMemInstValueForLoad(DepMI, Offset, LoadType, DL)) { | |||
1454 | LLVM_DEBUG(dbgs() << "Coercing load from meminst " << *DepMIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Coercing load from meminst " << *DepMI << " to constant " << *PossibleConstant << "\n"; } } while (false) | |||
1455 | << " to constant " << *PossibleConstant << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Coercing load from meminst " << *DepMI << " to constant " << *PossibleConstant << "\n"; } } while (false); | |||
1456 | return createConstantExpression(PossibleConstant); | |||
1457 | } | |||
1458 | } | |||
1459 | } | |||
1460 | ||||
1461 | // All of the below are only true if the loaded pointer is produced | |||
1462 | // by the dependent instruction. | |||
1463 | if (LoadPtr != lookupOperandLeader(DepInst) && | |||
1464 | !AA->isMustAlias(LoadPtr, DepInst)) | |||
1465 | return nullptr; | |||
1466 | // If this load really doesn't depend on anything, then we must be loading an | |||
1467 | // undef value. This can happen when loading for a fresh allocation with no | |||
1468 | // intervening stores, for example. Note that this is only true in the case | |||
1469 | // that the result of the allocation is pointer equal to the load ptr. | |||
1470 | if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI)) { | |||
1471 | return createConstantExpression(UndefValue::get(LoadType)); | |||
1472 | } | |||
1473 | // If this load occurs either right after a lifetime begin, | |||
1474 | // then the loaded value is undefined. | |||
1475 | else if (auto *II = dyn_cast<IntrinsicInst>(DepInst)) { | |||
1476 | if (II->getIntrinsicID() == Intrinsic::lifetime_start) | |||
1477 | return createConstantExpression(UndefValue::get(LoadType)); | |||
1478 | } | |||
1479 | // If this load follows a calloc (which zero initializes memory), | |||
1480 | // then the loaded value is zero | |||
1481 | else if (isCallocLikeFn(DepInst, TLI)) { | |||
1482 | return createConstantExpression(Constant::getNullValue(LoadType)); | |||
1483 | } | |||
1484 | ||||
1485 | return nullptr; | |||
1486 | } | |||
1487 | ||||
1488 | const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const { | |||
1489 | auto *LI = cast<LoadInst>(I); | |||
1490 | ||||
1491 | // We can eliminate in favor of non-simple loads, but we won't be able to | |||
1492 | // eliminate the loads themselves. | |||
1493 | if (!LI->isSimple()) | |||
1494 | return nullptr; | |||
1495 | ||||
1496 | Value *LoadAddressLeader = lookupOperandLeader(LI->getPointerOperand()); | |||
1497 | // Load of undef is undef. | |||
1498 | if (isa<UndefValue>(LoadAddressLeader)) | |||
1499 | return createConstantExpression(UndefValue::get(LI->getType())); | |||
1500 | MemoryAccess *OriginalAccess = getMemoryAccess(I); | |||
1501 | MemoryAccess *DefiningAccess = | |||
1502 | MSSAWalker->getClobberingMemoryAccess(OriginalAccess); | |||
1503 | ||||
1504 | if (!MSSA->isLiveOnEntryDef(DefiningAccess)) { | |||
1505 | if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) { | |||
1506 | Instruction *DefiningInst = MD->getMemoryInst(); | |||
1507 | // If the defining instruction is not reachable, replace with undef. | |||
1508 | if (!ReachableBlocks.count(DefiningInst->getParent())) | |||
1509 | return createConstantExpression(UndefValue::get(LI->getType())); | |||
1510 | // This will handle stores and memory insts. We only do if it the | |||
1511 | // defining access has a different type, or it is a pointer produced by | |||
1512 | // certain memory operations that cause the memory to have a fixed value | |||
1513 | // (IE things like calloc). | |||
1514 | if (const auto *CoercionResult = | |||
1515 | performSymbolicLoadCoercion(LI->getType(), LoadAddressLeader, LI, | |||
1516 | DefiningInst, DefiningAccess)) | |||
1517 | return CoercionResult; | |||
1518 | } | |||
1519 | } | |||
1520 | ||||
1521 | const auto *LE = createLoadExpression(LI->getType(), LoadAddressLeader, LI, | |||
1522 | DefiningAccess); | |||
1523 | // If our MemoryLeader is not our defining access, add a use to the | |||
1524 | // MemoryLeader, so that we get reprocessed when it changes. | |||
1525 | if (LE->getMemoryLeader() != DefiningAccess) | |||
1526 | addMemoryUsers(LE->getMemoryLeader(), OriginalAccess); | |||
1527 | return LE; | |||
1528 | } | |||
1529 | ||||
1530 | const Expression * | |||
1531 | NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) const { | |||
1532 | auto *PI = PredInfo->getPredicateInfoFor(I); | |||
1533 | if (!PI) | |||
1534 | return nullptr; | |||
1535 | ||||
1536 | LLVM_DEBUG(dbgs() << "Found predicate info from instruction !\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Found predicate info from instruction !\n" ; } } while (false); | |||
1537 | ||||
1538 | auto *PWC = dyn_cast<PredicateWithCondition>(PI); | |||
1539 | if (!PWC) | |||
1540 | return nullptr; | |||
1541 | ||||
1542 | auto *CopyOf = I->getOperand(0); | |||
1543 | auto *Cond = PWC->Condition; | |||
1544 | ||||
1545 | // If this a copy of the condition, it must be either true or false depending | |||
1546 | // on the predicate info type and edge. | |||
1547 | if (CopyOf == Cond) { | |||
1548 | // We should not need to add predicate users because the predicate info is | |||
1549 | // already a use of this operand. | |||
1550 | if (isa<PredicateAssume>(PI)) | |||
1551 | return createConstantExpression(ConstantInt::getTrue(Cond->getType())); | |||
1552 | if (auto *PBranch = dyn_cast<PredicateBranch>(PI)) { | |||
1553 | if (PBranch->TrueEdge) | |||
1554 | return createConstantExpression(ConstantInt::getTrue(Cond->getType())); | |||
1555 | return createConstantExpression(ConstantInt::getFalse(Cond->getType())); | |||
1556 | } | |||
1557 | if (auto *PSwitch = dyn_cast<PredicateSwitch>(PI)) | |||
1558 | return createConstantExpression(cast<Constant>(PSwitch->CaseValue)); | |||
1559 | } | |||
1560 | ||||
1561 | // Not a copy of the condition, so see what the predicates tell us about this | |||
1562 | // value. First, though, we check to make sure the value is actually a copy | |||
1563 | // of one of the condition operands. It's possible, in certain cases, for it | |||
1564 | // to be a copy of a predicateinfo copy. In particular, if two branch | |||
1565 | // operations use the same condition, and one branch dominates the other, we | |||
1566 | // will end up with a copy of a copy. This is currently a small deficiency in | |||
1567 | // predicateinfo. What will end up happening here is that we will value | |||
1568 | // number both copies the same anyway. | |||
1569 | ||||
1570 | // Everything below relies on the condition being a comparison. | |||
1571 | auto *Cmp = dyn_cast<CmpInst>(Cond); | |||
1572 | if (!Cmp) | |||
1573 | return nullptr; | |||
1574 | ||||
1575 | if (CopyOf != Cmp->getOperand(0) && CopyOf != Cmp->getOperand(1)) { | |||
1576 | LLVM_DEBUG(dbgs() << "Copy is not of any condition operands!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Copy is not of any condition operands!\n" ; } } while (false); | |||
1577 | return nullptr; | |||
1578 | } | |||
1579 | Value *FirstOp = lookupOperandLeader(Cmp->getOperand(0)); | |||
1580 | Value *SecondOp = lookupOperandLeader(Cmp->getOperand(1)); | |||
1581 | bool SwappedOps = false; | |||
1582 | // Sort the ops. | |||
1583 | if (shouldSwapOperands(FirstOp, SecondOp)) { | |||
1584 | std::swap(FirstOp, SecondOp); | |||
1585 | SwappedOps = true; | |||
1586 | } | |||
1587 | CmpInst::Predicate Predicate = | |||
1588 | SwappedOps ? Cmp->getSwappedPredicate() : Cmp->getPredicate(); | |||
1589 | ||||
1590 | if (isa<PredicateAssume>(PI)) { | |||
1591 | // If we assume the operands are equal, then they are equal. | |||
1592 | if (Predicate == CmpInst::ICMP_EQ) { | |||
1593 | addPredicateUsers(PI, I); | |||
1594 | addAdditionalUsers(SwappedOps ? Cmp->getOperand(1) : Cmp->getOperand(0), | |||
1595 | I); | |||
1596 | return createVariableOrConstant(FirstOp); | |||
1597 | } | |||
1598 | } | |||
1599 | if (const auto *PBranch = dyn_cast<PredicateBranch>(PI)) { | |||
1600 | // If we are *not* a copy of the comparison, we may equal to the other | |||
1601 | // operand when the predicate implies something about equality of | |||
1602 | // operations. In particular, if the comparison is true/false when the | |||
1603 | // operands are equal, and we are on the right edge, we know this operation | |||
1604 | // is equal to something. | |||
1605 | if ((PBranch->TrueEdge && Predicate == CmpInst::ICMP_EQ) || | |||
1606 | (!PBranch->TrueEdge && Predicate == CmpInst::ICMP_NE)) { | |||
1607 | addPredicateUsers(PI, I); | |||
1608 | addAdditionalUsers(SwappedOps ? Cmp->getOperand(1) : Cmp->getOperand(0), | |||
1609 | I); | |||
1610 | return createVariableOrConstant(FirstOp); | |||
1611 | } | |||
1612 | // Handle the special case of floating point. | |||
1613 | if (((PBranch->TrueEdge && Predicate == CmpInst::FCMP_OEQ) || | |||
1614 | (!PBranch->TrueEdge && Predicate == CmpInst::FCMP_UNE)) && | |||
1615 | isa<ConstantFP>(FirstOp) && !cast<ConstantFP>(FirstOp)->isZero()) { | |||
1616 | addPredicateUsers(PI, I); | |||
1617 | addAdditionalUsers(SwappedOps ? Cmp->getOperand(1) : Cmp->getOperand(0), | |||
1618 | I); | |||
1619 | return createConstantExpression(cast<Constant>(FirstOp)); | |||
1620 | } | |||
1621 | } | |||
1622 | return nullptr; | |||
1623 | } | |||
1624 | ||||
1625 | // Evaluate read only and pure calls, and create an expression result. | |||
1626 | const Expression *NewGVN::performSymbolicCallEvaluation(Instruction *I) const { | |||
1627 | auto *CI = cast<CallInst>(I); | |||
1628 | if (auto *II = dyn_cast<IntrinsicInst>(I)) { | |||
1629 | // Intrinsics with the returned attribute are copies of arguments. | |||
1630 | if (auto *ReturnedValue = II->getReturnedArgOperand()) { | |||
1631 | if (II->getIntrinsicID() == Intrinsic::ssa_copy) | |||
1632 | if (const auto *Result = performSymbolicPredicateInfoEvaluation(I)) | |||
1633 | return Result; | |||
1634 | return createVariableOrConstant(ReturnedValue); | |||
1635 | } | |||
1636 | } | |||
1637 | if (AA->doesNotAccessMemory(CI)) { | |||
1638 | return createCallExpression(CI, TOPClass->getMemoryLeader()); | |||
1639 | } else if (AA->onlyReadsMemory(CI)) { | |||
1640 | MemoryAccess *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(CI); | |||
1641 | return createCallExpression(CI, DefiningAccess); | |||
1642 | } | |||
1643 | return nullptr; | |||
1644 | } | |||
1645 | ||||
1646 | // Retrieve the memory class for a given MemoryAccess. | |||
1647 | CongruenceClass *NewGVN::getMemoryClass(const MemoryAccess *MA) const { | |||
1648 | auto *Result = MemoryAccessToClass.lookup(MA); | |||
1649 | assert(Result && "Should have found memory class")((Result && "Should have found memory class") ? static_cast <void> (0) : __assert_fail ("Result && \"Should have found memory class\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1649, __PRETTY_FUNCTION__)); | |||
1650 | return Result; | |||
1651 | } | |||
1652 | ||||
1653 | // Update the MemoryAccess equivalence table to say that From is equal to To, | |||
1654 | // and return true if this is different from what already existed in the table. | |||
1655 | bool NewGVN::setMemoryClass(const MemoryAccess *From, | |||
1656 | CongruenceClass *NewClass) { | |||
1657 | assert(NewClass &&((NewClass && "Every MemoryAccess should be getting mapped to a non-null class" ) ? static_cast<void> (0) : __assert_fail ("NewClass && \"Every MemoryAccess should be getting mapped to a non-null class\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1658, __PRETTY_FUNCTION__)) | |||
1658 | "Every MemoryAccess should be getting mapped to a non-null class")((NewClass && "Every MemoryAccess should be getting mapped to a non-null class" ) ? static_cast<void> (0) : __assert_fail ("NewClass && \"Every MemoryAccess should be getting mapped to a non-null class\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1658, __PRETTY_FUNCTION__)); | |||
1659 | LLVM_DEBUG(dbgs() << "Setting " << *From)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Setting " << *From; } } while (false); | |||
1660 | LLVM_DEBUG(dbgs() << " equivalent to congruence class ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << " equivalent to congruence class " ; } } while (false); | |||
1661 | LLVM_DEBUG(dbgs() << NewClass->getID()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << NewClass->getID() << " with current MemoryAccess leader " ; } } while (false) | |||
1662 | << " with current MemoryAccess leader ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << NewClass->getID() << " with current MemoryAccess leader " ; } } while (false); | |||
1663 | LLVM_DEBUG(dbgs() << *NewClass->getMemoryLeader() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << *NewClass->getMemoryLeader() << "\n"; } } while (false); | |||
1664 | ||||
1665 | auto LookupResult = MemoryAccessToClass.find(From); | |||
1666 | bool Changed = false; | |||
1667 | // If it's already in the table, see if the value changed. | |||
1668 | if (LookupResult != MemoryAccessToClass.end()) { | |||
1669 | auto *OldClass = LookupResult->second; | |||
1670 | if (OldClass != NewClass) { | |||
1671 | // If this is a phi, we have to handle memory member updates. | |||
1672 | if (auto *MP = dyn_cast<MemoryPhi>(From)) { | |||
1673 | OldClass->memory_erase(MP); | |||
1674 | NewClass->memory_insert(MP); | |||
1675 | // This may have killed the class if it had no non-memory members | |||
1676 | if (OldClass->getMemoryLeader() == From) { | |||
1677 | if (OldClass->definesNoMemory()) { | |||
1678 | OldClass->setMemoryLeader(nullptr); | |||
1679 | } else { | |||
1680 | OldClass->setMemoryLeader(getNextMemoryLeader(OldClass)); | |||
1681 | LLVM_DEBUG(dbgs() << "Memory class leader change for class "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << OldClass->getID() << " to " << *OldClass ->getMemoryLeader() << " due to removal of a memory member " << *From << "\n"; } } while (false) | |||
1682 | << OldClass->getID() << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << OldClass->getID() << " to " << *OldClass ->getMemoryLeader() << " due to removal of a memory member " << *From << "\n"; } } while (false) | |||
1683 | << *OldClass->getMemoryLeader()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << OldClass->getID() << " to " << *OldClass ->getMemoryLeader() << " due to removal of a memory member " << *From << "\n"; } } while (false) | |||
1684 | << " due to removal of a memory member " << *Fromdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << OldClass->getID() << " to " << *OldClass ->getMemoryLeader() << " due to removal of a memory member " << *From << "\n"; } } while (false) | |||
1685 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << OldClass->getID() << " to " << *OldClass ->getMemoryLeader() << " due to removal of a memory member " << *From << "\n"; } } while (false); | |||
1686 | markMemoryLeaderChangeTouched(OldClass); | |||
1687 | } | |||
1688 | } | |||
1689 | } | |||
1690 | // It wasn't equivalent before, and now it is. | |||
1691 | LookupResult->second = NewClass; | |||
1692 | Changed = true; | |||
1693 | } | |||
1694 | } | |||
1695 | ||||
1696 | return Changed; | |||
1697 | } | |||
1698 | ||||
1699 | // Determine if a instruction is cycle-free. That means the values in the | |||
1700 | // instruction don't depend on any expressions that can change value as a result | |||
1701 | // of the instruction. For example, a non-cycle free instruction would be v = | |||
1702 | // phi(0, v+1). | |||
1703 | bool NewGVN::isCycleFree(const Instruction *I) const { | |||
1704 | // In order to compute cycle-freeness, we do SCC finding on the instruction, | |||
1705 | // and see what kind of SCC it ends up in. If it is a singleton, it is | |||
1706 | // cycle-free. If it is not in a singleton, it is only cycle free if the | |||
1707 | // other members are all phi nodes (as they do not compute anything, they are | |||
1708 | // copies). | |||
1709 | auto ICS = InstCycleState.lookup(I); | |||
1710 | if (ICS == ICS_Unknown) { | |||
1711 | SCCFinder.Start(I); | |||
1712 | auto &SCC = SCCFinder.getComponentFor(I); | |||
1713 | // It's cycle free if it's size 1 or the SCC is *only* phi nodes. | |||
1714 | if (SCC.size() == 1) | |||
1715 | InstCycleState.insert({I, ICS_CycleFree}); | |||
1716 | else { | |||
1717 | bool AllPhis = llvm::all_of(SCC, [](const Value *V) { | |||
1718 | return isa<PHINode>(V) || isCopyOfAPHI(V); | |||
1719 | }); | |||
1720 | ICS = AllPhis ? ICS_CycleFree : ICS_Cycle; | |||
1721 | for (auto *Member : SCC) | |||
1722 | if (auto *MemberPhi = dyn_cast<PHINode>(Member)) | |||
1723 | InstCycleState.insert({MemberPhi, ICS}); | |||
1724 | } | |||
1725 | } | |||
1726 | if (ICS == ICS_Cycle) | |||
1727 | return false; | |||
1728 | return true; | |||
1729 | } | |||
1730 | ||||
1731 | // Evaluate PHI nodes symbolically and create an expression result. | |||
1732 | const Expression * | |||
1733 | NewGVN::performSymbolicPHIEvaluation(ArrayRef<ValPair> PHIOps, | |||
1734 | Instruction *I, | |||
1735 | BasicBlock *PHIBlock) const { | |||
1736 | // True if one of the incoming phi edges is a backedge. | |||
1737 | bool HasBackedge = false; | |||
1738 | // All constant tracks the state of whether all the *original* phi operands | |||
1739 | // This is really shorthand for "this phi cannot cycle due to forward | |||
1740 | // change in value of the phi is guaranteed not to later change the value of | |||
1741 | // the phi. IE it can't be v = phi(undef, v+1) | |||
1742 | bool OriginalOpsConstant = true; | |||
1743 | auto *E = cast<PHIExpression>(createPHIExpression( | |||
1744 | PHIOps, I, PHIBlock, HasBackedge, OriginalOpsConstant)); | |||
1745 | // We match the semantics of SimplifyPhiNode from InstructionSimplify here. | |||
1746 | // See if all arguments are the same. | |||
1747 | // We track if any were undef because they need special handling. | |||
1748 | bool HasUndef = false; | |||
1749 | auto Filtered = make_filter_range(E->operands(), [&](Value *Arg) { | |||
1750 | if (isa<UndefValue>(Arg)) { | |||
1751 | HasUndef = true; | |||
1752 | return false; | |||
1753 | } | |||
1754 | return true; | |||
1755 | }); | |||
1756 | // If we are left with no operands, it's dead. | |||
1757 | if (empty(Filtered)) { | |||
1758 | // If it has undef at this point, it means there are no-non-undef arguments, | |||
1759 | // and thus, the value of the phi node must be undef. | |||
1760 | if (HasUndef) { | |||
1761 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "PHI Node " << *I << " has no non-undef arguments, valuing it as undef\n"; } } while (false) | |||
1762 | dbgs() << "PHI Node " << *Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "PHI Node " << *I << " has no non-undef arguments, valuing it as undef\n"; } } while (false) | |||
1763 | << " has no non-undef arguments, valuing it as undef\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "PHI Node " << *I << " has no non-undef arguments, valuing it as undef\n"; } } while (false); | |||
1764 | return createConstantExpression(UndefValue::get(I->getType())); | |||
1765 | } | |||
1766 | ||||
1767 | LLVM_DEBUG(dbgs() << "No arguments of PHI node " << *I << " are live\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "No arguments of PHI node " << *I << " are live\n"; } } while (false); | |||
1768 | deleteExpression(E); | |||
1769 | return createDeadExpression(); | |||
1770 | } | |||
1771 | Value *AllSameValue = *(Filtered.begin()); | |||
1772 | ++Filtered.begin(); | |||
1773 | // Can't use std::equal here, sadly, because filter.begin moves. | |||
1774 | if (llvm::all_of(Filtered, [&](Value *Arg) { return Arg == AllSameValue; })) { | |||
1775 | // In LLVM's non-standard representation of phi nodes, it's possible to have | |||
1776 | // phi nodes with cycles (IE dependent on other phis that are .... dependent | |||
1777 | // on the original phi node), especially in weird CFG's where some arguments | |||
1778 | // are unreachable, or uninitialized along certain paths. This can cause | |||
1779 | // infinite loops during evaluation. We work around this by not trying to | |||
1780 | // really evaluate them independently, but instead using a variable | |||
1781 | // expression to say if one is equivalent to the other. | |||
1782 | // We also special case undef, so that if we have an undef, we can't use the | |||
1783 | // common value unless it dominates the phi block. | |||
1784 | if (HasUndef) { | |||
1785 | // If we have undef and at least one other value, this is really a | |||
1786 | // multivalued phi, and we need to know if it's cycle free in order to | |||
1787 | // evaluate whether we can ignore the undef. The other parts of this are | |||
1788 | // just shortcuts. If there is no backedge, or all operands are | |||
1789 | // constants, it also must be cycle free. | |||
1790 | if (HasBackedge && !OriginalOpsConstant && | |||
1791 | !isa<UndefValue>(AllSameValue) && !isCycleFree(I)) | |||
1792 | return E; | |||
1793 | ||||
1794 | // Only have to check for instructions | |||
1795 | if (auto *AllSameInst = dyn_cast<Instruction>(AllSameValue)) | |||
1796 | if (!someEquivalentDominates(AllSameInst, I)) | |||
1797 | return E; | |||
1798 | } | |||
1799 | // Can't simplify to something that comes later in the iteration. | |||
1800 | // Otherwise, when and if it changes congruence class, we will never catch | |||
1801 | // up. We will always be a class behind it. | |||
1802 | if (isa<Instruction>(AllSameValue) && | |||
1803 | InstrToDFSNum(AllSameValue) > InstrToDFSNum(I)) | |||
1804 | return E; | |||
1805 | NumGVNPhisAllSame++; | |||
1806 | LLVM_DEBUG(dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValuedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue << "\n"; } } while (false) | |||
1807 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue << "\n"; } } while (false); | |||
1808 | deleteExpression(E); | |||
1809 | return createVariableOrConstant(AllSameValue); | |||
1810 | } | |||
1811 | return E; | |||
1812 | } | |||
1813 | ||||
1814 | const Expression * | |||
1815 | NewGVN::performSymbolicAggrValueEvaluation(Instruction *I) const { | |||
1816 | if (auto *EI = dyn_cast<ExtractValueInst>(I)) { | |||
1817 | auto *WO = dyn_cast<WithOverflowInst>(EI->getAggregateOperand()); | |||
1818 | if (WO && EI->getNumIndices() == 1 && *EI->idx_begin() == 0) | |||
1819 | // EI is an extract from one of our with.overflow intrinsics. Synthesize | |||
1820 | // a semantically equivalent expression instead of an extract value | |||
1821 | // expression. | |||
1822 | return createBinaryExpression(WO->getBinaryOp(), EI->getType(), | |||
1823 | WO->getLHS(), WO->getRHS(), I); | |||
1824 | } | |||
1825 | ||||
1826 | return createAggregateValueExpression(I); | |||
1827 | } | |||
1828 | ||||
1829 | const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) const { | |||
1830 | assert(isa<CmpInst>(I) && "Expected a cmp instruction.")((isa<CmpInst>(I) && "Expected a cmp instruction." ) ? static_cast<void> (0) : __assert_fail ("isa<CmpInst>(I) && \"Expected a cmp instruction.\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 1830, __PRETTY_FUNCTION__)); | |||
1831 | ||||
1832 | auto *CI = cast<CmpInst>(I); | |||
1833 | // See if our operands are equal to those of a previous predicate, and if so, | |||
1834 | // if it implies true or false. | |||
1835 | auto Op0 = lookupOperandLeader(CI->getOperand(0)); | |||
1836 | auto Op1 = lookupOperandLeader(CI->getOperand(1)); | |||
1837 | auto OurPredicate = CI->getPredicate(); | |||
1838 | if (shouldSwapOperands(Op0, Op1)) { | |||
1839 | std::swap(Op0, Op1); | |||
1840 | OurPredicate = CI->getSwappedPredicate(); | |||
1841 | } | |||
1842 | ||||
1843 | // Avoid processing the same info twice. | |||
1844 | const PredicateBase *LastPredInfo = nullptr; | |||
1845 | // See if we know something about the comparison itself, like it is the target | |||
1846 | // of an assume. | |||
1847 | auto *CmpPI = PredInfo->getPredicateInfoFor(I); | |||
1848 | if (dyn_cast_or_null<PredicateAssume>(CmpPI)) | |||
1849 | return createConstantExpression(ConstantInt::getTrue(CI->getType())); | |||
1850 | ||||
1851 | if (Op0 == Op1) { | |||
1852 | // This condition does not depend on predicates, no need to add users | |||
1853 | if (CI->isTrueWhenEqual()) | |||
1854 | return createConstantExpression(ConstantInt::getTrue(CI->getType())); | |||
1855 | else if (CI->isFalseWhenEqual()) | |||
1856 | return createConstantExpression(ConstantInt::getFalse(CI->getType())); | |||
1857 | } | |||
1858 | ||||
1859 | // NOTE: Because we are comparing both operands here and below, and using | |||
1860 | // previous comparisons, we rely on fact that predicateinfo knows to mark | |||
1861 | // comparisons that use renamed operands as users of the earlier comparisons. | |||
1862 | // It is *not* enough to just mark predicateinfo renamed operands as users of | |||
1863 | // the earlier comparisons, because the *other* operand may have changed in a | |||
1864 | // previous iteration. | |||
1865 | // Example: | |||
1866 | // icmp slt %a, %b | |||
1867 | // %b.0 = ssa.copy(%b) | |||
1868 | // false branch: | |||
1869 | // icmp slt %c, %b.0 | |||
1870 | ||||
1871 | // %c and %a may start out equal, and thus, the code below will say the second | |||
1872 | // %icmp is false. c may become equal to something else, and in that case the | |||
1873 | // %second icmp *must* be reexamined, but would not if only the renamed | |||
1874 | // %operands are considered users of the icmp. | |||
1875 | ||||
1876 | // *Currently* we only check one level of comparisons back, and only mark one | |||
1877 | // level back as touched when changes happen. If you modify this code to look | |||
1878 | // back farther through comparisons, you *must* mark the appropriate | |||
1879 | // comparisons as users in PredicateInfo.cpp, or you will cause bugs. See if | |||
1880 | // we know something just from the operands themselves | |||
1881 | ||||
1882 | // See if our operands have predicate info, so that we may be able to derive | |||
1883 | // something from a previous comparison. | |||
1884 | for (const auto &Op : CI->operands()) { | |||
1885 | auto *PI = PredInfo->getPredicateInfoFor(Op); | |||
1886 | if (const auto *PBranch = dyn_cast_or_null<PredicateBranch>(PI)) { | |||
1887 | if (PI == LastPredInfo) | |||
1888 | continue; | |||
1889 | LastPredInfo = PI; | |||
1890 | // In phi of ops cases, we may have predicate info that we are evaluating | |||
1891 | // in a different context. | |||
1892 | if (!DT->dominates(PBranch->To, getBlockForValue(I))) | |||
1893 | continue; | |||
1894 | // TODO: Along the false edge, we may know more things too, like | |||
1895 | // icmp of | |||
1896 | // same operands is false. | |||
1897 | // TODO: We only handle actual comparison conditions below, not | |||
1898 | // and/or. | |||
1899 | auto *BranchCond = dyn_cast<CmpInst>(PBranch->Condition); | |||
1900 | if (!BranchCond) | |||
1901 | continue; | |||
1902 | auto *BranchOp0 = lookupOperandLeader(BranchCond->getOperand(0)); | |||
1903 | auto *BranchOp1 = lookupOperandLeader(BranchCond->getOperand(1)); | |||
1904 | auto BranchPredicate = BranchCond->getPredicate(); | |||
1905 | if (shouldSwapOperands(BranchOp0, BranchOp1)) { | |||
1906 | std::swap(BranchOp0, BranchOp1); | |||
1907 | BranchPredicate = BranchCond->getSwappedPredicate(); | |||
1908 | } | |||
1909 | if (BranchOp0 == Op0 && BranchOp1 == Op1) { | |||
1910 | if (PBranch->TrueEdge) { | |||
1911 | // If we know the previous predicate is true and we are in the true | |||
1912 | // edge then we may be implied true or false. | |||
1913 | if (CmpInst::isImpliedTrueByMatchingCmp(BranchPredicate, | |||
1914 | OurPredicate)) { | |||
1915 | addPredicateUsers(PI, I); | |||
1916 | return createConstantExpression( | |||
1917 | ConstantInt::getTrue(CI->getType())); | |||
1918 | } | |||
1919 | ||||
1920 | if (CmpInst::isImpliedFalseByMatchingCmp(BranchPredicate, | |||
1921 | OurPredicate)) { | |||
1922 | addPredicateUsers(PI, I); | |||
1923 | return createConstantExpression( | |||
1924 | ConstantInt::getFalse(CI->getType())); | |||
1925 | } | |||
1926 | } else { | |||
1927 | // Just handle the ne and eq cases, where if we have the same | |||
1928 | // operands, we may know something. | |||
1929 | if (BranchPredicate == OurPredicate) { | |||
1930 | addPredicateUsers(PI, I); | |||
1931 | // Same predicate, same ops,we know it was false, so this is false. | |||
1932 | return createConstantExpression( | |||
1933 | ConstantInt::getFalse(CI->getType())); | |||
1934 | } else if (BranchPredicate == | |||
1935 | CmpInst::getInversePredicate(OurPredicate)) { | |||
1936 | addPredicateUsers(PI, I); | |||
1937 | // Inverse predicate, we know the other was false, so this is true. | |||
1938 | return createConstantExpression( | |||
1939 | ConstantInt::getTrue(CI->getType())); | |||
1940 | } | |||
1941 | } | |||
1942 | } | |||
1943 | } | |||
1944 | } | |||
1945 | // Create expression will take care of simplifyCmpInst | |||
1946 | return createExpression(I); | |||
1947 | } | |||
1948 | ||||
1949 | // Substitute and symbolize the value before value numbering. | |||
1950 | const Expression * | |||
1951 | NewGVN::performSymbolicEvaluation(Value *V, | |||
1952 | SmallPtrSetImpl<Value *> &Visited) const { | |||
1953 | const Expression *E = nullptr; | |||
1954 | if (auto *C = dyn_cast<Constant>(V)) | |||
1955 | E = createConstantExpression(C); | |||
1956 | else if (isa<Argument>(V) || isa<GlobalVariable>(V)) { | |||
1957 | E = createVariableExpression(V); | |||
1958 | } else { | |||
1959 | // TODO: memory intrinsics. | |||
1960 | // TODO: Some day, we should do the forward propagation and reassociation | |||
1961 | // parts of the algorithm. | |||
1962 | auto *I = cast<Instruction>(V); | |||
1963 | switch (I->getOpcode()) { | |||
1964 | case Instruction::ExtractValue: | |||
1965 | case Instruction::InsertValue: | |||
1966 | E = performSymbolicAggrValueEvaluation(I); | |||
1967 | break; | |||
1968 | case Instruction::PHI: { | |||
1969 | SmallVector<ValPair, 3> Ops; | |||
1970 | auto *PN = cast<PHINode>(I); | |||
1971 | for (unsigned i = 0; i < PN->getNumOperands(); ++i) | |||
1972 | Ops.push_back({PN->getIncomingValue(i), PN->getIncomingBlock(i)}); | |||
1973 | // Sort to ensure the invariant createPHIExpression requires is met. | |||
1974 | sortPHIOps(Ops); | |||
1975 | E = performSymbolicPHIEvaluation(Ops, I, getBlockForValue(I)); | |||
1976 | } break; | |||
1977 | case Instruction::Call: | |||
1978 | E = performSymbolicCallEvaluation(I); | |||
1979 | break; | |||
1980 | case Instruction::Store: | |||
1981 | E = performSymbolicStoreEvaluation(I); | |||
1982 | break; | |||
1983 | case Instruction::Load: | |||
1984 | E = performSymbolicLoadEvaluation(I); | |||
1985 | break; | |||
1986 | case Instruction::BitCast: | |||
1987 | E = createExpression(I); | |||
1988 | break; | |||
1989 | case Instruction::ICmp: | |||
1990 | case Instruction::FCmp: | |||
1991 | E = performSymbolicCmpEvaluation(I); | |||
1992 | break; | |||
1993 | case Instruction::Add: | |||
1994 | case Instruction::FAdd: | |||
1995 | case Instruction::Sub: | |||
1996 | case Instruction::FSub: | |||
1997 | case Instruction::Mul: | |||
1998 | case Instruction::FMul: | |||
1999 | case Instruction::UDiv: | |||
2000 | case Instruction::SDiv: | |||
2001 | case Instruction::FDiv: | |||
2002 | case Instruction::URem: | |||
2003 | case Instruction::SRem: | |||
2004 | case Instruction::FRem: | |||
2005 | case Instruction::Shl: | |||
2006 | case Instruction::LShr: | |||
2007 | case Instruction::AShr: | |||
2008 | case Instruction::And: | |||
2009 | case Instruction::Or: | |||
2010 | case Instruction::Xor: | |||
2011 | case Instruction::Trunc: | |||
2012 | case Instruction::ZExt: | |||
2013 | case Instruction::SExt: | |||
2014 | case Instruction::FPToUI: | |||
2015 | case Instruction::FPToSI: | |||
2016 | case Instruction::UIToFP: | |||
2017 | case Instruction::SIToFP: | |||
2018 | case Instruction::FPTrunc: | |||
2019 | case Instruction::FPExt: | |||
2020 | case Instruction::PtrToInt: | |||
2021 | case Instruction::IntToPtr: | |||
2022 | case Instruction::Select: | |||
2023 | case Instruction::ExtractElement: | |||
2024 | case Instruction::InsertElement: | |||
2025 | case Instruction::ShuffleVector: | |||
2026 | case Instruction::GetElementPtr: | |||
2027 | E = createExpression(I); | |||
2028 | break; | |||
2029 | default: | |||
2030 | return nullptr; | |||
2031 | } | |||
2032 | } | |||
2033 | return E; | |||
2034 | } | |||
2035 | ||||
2036 | // Look up a container in a map, and then call a function for each thing in the | |||
2037 | // found container. | |||
2038 | template <typename Map, typename KeyType, typename Func> | |||
2039 | void NewGVN::for_each_found(Map &M, const KeyType &Key, Func F) { | |||
2040 | const auto Result = M.find_as(Key); | |||
2041 | if (Result != M.end()) | |||
2042 | for (typename Map::mapped_type::value_type Mapped : Result->second) | |||
2043 | F(Mapped); | |||
2044 | } | |||
2045 | ||||
2046 | // Look up a container of values/instructions in a map, and touch all the | |||
2047 | // instructions in the container. Then erase value from the map. | |||
2048 | template <typename Map, typename KeyType> | |||
2049 | void NewGVN::touchAndErase(Map &M, const KeyType &Key) { | |||
2050 | const auto Result = M.find_as(Key); | |||
2051 | if (Result != M.end()) { | |||
2052 | for (const typename Map::mapped_type::value_type Mapped : Result->second) | |||
2053 | TouchedInstructions.set(InstrToDFSNum(Mapped)); | |||
2054 | M.erase(Result); | |||
2055 | } | |||
2056 | } | |||
2057 | ||||
2058 | void NewGVN::addAdditionalUsers(Value *To, Value *User) const { | |||
2059 | assert(User && To != User)((User && To != User) ? static_cast<void> (0) : __assert_fail ("User && To != User", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2059, __PRETTY_FUNCTION__)); | |||
2060 | if (isa<Instruction>(To)) | |||
2061 | AdditionalUsers[To].insert(User); | |||
2062 | } | |||
2063 | ||||
2064 | void NewGVN::markUsersTouched(Value *V) { | |||
2065 | // Now mark the users as touched. | |||
2066 | for (auto *User : V->users()) { | |||
2067 | assert(isa<Instruction>(User) && "Use of value not within an instruction?")((isa<Instruction>(User) && "Use of value not within an instruction?" ) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(User) && \"Use of value not within an instruction?\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2067, __PRETTY_FUNCTION__)); | |||
2068 | TouchedInstructions.set(InstrToDFSNum(User)); | |||
2069 | } | |||
2070 | touchAndErase(AdditionalUsers, V); | |||
2071 | } | |||
2072 | ||||
2073 | void NewGVN::addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const { | |||
2074 | LLVM_DEBUG(dbgs() << "Adding memory user " << *U << " to " << *To << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Adding memory user " << * U << " to " << *To << "\n"; } } while (false ); | |||
2075 | MemoryToUsers[To].insert(U); | |||
2076 | } | |||
2077 | ||||
2078 | void NewGVN::markMemoryDefTouched(const MemoryAccess *MA) { | |||
2079 | TouchedInstructions.set(MemoryToDFSNum(MA)); | |||
2080 | } | |||
2081 | ||||
2082 | void NewGVN::markMemoryUsersTouched(const MemoryAccess *MA) { | |||
2083 | if (isa<MemoryUse>(MA)) | |||
2084 | return; | |||
2085 | for (auto U : MA->users()) | |||
2086 | TouchedInstructions.set(MemoryToDFSNum(U)); | |||
2087 | touchAndErase(MemoryToUsers, MA); | |||
2088 | } | |||
2089 | ||||
2090 | // Add I to the set of users of a given predicate. | |||
2091 | void NewGVN::addPredicateUsers(const PredicateBase *PB, Instruction *I) const { | |||
2092 | // Don't add temporary instructions to the user lists. | |||
2093 | if (AllTempInstructions.count(I)) | |||
2094 | return; | |||
2095 | ||||
2096 | if (auto *PBranch = dyn_cast<PredicateBranch>(PB)) | |||
2097 | PredicateToUsers[PBranch->Condition].insert(I); | |||
2098 | else if (auto *PAssume = dyn_cast<PredicateBranch>(PB)) | |||
2099 | PredicateToUsers[PAssume->Condition].insert(I); | |||
2100 | } | |||
2101 | ||||
2102 | // Touch all the predicates that depend on this instruction. | |||
2103 | void NewGVN::markPredicateUsersTouched(Instruction *I) { | |||
2104 | touchAndErase(PredicateToUsers, I); | |||
2105 | } | |||
2106 | ||||
2107 | // Mark users affected by a memory leader change. | |||
2108 | void NewGVN::markMemoryLeaderChangeTouched(CongruenceClass *CC) { | |||
2109 | for (auto M : CC->memory()) | |||
2110 | markMemoryDefTouched(M); | |||
2111 | } | |||
2112 | ||||
2113 | // Touch the instructions that need to be updated after a congruence class has a | |||
2114 | // leader change, and mark changed values. | |||
2115 | void NewGVN::markValueLeaderChangeTouched(CongruenceClass *CC) { | |||
2116 | for (auto M : *CC) { | |||
2117 | if (auto *I = dyn_cast<Instruction>(M)) | |||
2118 | TouchedInstructions.set(InstrToDFSNum(I)); | |||
2119 | LeaderChanges.insert(M); | |||
2120 | } | |||
2121 | } | |||
2122 | ||||
2123 | // Give a range of things that have instruction DFS numbers, this will return | |||
2124 | // the member of the range with the smallest dfs number. | |||
2125 | template <class T, class Range> | |||
2126 | T *NewGVN::getMinDFSOfRange(const Range &R) const { | |||
2127 | std::pair<T *, unsigned> MinDFS = {nullptr, ~0U}; | |||
2128 | for (const auto X : R) { | |||
2129 | auto DFSNum = InstrToDFSNum(X); | |||
2130 | if (DFSNum < MinDFS.second) | |||
2131 | MinDFS = {X, DFSNum}; | |||
2132 | } | |||
2133 | return MinDFS.first; | |||
2134 | } | |||
2135 | ||||
2136 | // This function returns the MemoryAccess that should be the next leader of | |||
2137 | // congruence class CC, under the assumption that the current leader is going to | |||
2138 | // disappear. | |||
2139 | const MemoryAccess *NewGVN::getNextMemoryLeader(CongruenceClass *CC) const { | |||
2140 | // TODO: If this ends up to slow, we can maintain a next memory leader like we | |||
2141 | // do for regular leaders. | |||
2142 | // Make sure there will be a leader to find. | |||
2143 | assert(!CC->definesNoMemory() && "Can't get next leader if there is none")((!CC->definesNoMemory() && "Can't get next leader if there is none" ) ? static_cast<void> (0) : __assert_fail ("!CC->definesNoMemory() && \"Can't get next leader if there is none\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2143, __PRETTY_FUNCTION__)); | |||
2144 | if (CC->getStoreCount() > 0) { | |||
2145 | if (auto *NL = dyn_cast_or_null<StoreInst>(CC->getNextLeader().first)) | |||
2146 | return getMemoryAccess(NL); | |||
2147 | // Find the store with the minimum DFS number. | |||
2148 | auto *V = getMinDFSOfRange<Value>(make_filter_range( | |||
2149 | *CC, [&](const Value *V) { return isa<StoreInst>(V); })); | |||
2150 | return getMemoryAccess(cast<StoreInst>(V)); | |||
2151 | } | |||
2152 | assert(CC->getStoreCount() == 0)((CC->getStoreCount() == 0) ? static_cast<void> (0) : __assert_fail ("CC->getStoreCount() == 0", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2152, __PRETTY_FUNCTION__)); | |||
2153 | ||||
2154 | // Given our assertion, hitting this part must mean | |||
2155 | // !OldClass->memory_empty() | |||
2156 | if (CC->memory_size() == 1) | |||
2157 | return *CC->memory_begin(); | |||
2158 | return getMinDFSOfRange<const MemoryPhi>(CC->memory()); | |||
2159 | } | |||
2160 | ||||
2161 | // This function returns the next value leader of a congruence class, under the | |||
2162 | // assumption that the current leader is going away. This should end up being | |||
2163 | // the next most dominating member. | |||
2164 | Value *NewGVN::getNextValueLeader(CongruenceClass *CC) const { | |||
2165 | // We don't need to sort members if there is only 1, and we don't care about | |||
2166 | // sorting the TOP class because everything either gets out of it or is | |||
2167 | // unreachable. | |||
2168 | ||||
2169 | if (CC->size() == 1 || CC == TOPClass) { | |||
2170 | return *(CC->begin()); | |||
2171 | } else if (CC->getNextLeader().first) { | |||
2172 | ++NumGVNAvoidedSortedLeaderChanges; | |||
2173 | return CC->getNextLeader().first; | |||
2174 | } else { | |||
2175 | ++NumGVNSortedLeaderChanges; | |||
2176 | // NOTE: If this ends up to slow, we can maintain a dual structure for | |||
2177 | // member testing/insertion, or keep things mostly sorted, and sort only | |||
2178 | // here, or use SparseBitVector or .... | |||
2179 | return getMinDFSOfRange<Value>(*CC); | |||
2180 | } | |||
2181 | } | |||
2182 | ||||
2183 | // Move a MemoryAccess, currently in OldClass, to NewClass, including updates to | |||
2184 | // the memory members, etc for the move. | |||
2185 | // | |||
2186 | // The invariants of this function are: | |||
2187 | // | |||
2188 | // - I must be moving to NewClass from OldClass | |||
2189 | // - The StoreCount of OldClass and NewClass is expected to have been updated | |||
2190 | // for I already if it is a store. | |||
2191 | // - The OldClass memory leader has not been updated yet if I was the leader. | |||
2192 | void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I, | |||
2193 | MemoryAccess *InstMA, | |||
2194 | CongruenceClass *OldClass, | |||
2195 | CongruenceClass *NewClass) { | |||
2196 | // If the leader is I, and we had a representative MemoryAccess, it should | |||
2197 | // be the MemoryAccess of OldClass. | |||
2198 | assert((!InstMA || !OldClass->getMemoryLeader() ||(((!InstMA || !OldClass->getMemoryLeader() || OldClass-> getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader ()) == MemoryAccessToClass.lookup(InstMA)) && "Representative MemoryAccess mismatch" ) ? static_cast<void> (0) : __assert_fail ("(!InstMA || !OldClass->getMemoryLeader() || OldClass->getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) == MemoryAccessToClass.lookup(InstMA)) && \"Representative MemoryAccess mismatch\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2202, __PRETTY_FUNCTION__)) | |||
2199 | OldClass->getLeader() != I ||(((!InstMA || !OldClass->getMemoryLeader() || OldClass-> getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader ()) == MemoryAccessToClass.lookup(InstMA)) && "Representative MemoryAccess mismatch" ) ? static_cast<void> (0) : __assert_fail ("(!InstMA || !OldClass->getMemoryLeader() || OldClass->getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) == MemoryAccessToClass.lookup(InstMA)) && \"Representative MemoryAccess mismatch\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2202, __PRETTY_FUNCTION__)) | |||
2200 | MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) ==(((!InstMA || !OldClass->getMemoryLeader() || OldClass-> getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader ()) == MemoryAccessToClass.lookup(InstMA)) && "Representative MemoryAccess mismatch" ) ? static_cast<void> (0) : __assert_fail ("(!InstMA || !OldClass->getMemoryLeader() || OldClass->getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) == MemoryAccessToClass.lookup(InstMA)) && \"Representative MemoryAccess mismatch\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2202, __PRETTY_FUNCTION__)) | |||
2201 | MemoryAccessToClass.lookup(InstMA)) &&(((!InstMA || !OldClass->getMemoryLeader() || OldClass-> getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader ()) == MemoryAccessToClass.lookup(InstMA)) && "Representative MemoryAccess mismatch" ) ? static_cast<void> (0) : __assert_fail ("(!InstMA || !OldClass->getMemoryLeader() || OldClass->getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) == MemoryAccessToClass.lookup(InstMA)) && \"Representative MemoryAccess mismatch\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2202, __PRETTY_FUNCTION__)) | |||
2202 | "Representative MemoryAccess mismatch")(((!InstMA || !OldClass->getMemoryLeader() || OldClass-> getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader ()) == MemoryAccessToClass.lookup(InstMA)) && "Representative MemoryAccess mismatch" ) ? static_cast<void> (0) : __assert_fail ("(!InstMA || !OldClass->getMemoryLeader() || OldClass->getLeader() != I || MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) == MemoryAccessToClass.lookup(InstMA)) && \"Representative MemoryAccess mismatch\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2202, __PRETTY_FUNCTION__)); | |||
2203 | // First, see what happens to the new class | |||
2204 | if (!NewClass->getMemoryLeader()) { | |||
2205 | // Should be a new class, or a store becoming a leader of a new class. | |||
2206 | assert(NewClass->size() == 1 ||((NewClass->size() == 1 || (isa<StoreInst>(I) && NewClass->getStoreCount() == 1)) ? static_cast<void> (0) : __assert_fail ("NewClass->size() == 1 || (isa<StoreInst>(I) && NewClass->getStoreCount() == 1)" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2207, __PRETTY_FUNCTION__)) | |||
2207 | (isa<StoreInst>(I) && NewClass->getStoreCount() == 1))((NewClass->size() == 1 || (isa<StoreInst>(I) && NewClass->getStoreCount() == 1)) ? static_cast<void> (0) : __assert_fail ("NewClass->size() == 1 || (isa<StoreInst>(I) && NewClass->getStoreCount() == 1)" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2207, __PRETTY_FUNCTION__)); | |||
2208 | NewClass->setMemoryLeader(InstMA); | |||
2209 | // Mark it touched if we didn't just create a singleton | |||
2210 | LLVM_DEBUG(dbgs() << "Memory class leader change for class "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << NewClass->getID() << " due to new memory instruction becoming leader\n" ; } } while (false) | |||
2211 | << NewClass->getID()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << NewClass->getID() << " due to new memory instruction becoming leader\n" ; } } while (false) | |||
2212 | << " due to new memory instruction becoming leader\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << NewClass->getID() << " due to new memory instruction becoming leader\n" ; } } while (false); | |||
2213 | markMemoryLeaderChangeTouched(NewClass); | |||
2214 | } | |||
2215 | setMemoryClass(InstMA, NewClass); | |||
2216 | // Now, fixup the old class if necessary | |||
2217 | if (OldClass->getMemoryLeader() == InstMA) { | |||
2218 | if (!OldClass->definesNoMemory()) { | |||
2219 | OldClass->setMemoryLeader(getNextMemoryLeader(OldClass)); | |||
2220 | LLVM_DEBUG(dbgs() << "Memory class leader change for class "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << OldClass->getID() << " to " << *OldClass ->getMemoryLeader() << " due to removal of old leader " << *InstMA << "\n"; } } while (false) | |||
2221 | << OldClass->getID() << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << OldClass->getID() << " to " << *OldClass ->getMemoryLeader() << " due to removal of old leader " << *InstMA << "\n"; } } while (false) | |||
2222 | << *OldClass->getMemoryLeader()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << OldClass->getID() << " to " << *OldClass ->getMemoryLeader() << " due to removal of old leader " << *InstMA << "\n"; } } while (false) | |||
2223 | << " due to removal of old leader " << *InstMA << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory class leader change for class " << OldClass->getID() << " to " << *OldClass ->getMemoryLeader() << " due to removal of old leader " << *InstMA << "\n"; } } while (false); | |||
2224 | markMemoryLeaderChangeTouched(OldClass); | |||
2225 | } else | |||
2226 | OldClass->setMemoryLeader(nullptr); | |||
2227 | } | |||
2228 | } | |||
2229 | ||||
2230 | // Move a value, currently in OldClass, to be part of NewClass | |||
2231 | // Update OldClass and NewClass for the move (including changing leaders, etc). | |||
2232 | void NewGVN::moveValueToNewCongruenceClass(Instruction *I, const Expression *E, | |||
2233 | CongruenceClass *OldClass, | |||
2234 | CongruenceClass *NewClass) { | |||
2235 | if (I == OldClass->getNextLeader().first) | |||
2236 | OldClass->resetNextLeader(); | |||
2237 | ||||
2238 | OldClass->erase(I); | |||
2239 | NewClass->insert(I); | |||
2240 | ||||
2241 | if (NewClass->getLeader() != I) | |||
2242 | NewClass->addPossibleNextLeader({I, InstrToDFSNum(I)}); | |||
2243 | // Handle our special casing of stores. | |||
2244 | if (auto *SI = dyn_cast<StoreInst>(I)) { | |||
2245 | OldClass->decStoreCount(); | |||
2246 | // Okay, so when do we want to make a store a leader of a class? | |||
2247 | // If we have a store defined by an earlier load, we want the earlier load | |||
2248 | // to lead the class. | |||
2249 | // If we have a store defined by something else, we want the store to lead | |||
2250 | // the class so everything else gets the "something else" as a value. | |||
2251 | // If we have a store as the single member of the class, we want the store | |||
2252 | // as the leader | |||
2253 | if (NewClass->getStoreCount() == 0 && !NewClass->getStoredValue()) { | |||
2254 | // If it's a store expression we are using, it means we are not equivalent | |||
2255 | // to something earlier. | |||
2256 | if (auto *SE = dyn_cast<StoreExpression>(E)) { | |||
2257 | NewClass->setStoredValue(SE->getStoredValue()); | |||
2258 | markValueLeaderChangeTouched(NewClass); | |||
2259 | // Shift the new class leader to be the store | |||
2260 | LLVM_DEBUG(dbgs() << "Changing leader of congruence class "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Changing leader of congruence class " << NewClass->getID() << " from " << *NewClass ->getLeader() << " to " << *SI << " because store joined class\n" ; } } while (false) | |||
2261 | << NewClass->getID() << " from "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Changing leader of congruence class " << NewClass->getID() << " from " << *NewClass ->getLeader() << " to " << *SI << " because store joined class\n" ; } } while (false) | |||
2262 | << *NewClass->getLeader() << " to " << *SIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Changing leader of congruence class " << NewClass->getID() << " from " << *NewClass ->getLeader() << " to " << *SI << " because store joined class\n" ; } } while (false) | |||
2263 | << " because store joined class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Changing leader of congruence class " << NewClass->getID() << " from " << *NewClass ->getLeader() << " to " << *SI << " because store joined class\n" ; } } while (false); | |||
2264 | // If we changed the leader, we have to mark it changed because we don't | |||
2265 | // know what it will do to symbolic evaluation. | |||
2266 | NewClass->setLeader(SI); | |||
2267 | } | |||
2268 | // We rely on the code below handling the MemoryAccess change. | |||
2269 | } | |||
2270 | NewClass->incStoreCount(); | |||
2271 | } | |||
2272 | // True if there is no memory instructions left in a class that had memory | |||
2273 | // instructions before. | |||
2274 | ||||
2275 | // If it's not a memory use, set the MemoryAccess equivalence | |||
2276 | auto *InstMA = dyn_cast_or_null<MemoryDef>(getMemoryAccess(I)); | |||
2277 | if (InstMA) | |||
2278 | moveMemoryToNewCongruenceClass(I, InstMA, OldClass, NewClass); | |||
2279 | ValueToClass[I] = NewClass; | |||
2280 | // See if we destroyed the class or need to swap leaders. | |||
2281 | if (OldClass->empty() && OldClass != TOPClass) { | |||
2282 | if (OldClass->getDefiningExpr()) { | |||
2283 | LLVM_DEBUG(dbgs() << "Erasing expression " << *OldClass->getDefiningExpr()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Erasing expression " << * OldClass->getDefiningExpr() << " from table\n"; } } while (false) | |||
2284 | << " from table\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Erasing expression " << * OldClass->getDefiningExpr() << " from table\n"; } } while (false); | |||
2285 | // We erase it as an exact expression to make sure we don't just erase an | |||
2286 | // equivalent one. | |||
2287 | auto Iter = ExpressionToClass.find_as( | |||
2288 | ExactEqualsExpression(*OldClass->getDefiningExpr())); | |||
2289 | if (Iter != ExpressionToClass.end()) | |||
2290 | ExpressionToClass.erase(Iter); | |||
2291 | #ifdef EXPENSIVE_CHECKS | |||
2292 | assert((((*OldClass->getDefiningExpr() != *E || ExpressionToClass .lookup(E)) && "We erased the expression we just inserted, which should not happen" ) ? static_cast<void> (0) : __assert_fail ("(*OldClass->getDefiningExpr() != *E || ExpressionToClass.lookup(E)) && \"We erased the expression we just inserted, which should not happen\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2294, __PRETTY_FUNCTION__)) | |||
2293 | (*OldClass->getDefiningExpr() != *E || ExpressionToClass.lookup(E)) &&(((*OldClass->getDefiningExpr() != *E || ExpressionToClass .lookup(E)) && "We erased the expression we just inserted, which should not happen" ) ? static_cast<void> (0) : __assert_fail ("(*OldClass->getDefiningExpr() != *E || ExpressionToClass.lookup(E)) && \"We erased the expression we just inserted, which should not happen\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2294, __PRETTY_FUNCTION__)) | |||
2294 | "We erased the expression we just inserted, which should not happen")(((*OldClass->getDefiningExpr() != *E || ExpressionToClass .lookup(E)) && "We erased the expression we just inserted, which should not happen" ) ? static_cast<void> (0) : __assert_fail ("(*OldClass->getDefiningExpr() != *E || ExpressionToClass.lookup(E)) && \"We erased the expression we just inserted, which should not happen\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2294, __PRETTY_FUNCTION__)); | |||
2295 | #endif | |||
2296 | } | |||
2297 | } else if (OldClass->getLeader() == I) { | |||
2298 | // When the leader changes, the value numbering of | |||
2299 | // everything may change due to symbolization changes, so we need to | |||
2300 | // reprocess. | |||
2301 | LLVM_DEBUG(dbgs() << "Value class leader change for class "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Value class leader change for class " << OldClass->getID() << "\n"; } } while (false ) | |||
2302 | << OldClass->getID() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Value class leader change for class " << OldClass->getID() << "\n"; } } while (false ); | |||
2303 | ++NumGVNLeaderChanges; | |||
2304 | // Destroy the stored value if there are no more stores to represent it. | |||
2305 | // Note that this is basically clean up for the expression removal that | |||
2306 | // happens below. If we remove stores from a class, we may leave it as a | |||
2307 | // class of equivalent memory phis. | |||
2308 | if (OldClass->getStoreCount() == 0) { | |||
2309 | if (OldClass->getStoredValue()) | |||
2310 | OldClass->setStoredValue(nullptr); | |||
2311 | } | |||
2312 | OldClass->setLeader(getNextValueLeader(OldClass)); | |||
2313 | OldClass->resetNextLeader(); | |||
2314 | markValueLeaderChangeTouched(OldClass); | |||
2315 | } | |||
2316 | } | |||
2317 | ||||
2318 | // For a given expression, mark the phi of ops instructions that could have | |||
2319 | // changed as a result. | |||
2320 | void NewGVN::markPhiOfOpsChanged(const Expression *E) { | |||
2321 | touchAndErase(ExpressionToPhiOfOps, E); | |||
2322 | } | |||
2323 | ||||
2324 | // Perform congruence finding on a given value numbering expression. | |||
2325 | void NewGVN::performCongruenceFinding(Instruction *I, const Expression *E) { | |||
2326 | // This is guaranteed to return something, since it will at least find | |||
2327 | // TOP. | |||
2328 | ||||
2329 | CongruenceClass *IClass = ValueToClass.lookup(I); | |||
2330 | assert(IClass && "Should have found a IClass")((IClass && "Should have found a IClass") ? static_cast <void> (0) : __assert_fail ("IClass && \"Should have found a IClass\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2330, __PRETTY_FUNCTION__)); | |||
2331 | // Dead classes should have been eliminated from the mapping. | |||
2332 | assert(!IClass->isDead() && "Found a dead class")((!IClass->isDead() && "Found a dead class") ? static_cast <void> (0) : __assert_fail ("!IClass->isDead() && \"Found a dead class\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2332, __PRETTY_FUNCTION__)); | |||
2333 | ||||
2334 | CongruenceClass *EClass = nullptr; | |||
2335 | if (const auto *VE = dyn_cast<VariableExpression>(E)) { | |||
2336 | EClass = ValueToClass.lookup(VE->getVariableValue()); | |||
2337 | } else if (isa<DeadExpression>(E)) { | |||
2338 | EClass = TOPClass; | |||
2339 | } | |||
2340 | if (!EClass) { | |||
2341 | auto lookupResult = ExpressionToClass.insert({E, nullptr}); | |||
2342 | ||||
2343 | // If it's not in the value table, create a new congruence class. | |||
2344 | if (lookupResult.second) { | |||
2345 | CongruenceClass *NewClass = createCongruenceClass(nullptr, E); | |||
2346 | auto place = lookupResult.first; | |||
2347 | place->second = NewClass; | |||
2348 | ||||
2349 | // Constants and variables should always be made the leader. | |||
2350 | if (const auto *CE = dyn_cast<ConstantExpression>(E)) { | |||
2351 | NewClass->setLeader(CE->getConstantValue()); | |||
2352 | } else if (const auto *SE = dyn_cast<StoreExpression>(E)) { | |||
2353 | StoreInst *SI = SE->getStoreInst(); | |||
2354 | NewClass->setLeader(SI); | |||
2355 | NewClass->setStoredValue(SE->getStoredValue()); | |||
2356 | // The RepMemoryAccess field will be filled in properly by the | |||
2357 | // moveValueToNewCongruenceClass call. | |||
2358 | } else { | |||
2359 | NewClass->setLeader(I); | |||
2360 | } | |||
2361 | assert(!isa<VariableExpression>(E) &&((!isa<VariableExpression>(E) && "VariableExpression should have been handled already" ) ? static_cast<void> (0) : __assert_fail ("!isa<VariableExpression>(E) && \"VariableExpression should have been handled already\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2362, __PRETTY_FUNCTION__)) | |||
2362 | "VariableExpression should have been handled already")((!isa<VariableExpression>(E) && "VariableExpression should have been handled already" ) ? static_cast<void> (0) : __assert_fail ("!isa<VariableExpression>(E) && \"VariableExpression should have been handled already\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2362, __PRETTY_FUNCTION__)); | |||
2363 | ||||
2364 | EClass = NewClass; | |||
2365 | LLVM_DEBUG(dbgs() << "Created new congruence class for " << *Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Created new congruence class for " << *I << " using expression " << *E << " at " << NewClass->getID() << " and leader " << *(NewClass->getLeader()); } } while (false) | |||
2366 | << " using expression " << *E << " at "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Created new congruence class for " << *I << " using expression " << *E << " at " << NewClass->getID() << " and leader " << *(NewClass->getLeader()); } } while (false) | |||
2367 | << NewClass->getID() << " and leader "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Created new congruence class for " << *I << " using expression " << *E << " at " << NewClass->getID() << " and leader " << *(NewClass->getLeader()); } } while (false) | |||
2368 | << *(NewClass->getLeader()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Created new congruence class for " << *I << " using expression " << *E << " at " << NewClass->getID() << " and leader " << *(NewClass->getLeader()); } } while (false); | |||
2369 | if (NewClass->getStoredValue()) | |||
2370 | LLVM_DEBUG(dbgs() << " and stored value "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << " and stored value " << * (NewClass->getStoredValue()); } } while (false) | |||
2371 | << *(NewClass->getStoredValue()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << " and stored value " << * (NewClass->getStoredValue()); } } while (false); | |||
2372 | LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "\n"; } } while (false); | |||
2373 | } else { | |||
2374 | EClass = lookupResult.first->second; | |||
2375 | if (isa<ConstantExpression>(E)) | |||
2376 | assert((isa<Constant>(EClass->getLeader()) ||(((isa<Constant>(EClass->getLeader()) || (EClass-> getStoredValue() && isa<Constant>(EClass->getStoredValue ()))) && "Any class with a constant expression should have a " "constant leader") ? static_cast<void> (0) : __assert_fail ("(isa<Constant>(EClass->getLeader()) || (EClass->getStoredValue() && isa<Constant>(EClass->getStoredValue()))) && \"Any class with a constant expression should have a \" \"constant leader\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2380, __PRETTY_FUNCTION__)) | |||
2377 | (EClass->getStoredValue() &&(((isa<Constant>(EClass->getLeader()) || (EClass-> getStoredValue() && isa<Constant>(EClass->getStoredValue ()))) && "Any class with a constant expression should have a " "constant leader") ? static_cast<void> (0) : __assert_fail ("(isa<Constant>(EClass->getLeader()) || (EClass->getStoredValue() && isa<Constant>(EClass->getStoredValue()))) && \"Any class with a constant expression should have a \" \"constant leader\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2380, __PRETTY_FUNCTION__)) | |||
2378 | isa<Constant>(EClass->getStoredValue()))) &&(((isa<Constant>(EClass->getLeader()) || (EClass-> getStoredValue() && isa<Constant>(EClass->getStoredValue ()))) && "Any class with a constant expression should have a " "constant leader") ? static_cast<void> (0) : __assert_fail ("(isa<Constant>(EClass->getLeader()) || (EClass->getStoredValue() && isa<Constant>(EClass->getStoredValue()))) && \"Any class with a constant expression should have a \" \"constant leader\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2380, __PRETTY_FUNCTION__)) | |||
2379 | "Any class with a constant expression should have a "(((isa<Constant>(EClass->getLeader()) || (EClass-> getStoredValue() && isa<Constant>(EClass->getStoredValue ()))) && "Any class with a constant expression should have a " "constant leader") ? static_cast<void> (0) : __assert_fail ("(isa<Constant>(EClass->getLeader()) || (EClass->getStoredValue() && isa<Constant>(EClass->getStoredValue()))) && \"Any class with a constant expression should have a \" \"constant leader\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2380, __PRETTY_FUNCTION__)) | |||
2380 | "constant leader")(((isa<Constant>(EClass->getLeader()) || (EClass-> getStoredValue() && isa<Constant>(EClass->getStoredValue ()))) && "Any class with a constant expression should have a " "constant leader") ? static_cast<void> (0) : __assert_fail ("(isa<Constant>(EClass->getLeader()) || (EClass->getStoredValue() && isa<Constant>(EClass->getStoredValue()))) && \"Any class with a constant expression should have a \" \"constant leader\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2380, __PRETTY_FUNCTION__)); | |||
2381 | ||||
2382 | assert(EClass && "Somehow don't have an eclass")((EClass && "Somehow don't have an eclass") ? static_cast <void> (0) : __assert_fail ("EClass && \"Somehow don't have an eclass\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2382, __PRETTY_FUNCTION__)); | |||
2383 | ||||
2384 | assert(!EClass->isDead() && "We accidentally looked up a dead class")((!EClass->isDead() && "We accidentally looked up a dead class" ) ? static_cast<void> (0) : __assert_fail ("!EClass->isDead() && \"We accidentally looked up a dead class\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 2384, __PRETTY_FUNCTION__)); | |||
2385 | } | |||
2386 | } | |||
2387 | bool ClassChanged = IClass != EClass; | |||
2388 | bool LeaderChanged = LeaderChanges.erase(I); | |||
2389 | if (ClassChanged || LeaderChanged) { | |||
2390 | LLVM_DEBUG(dbgs() << "New class " << EClass->getID() << " for expression "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "New class " << EClass-> getID() << " for expression " << *E << "\n" ; } } while (false) | |||
2391 | << *E << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "New class " << EClass-> getID() << " for expression " << *E << "\n" ; } } while (false); | |||
2392 | if (ClassChanged) { | |||
2393 | moveValueToNewCongruenceClass(I, E, IClass, EClass); | |||
2394 | markPhiOfOpsChanged(E); | |||
2395 | } | |||
2396 | ||||
2397 | markUsersTouched(I); | |||
2398 | if (MemoryAccess *MA = getMemoryAccess(I)) | |||
2399 | markMemoryUsersTouched(MA); | |||
2400 | if (auto *CI = dyn_cast<CmpInst>(I)) | |||
2401 | markPredicateUsersTouched(CI); | |||
2402 | } | |||
2403 | // If we changed the class of the store, we want to ensure nothing finds the | |||
2404 | // old store expression. In particular, loads do not compare against stored | |||
2405 | // value, so they will find old store expressions (and associated class | |||
2406 | // mappings) if we leave them in the table. | |||
2407 | if (ClassChanged && isa<StoreInst>(I)) { | |||
2408 | auto *OldE = ValueToExpression.lookup(I); | |||
2409 | // It could just be that the old class died. We don't want to erase it if we | |||
2410 | // just moved classes. | |||
2411 | if (OldE && isa<StoreExpression>(OldE) && *E != *OldE) { | |||
2412 | // Erase this as an exact expression to ensure we don't erase expressions | |||
2413 | // equivalent to it. | |||
2414 | auto Iter = ExpressionToClass.find_as(ExactEqualsExpression(*OldE)); | |||
2415 | if (Iter != ExpressionToClass.end()) | |||
2416 | ExpressionToClass.erase(Iter); | |||
2417 | } | |||
2418 | } | |||
2419 | ValueToExpression[I] = E; | |||
2420 | } | |||
2421 | ||||
2422 | // Process the fact that Edge (from, to) is reachable, including marking | |||
2423 | // any newly reachable blocks and instructions for processing. | |||
2424 | void NewGVN::updateReachableEdge(BasicBlock *From, BasicBlock *To) { | |||
2425 | // Check if the Edge was reachable before. | |||
2426 | if (ReachableEdges.insert({From, To}).second) { | |||
2427 | // If this block wasn't reachable before, all instructions are touched. | |||
2428 | if (ReachableBlocks.insert(To).second) { | |||
2429 | LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Block " << getBlockName( To) << " marked reachable\n"; } } while (false) | |||
2430 | << " marked reachable\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Block " << getBlockName( To) << " marked reachable\n"; } } while (false); | |||
2431 | const auto &InstRange = BlockInstRange.lookup(To); | |||
2432 | TouchedInstructions.set(InstRange.first, InstRange.second); | |||
2433 | } else { | |||
2434 | LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Block " << getBlockName( To) << " was reachable, but new edge {" << getBlockName (From) << "," << getBlockName(To) << "} to it found\n" ; } } while (false) | |||
2435 | << " was reachable, but new edge {"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Block " << getBlockName( To) << " was reachable, but new edge {" << getBlockName (From) << "," << getBlockName(To) << "} to it found\n" ; } } while (false) | |||
2436 | << getBlockName(From) << "," << getBlockName(To)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Block " << getBlockName( To) << " was reachable, but new edge {" << getBlockName (From) << "," << getBlockName(To) << "} to it found\n" ; } } while (false) | |||
2437 | << "} to it found\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Block " << getBlockName( To) << " was reachable, but new edge {" << getBlockName (From) << "," << getBlockName(To) << "} to it found\n" ; } } while (false); | |||
2438 | ||||
2439 | // We've made an edge reachable to an existing block, which may | |||
2440 | // impact predicates. Otherwise, only mark the phi nodes as touched, as | |||
2441 | // they are the only thing that depend on new edges. Anything using their | |||
2442 | // values will get propagated to if necessary. | |||
2443 | if (MemoryAccess *MemPhi = getMemoryAccess(To)) | |||
2444 | TouchedInstructions.set(InstrToDFSNum(MemPhi)); | |||
2445 | ||||
2446 | // FIXME: We should just add a union op on a Bitvector and | |||
2447 | // SparseBitVector. We can do it word by word faster than we are doing it | |||
2448 | // here. | |||
2449 | for (auto InstNum : RevisitOnReachabilityChange[To]) | |||
2450 | TouchedInstructions.set(InstNum); | |||
2451 | } | |||
2452 | } | |||
2453 | } | |||
2454 | ||||
2455 | // Given a predicate condition (from a switch, cmp, or whatever) and a block, | |||
2456 | // see if we know some constant value for it already. | |||
2457 | Value *NewGVN::findConditionEquivalence(Value *Cond) const { | |||
2458 | auto Result = lookupOperandLeader(Cond); | |||
2459 | return isa<Constant>(Result) ? Result : nullptr; | |||
2460 | } | |||
2461 | ||||
2462 | // Process the outgoing edges of a block for reachability. | |||
2463 | void NewGVN::processOutgoingEdges(Instruction *TI, BasicBlock *B) { | |||
2464 | // Evaluate reachability of terminator instruction. | |||
2465 | BranchInst *BR; | |||
2466 | if ((BR = dyn_cast<BranchInst>(TI)) && BR->isConditional()) { | |||
2467 | Value *Cond = BR->getCondition(); | |||
2468 | Value *CondEvaluated = findConditionEquivalence(Cond); | |||
2469 | if (!CondEvaluated) { | |||
2470 | if (auto *I = dyn_cast<Instruction>(Cond)) { | |||
2471 | const Expression *E = createExpression(I); | |||
2472 | if (const auto *CE = dyn_cast<ConstantExpression>(E)) { | |||
2473 | CondEvaluated = CE->getConstantValue(); | |||
2474 | } | |||
2475 | } else if (isa<ConstantInt>(Cond)) { | |||
2476 | CondEvaluated = Cond; | |||
2477 | } | |||
2478 | } | |||
2479 | ConstantInt *CI; | |||
2480 | BasicBlock *TrueSucc = BR->getSuccessor(0); | |||
2481 | BasicBlock *FalseSucc = BR->getSuccessor(1); | |||
2482 | if (CondEvaluated && (CI = dyn_cast<ConstantInt>(CondEvaluated))) { | |||
2483 | if (CI->isOne()) { | |||
2484 | LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Condition for Terminator " << *TI << " evaluated to true\n"; } } while (false) | |||
2485 | << " evaluated to true\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Condition for Terminator " << *TI << " evaluated to true\n"; } } while (false); | |||
2486 | updateReachableEdge(B, TrueSucc); | |||
2487 | } else if (CI->isZero()) { | |||
2488 | LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Condition for Terminator " << *TI << " evaluated to false\n"; } } while (false) | |||
2489 | << " evaluated to false\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Condition for Terminator " << *TI << " evaluated to false\n"; } } while (false); | |||
2490 | updateReachableEdge(B, FalseSucc); | |||
2491 | } | |||
2492 | } else { | |||
2493 | updateReachableEdge(B, TrueSucc); | |||
2494 | updateReachableEdge(B, FalseSucc); | |||
2495 | } | |||
2496 | } else if (auto *SI = dyn_cast<SwitchInst>(TI)) { | |||
2497 | // For switches, propagate the case values into the case | |||
2498 | // destinations. | |||
2499 | ||||
2500 | // Remember how many outgoing edges there are to every successor. | |||
2501 | SmallDenseMap<BasicBlock *, unsigned, 16> SwitchEdges; | |||
2502 | ||||
2503 | Value *SwitchCond = SI->getCondition(); | |||
2504 | Value *CondEvaluated = findConditionEquivalence(SwitchCond); | |||
2505 | // See if we were able to turn this switch statement into a constant. | |||
2506 | if (CondEvaluated && isa<ConstantInt>(CondEvaluated)) { | |||
2507 | auto *CondVal = cast<ConstantInt>(CondEvaluated); | |||
2508 | // We should be able to get case value for this. | |||
2509 | auto Case = *SI->findCaseValue(CondVal); | |||
2510 | if (Case.getCaseSuccessor() == SI->getDefaultDest()) { | |||
2511 | // We proved the value is outside of the range of the case. | |||
2512 | // We can't do anything other than mark the default dest as reachable, | |||
2513 | // and go home. | |||
2514 | updateReachableEdge(B, SI->getDefaultDest()); | |||
2515 | return; | |||
2516 | } | |||
2517 | // Now get where it goes and mark it reachable. | |||
2518 | BasicBlock *TargetBlock = Case.getCaseSuccessor(); | |||
2519 | updateReachableEdge(B, TargetBlock); | |||
2520 | } else { | |||
2521 | for (unsigned i = 0, e = SI->getNumSuccessors(); i != e; ++i) { | |||
2522 | BasicBlock *TargetBlock = SI->getSuccessor(i); | |||
2523 | ++SwitchEdges[TargetBlock]; | |||
2524 | updateReachableEdge(B, TargetBlock); | |||
2525 | } | |||
2526 | } | |||
2527 | } else { | |||
2528 | // Otherwise this is either unconditional, or a type we have no | |||
2529 | // idea about. Just mark successors as reachable. | |||
2530 | for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) { | |||
2531 | BasicBlock *TargetBlock = TI->getSuccessor(i); | |||
2532 | updateReachableEdge(B, TargetBlock); | |||
2533 | } | |||
2534 | ||||
2535 | // This also may be a memory defining terminator, in which case, set it | |||
2536 | // equivalent only to itself. | |||
2537 | // | |||
2538 | auto *MA = getMemoryAccess(TI); | |||
2539 | if (MA && !isa<MemoryUse>(MA)) { | |||
2540 | auto *CC = ensureLeaderOfMemoryClass(MA); | |||
2541 | if (setMemoryClass(MA, CC)) | |||
2542 | markMemoryUsersTouched(MA); | |||
2543 | } | |||
2544 | } | |||
2545 | } | |||
2546 | ||||
2547 | // Remove the PHI of Ops PHI for I | |||
2548 | void NewGVN::removePhiOfOps(Instruction *I, PHINode *PHITemp) { | |||
2549 | InstrDFS.erase(PHITemp); | |||
2550 | // It's still a temp instruction. We keep it in the array so it gets erased. | |||
2551 | // However, it's no longer used by I, or in the block | |||
2552 | TempToBlock.erase(PHITemp); | |||
2553 | RealToTemp.erase(I); | |||
2554 | // We don't remove the users from the phi node uses. This wastes a little | |||
2555 | // time, but such is life. We could use two sets to track which were there | |||
2556 | // are the start of NewGVN, and which were added, but right nowt he cost of | |||
2557 | // tracking is more than the cost of checking for more phi of ops. | |||
2558 | } | |||
2559 | ||||
2560 | // Add PHI Op in BB as a PHI of operations version of ExistingValue. | |||
2561 | void NewGVN::addPhiOfOps(PHINode *Op, BasicBlock *BB, | |||
2562 | Instruction *ExistingValue) { | |||
2563 | InstrDFS[Op] = InstrToDFSNum(ExistingValue); | |||
2564 | AllTempInstructions.insert(Op); | |||
2565 | TempToBlock[Op] = BB; | |||
2566 | RealToTemp[ExistingValue] = Op; | |||
2567 | // Add all users to phi node use, as they are now uses of the phi of ops phis | |||
2568 | // and may themselves be phi of ops. | |||
2569 | for (auto *U : ExistingValue->users()) | |||
2570 | if (auto *UI = dyn_cast<Instruction>(U)) | |||
2571 | PHINodeUses.insert(UI); | |||
2572 | } | |||
2573 | ||||
2574 | static bool okayForPHIOfOps(const Instruction *I) { | |||
2575 | if (!EnablePhiOfOps) | |||
2576 | return false; | |||
2577 | return isa<BinaryOperator>(I) || isa<SelectInst>(I) || isa<CmpInst>(I) || | |||
2578 | isa<LoadInst>(I); | |||
2579 | } | |||
2580 | ||||
2581 | bool NewGVN::OpIsSafeForPHIOfOpsHelper( | |||
2582 | Value *V, const BasicBlock *PHIBlock, | |||
2583 | SmallPtrSetImpl<const Value *> &Visited, | |||
2584 | SmallVectorImpl<Instruction *> &Worklist) { | |||
2585 | ||||
2586 | if (!isa<Instruction>(V)) | |||
2587 | return true; | |||
2588 | auto OISIt = OpSafeForPHIOfOps.find(V); | |||
2589 | if (OISIt != OpSafeForPHIOfOps.end()) | |||
2590 | return OISIt->second; | |||
2591 | ||||
2592 | // Keep walking until we either dominate the phi block, or hit a phi, or run | |||
2593 | // out of things to check. | |||
2594 | if (DT->properlyDominates(getBlockForValue(V), PHIBlock)) { | |||
2595 | OpSafeForPHIOfOps.insert({V, true}); | |||
2596 | return true; | |||
2597 | } | |||
2598 | // PHI in the same block. | |||
2599 | if (isa<PHINode>(V) && getBlockForValue(V) == PHIBlock) { | |||
2600 | OpSafeForPHIOfOps.insert({V, false}); | |||
2601 | return false; | |||
2602 | } | |||
2603 | ||||
2604 | auto *OrigI = cast<Instruction>(V); | |||
2605 | for (auto *Op : OrigI->operand_values()) { | |||
2606 | if (!isa<Instruction>(Op)) | |||
2607 | continue; | |||
2608 | // Stop now if we find an unsafe operand. | |||
2609 | auto OISIt = OpSafeForPHIOfOps.find(OrigI); | |||
2610 | if (OISIt != OpSafeForPHIOfOps.end()) { | |||
2611 | if (!OISIt->second) { | |||
2612 | OpSafeForPHIOfOps.insert({V, false}); | |||
2613 | return false; | |||
2614 | } | |||
2615 | continue; | |||
2616 | } | |||
2617 | if (!Visited.insert(Op).second) | |||
2618 | continue; | |||
2619 | Worklist.push_back(cast<Instruction>(Op)); | |||
2620 | } | |||
2621 | return true; | |||
2622 | } | |||
2623 | ||||
2624 | // Return true if this operand will be safe to use for phi of ops. | |||
2625 | // | |||
2626 | // The reason some operands are unsafe is that we are not trying to recursively | |||
2627 | // translate everything back through phi nodes. We actually expect some lookups | |||
2628 | // of expressions to fail. In particular, a lookup where the expression cannot | |||
2629 | // exist in the predecessor. This is true even if the expression, as shown, can | |||
2630 | // be determined to be constant. | |||
2631 | bool NewGVN::OpIsSafeForPHIOfOps(Value *V, const BasicBlock *PHIBlock, | |||
2632 | SmallPtrSetImpl<const Value *> &Visited) { | |||
2633 | SmallVector<Instruction *, 4> Worklist; | |||
2634 | if (!OpIsSafeForPHIOfOpsHelper(V, PHIBlock, Visited, Worklist)) | |||
2635 | return false; | |||
2636 | while (!Worklist.empty()) { | |||
2637 | auto *I = Worklist.pop_back_val(); | |||
2638 | if (!OpIsSafeForPHIOfOpsHelper(I, PHIBlock, Visited, Worklist)) | |||
2639 | return false; | |||
2640 | } | |||
2641 | OpSafeForPHIOfOps.insert({V, true}); | |||
2642 | return true; | |||
2643 | } | |||
2644 | ||||
2645 | // Try to find a leader for instruction TransInst, which is a phi translated | |||
2646 | // version of something in our original program. Visited is used to ensure we | |||
2647 | // don't infinite loop during translations of cycles. OrigInst is the | |||
2648 | // instruction in the original program, and PredBB is the predecessor we | |||
2649 | // translated it through. | |||
2650 | Value *NewGVN::findLeaderForInst(Instruction *TransInst, | |||
2651 | SmallPtrSetImpl<Value *> &Visited, | |||
2652 | MemoryAccess *MemAccess, Instruction *OrigInst, | |||
2653 | BasicBlock *PredBB) { | |||
2654 | unsigned IDFSNum = InstrToDFSNum(OrigInst); | |||
2655 | // Make sure it's marked as a temporary instruction. | |||
2656 | AllTempInstructions.insert(TransInst); | |||
2657 | // and make sure anything that tries to add it's DFS number is | |||
2658 | // redirected to the instruction we are making a phi of ops | |||
2659 | // for. | |||
2660 | TempToBlock.insert({TransInst, PredBB}); | |||
2661 | InstrDFS.insert({TransInst, IDFSNum}); | |||
2662 | ||||
2663 | const Expression *E = performSymbolicEvaluation(TransInst, Visited); | |||
2664 | InstrDFS.erase(TransInst); | |||
2665 | AllTempInstructions.erase(TransInst); | |||
2666 | TempToBlock.erase(TransInst); | |||
2667 | if (MemAccess) | |||
2668 | TempToMemory.erase(TransInst); | |||
2669 | if (!E) | |||
2670 | return nullptr; | |||
2671 | auto *FoundVal = findPHIOfOpsLeader(E, OrigInst, PredBB); | |||
2672 | if (!FoundVal) { | |||
2673 | ExpressionToPhiOfOps[E].insert(OrigInst); | |||
2674 | LLVM_DEBUG(dbgs() << "Cannot find phi of ops operand for " << *TransInstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Cannot find phi of ops operand for " << *TransInst << " in block " << getBlockName (PredBB) << "\n"; } } while (false) | |||
2675 | << " in block " << getBlockName(PredBB) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Cannot find phi of ops operand for " << *TransInst << " in block " << getBlockName (PredBB) << "\n"; } } while (false); | |||
2676 | return nullptr; | |||
2677 | } | |||
2678 | if (auto *SI = dyn_cast<StoreInst>(FoundVal)) | |||
2679 | FoundVal = SI->getValueOperand(); | |||
2680 | return FoundVal; | |||
2681 | } | |||
2682 | ||||
2683 | // When we see an instruction that is an op of phis, generate the equivalent phi | |||
2684 | // of ops form. | |||
2685 | const Expression * | |||
2686 | NewGVN::makePossiblePHIOfOps(Instruction *I, | |||
2687 | SmallPtrSetImpl<Value *> &Visited) { | |||
2688 | if (!okayForPHIOfOps(I)) | |||
| ||||
2689 | return nullptr; | |||
2690 | ||||
2691 | if (!Visited.insert(I).second) | |||
2692 | return nullptr; | |||
2693 | // For now, we require the instruction be cycle free because we don't | |||
2694 | // *always* create a phi of ops for instructions that could be done as phi | |||
2695 | // of ops, we only do it if we think it is useful. If we did do it all the | |||
2696 | // time, we could remove the cycle free check. | |||
2697 | if (!isCycleFree(I)) | |||
2698 | return nullptr; | |||
2699 | ||||
2700 | SmallPtrSet<const Value *, 8> ProcessedPHIs; | |||
2701 | // TODO: We don't do phi translation on memory accesses because it's | |||
2702 | // complicated. For a load, we'd need to be able to simulate a new memoryuse, | |||
2703 | // which we don't have a good way of doing ATM. | |||
2704 | auto *MemAccess = getMemoryAccess(I); | |||
2705 | // If the memory operation is defined by a memory operation this block that | |||
2706 | // isn't a MemoryPhi, transforming the pointer backwards through a scalar phi | |||
2707 | // can't help, as it would still be killed by that memory operation. | |||
2708 | if (MemAccess && !isa<MemoryPhi>(MemAccess->getDefiningAccess()) && | |||
2709 | MemAccess->getDefiningAccess()->getBlock() == I->getParent()) | |||
2710 | return nullptr; | |||
2711 | ||||
2712 | // Convert op of phis to phi of ops | |||
2713 | SmallPtrSet<const Value *, 10> VisitedOps; | |||
2714 | SmallVector<Value *, 4> Ops(I->operand_values()); | |||
2715 | BasicBlock *SamePHIBlock = nullptr; | |||
2716 | PHINode *OpPHI = nullptr; | |||
2717 | if (!DebugCounter::shouldExecute(PHIOfOpsCounter)) | |||
2718 | return nullptr; | |||
2719 | for (auto *Op : Ops) { | |||
2720 | if (!isa<PHINode>(Op)) { | |||
2721 | auto *ValuePHI = RealToTemp.lookup(Op); | |||
2722 | if (!ValuePHI) | |||
2723 | continue; | |||
2724 | LLVM_DEBUG(dbgs() << "Found possible dependent phi of ops\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Found possible dependent phi of ops\n" ; } } while (false); | |||
2725 | Op = ValuePHI; | |||
2726 | } | |||
2727 | OpPHI = cast<PHINode>(Op); | |||
2728 | if (!SamePHIBlock) { | |||
2729 | SamePHIBlock = getBlockForValue(OpPHI); | |||
2730 | } else if (SamePHIBlock != getBlockForValue(OpPHI)) { | |||
2731 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "PHIs for operands are not all in the same block, aborting\n" ; } } while (false) | |||
2732 | dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "PHIs for operands are not all in the same block, aborting\n" ; } } while (false) | |||
2733 | << "PHIs for operands are not all in the same block, aborting\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "PHIs for operands are not all in the same block, aborting\n" ; } } while (false); | |||
2734 | return nullptr; | |||
2735 | } | |||
2736 | // No point in doing this for one-operand phis. | |||
2737 | if (OpPHI->getNumOperands() == 1) { | |||
2738 | OpPHI = nullptr; | |||
2739 | continue; | |||
2740 | } | |||
2741 | } | |||
2742 | ||||
2743 | if (!OpPHI) | |||
2744 | return nullptr; | |||
2745 | ||||
2746 | SmallVector<ValPair, 4> PHIOps; | |||
2747 | SmallPtrSet<Value *, 4> Deps; | |||
2748 | auto *PHIBlock = getBlockForValue(OpPHI); | |||
2749 | RevisitOnReachabilityChange[PHIBlock].reset(InstrToDFSNum(I)); | |||
2750 | for (unsigned PredNum = 0; PredNum < OpPHI->getNumOperands(); ++PredNum) { | |||
2751 | auto *PredBB = OpPHI->getIncomingBlock(PredNum); | |||
2752 | Value *FoundVal = nullptr; | |||
2753 | SmallPtrSet<Value *, 4> CurrentDeps; | |||
2754 | // We could just skip unreachable edges entirely but it's tricky to do | |||
2755 | // with rewriting existing phi nodes. | |||
2756 | if (ReachableEdges.count({PredBB, PHIBlock})) { | |||
2757 | // Clone the instruction, create an expression from it that is | |||
2758 | // translated back into the predecessor, and see if we have a leader. | |||
2759 | Instruction *ValueOp = I->clone(); | |||
2760 | if (MemAccess) | |||
2761 | TempToMemory.insert({ValueOp, MemAccess}); | |||
2762 | bool SafeForPHIOfOps = true; | |||
2763 | VisitedOps.clear(); | |||
2764 | for (auto &Op : ValueOp->operands()) { | |||
2765 | auto *OrigOp = &*Op; | |||
2766 | // When these operand changes, it could change whether there is a | |||
2767 | // leader for us or not, so we have to add additional users. | |||
2768 | if (isa<PHINode>(Op)) { | |||
2769 | Op = Op->DoPHITranslation(PHIBlock, PredBB); | |||
2770 | if (Op != OrigOp && Op != I) | |||
2771 | CurrentDeps.insert(Op); | |||
2772 | } else if (auto *ValuePHI = RealToTemp.lookup(Op)) { | |||
2773 | if (getBlockForValue(ValuePHI) == PHIBlock) | |||
2774 | Op = ValuePHI->getIncomingValueForBlock(PredBB); | |||
2775 | } | |||
2776 | // If we phi-translated the op, it must be safe. | |||
2777 | SafeForPHIOfOps = | |||
2778 | SafeForPHIOfOps && | |||
2779 | (Op != OrigOp || OpIsSafeForPHIOfOps(Op, PHIBlock, VisitedOps)); | |||
2780 | } | |||
2781 | // FIXME: For those things that are not safe we could generate | |||
2782 | // expressions all the way down, and see if this comes out to a | |||
2783 | // constant. For anything where that is true, and unsafe, we should | |||
2784 | // have made a phi-of-ops (or value numbered it equivalent to something) | |||
2785 | // for the pieces already. | |||
2786 | FoundVal = !SafeForPHIOfOps ? nullptr | |||
2787 | : findLeaderForInst(ValueOp, Visited, | |||
2788 | MemAccess, I, PredBB); | |||
2789 | ValueOp->deleteValue(); | |||
2790 | if (!FoundVal) { | |||
2791 | // We failed to find a leader for the current ValueOp, but this might | |||
2792 | // change in case of the translated operands change. | |||
2793 | if (SafeForPHIOfOps) | |||
2794 | for (auto Dep : CurrentDeps) | |||
2795 | addAdditionalUsers(Dep, I); | |||
2796 | ||||
2797 | return nullptr; | |||
2798 | } | |||
2799 | Deps.insert(CurrentDeps.begin(), CurrentDeps.end()); | |||
2800 | } else { | |||
2801 | LLVM_DEBUG(dbgs() << "Skipping phi of ops operand for incoming block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Skipping phi of ops operand for incoming block " << getBlockName(PredBB) << " because the block is unreachable\n" ; } } while (false) | |||
2802 | << getBlockName(PredBB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Skipping phi of ops operand for incoming block " << getBlockName(PredBB) << " because the block is unreachable\n" ; } } while (false) | |||
2803 | << " because the block is unreachable\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Skipping phi of ops operand for incoming block " << getBlockName(PredBB) << " because the block is unreachable\n" ; } } while (false); | |||
2804 | FoundVal = UndefValue::get(I->getType()); | |||
2805 | RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I)); | |||
2806 | } | |||
2807 | ||||
2808 | PHIOps.push_back({FoundVal, PredBB}); | |||
2809 | LLVM_DEBUG(dbgs() << "Found phi of ops operand " << *FoundVal << " in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Found phi of ops operand " << *FoundVal << " in " << getBlockName(PredBB) << "\n"; } } while (false) | |||
2810 | << getBlockName(PredBB) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Found phi of ops operand " << *FoundVal << " in " << getBlockName(PredBB) << "\n"; } } while (false); | |||
2811 | } | |||
2812 | for (auto Dep : Deps) | |||
2813 | addAdditionalUsers(Dep, I); | |||
2814 | sortPHIOps(PHIOps); | |||
2815 | auto *E = performSymbolicPHIEvaluation(PHIOps, I, PHIBlock); | |||
2816 | if (isa<ConstantExpression>(E) || isa<VariableExpression>(E)) { | |||
2817 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Not creating real PHI of ops because it simplified to existing " "value or constant\n"; } } while (false) | |||
2818 | dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Not creating real PHI of ops because it simplified to existing " "value or constant\n"; } } while (false) | |||
2819 | << "Not creating real PHI of ops because it simplified to existing "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Not creating real PHI of ops because it simplified to existing " "value or constant\n"; } } while (false) | |||
2820 | "value or constant\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Not creating real PHI of ops because it simplified to existing " "value or constant\n"; } } while (false); | |||
2821 | return E; | |||
2822 | } | |||
2823 | auto *ValuePHI = RealToTemp.lookup(I); | |||
2824 | bool NewPHI = false; | |||
2825 | if (!ValuePHI) { | |||
2826 | ValuePHI = | |||
2827 | PHINode::Create(I->getType(), OpPHI->getNumOperands(), "phiofops"); | |||
2828 | addPhiOfOps(ValuePHI, PHIBlock, I); | |||
2829 | NewPHI = true; | |||
2830 | NumGVNPHIOfOpsCreated++; | |||
2831 | } | |||
2832 | if (NewPHI) { | |||
2833 | for (auto PHIOp : PHIOps) | |||
2834 | ValuePHI->addIncoming(PHIOp.first, PHIOp.second); | |||
2835 | } else { | |||
2836 | TempToBlock[ValuePHI] = PHIBlock; | |||
2837 | unsigned int i = 0; | |||
2838 | for (auto PHIOp : PHIOps) { | |||
2839 | ValuePHI->setIncomingValue(i, PHIOp.first); | |||
2840 | ValuePHI->setIncomingBlock(i, PHIOp.second); | |||
2841 | ++i; | |||
2842 | } | |||
2843 | } | |||
2844 | RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I)); | |||
2845 | LLVM_DEBUG(dbgs() << "Created phi of ops " << *ValuePHI << " for " << *Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Created phi of ops " << * ValuePHI << " for " << *I << "\n"; } } while (false) | |||
2846 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Created phi of ops " << * ValuePHI << " for " << *I << "\n"; } } while (false); | |||
2847 | ||||
2848 | return E; | |||
2849 | } | |||
2850 | ||||
2851 | // The algorithm initially places the values of the routine in the TOP | |||
2852 | // congruence class. The leader of TOP is the undetermined value `undef`. | |||
2853 | // When the algorithm has finished, values still in TOP are unreachable. | |||
2854 | void NewGVN::initializeCongruenceClasses(Function &F) { | |||
2855 | NextCongruenceNum = 0; | |||
2856 | ||||
2857 | // Note that even though we use the live on entry def as a representative | |||
2858 | // MemoryAccess, it is *not* the same as the actual live on entry def. We | |||
2859 | // have no real equivalemnt to undef for MemoryAccesses, and so we really | |||
2860 | // should be checking whether the MemoryAccess is top if we want to know if it | |||
2861 | // is equivalent to everything. Otherwise, what this really signifies is that | |||
2862 | // the access "it reaches all the way back to the beginning of the function" | |||
2863 | ||||
2864 | // Initialize all other instructions to be in TOP class. | |||
2865 | TOPClass = createCongruenceClass(nullptr, nullptr); | |||
2866 | TOPClass->setMemoryLeader(MSSA->getLiveOnEntryDef()); | |||
2867 | // The live on entry def gets put into it's own class | |||
2868 | MemoryAccessToClass[MSSA->getLiveOnEntryDef()] = | |||
2869 | createMemoryClass(MSSA->getLiveOnEntryDef()); | |||
2870 | ||||
2871 | for (auto DTN : nodes(DT)) { | |||
2872 | BasicBlock *BB = DTN->getBlock(); | |||
2873 | // All MemoryAccesses are equivalent to live on entry to start. They must | |||
2874 | // be initialized to something so that initial changes are noticed. For | |||
2875 | // the maximal answer, we initialize them all to be the same as | |||
2876 | // liveOnEntry. | |||
2877 | auto *MemoryBlockDefs = MSSA->getBlockDefs(BB); | |||
2878 | if (MemoryBlockDefs) | |||
2879 | for (const auto &Def : *MemoryBlockDefs) { | |||
2880 | MemoryAccessToClass[&Def] = TOPClass; | |||
2881 | auto *MD = dyn_cast<MemoryDef>(&Def); | |||
2882 | // Insert the memory phis into the member list. | |||
2883 | if (!MD) { | |||
2884 | const MemoryPhi *MP = cast<MemoryPhi>(&Def); | |||
2885 | TOPClass->memory_insert(MP); | |||
2886 | MemoryPhiState.insert({MP, MPS_TOP}); | |||
2887 | } | |||
2888 | ||||
2889 | if (MD && isa<StoreInst>(MD->getMemoryInst())) | |||
2890 | TOPClass->incStoreCount(); | |||
2891 | } | |||
2892 | ||||
2893 | // FIXME: This is trying to discover which instructions are uses of phi | |||
2894 | // nodes. We should move this into one of the myriad of places that walk | |||
2895 | // all the operands already. | |||
2896 | for (auto &I : *BB) { | |||
2897 | if (isa<PHINode>(&I)) | |||
2898 | for (auto *U : I.users()) | |||
2899 | if (auto *UInst = dyn_cast<Instruction>(U)) | |||
2900 | if (InstrToDFSNum(UInst) != 0 && okayForPHIOfOps(UInst)) | |||
2901 | PHINodeUses.insert(UInst); | |||
2902 | // Don't insert void terminators into the class. We don't value number | |||
2903 | // them, and they just end up sitting in TOP. | |||
2904 | if (I.isTerminator() && I.getType()->isVoidTy()) | |||
2905 | continue; | |||
2906 | TOPClass->insert(&I); | |||
2907 | ValueToClass[&I] = TOPClass; | |||
2908 | } | |||
2909 | } | |||
2910 | ||||
2911 | // Initialize arguments to be in their own unique congruence classes | |||
2912 | for (auto &FA : F.args()) | |||
2913 | createSingletonCongruenceClass(&FA); | |||
2914 | } | |||
2915 | ||||
2916 | void NewGVN::cleanupTables() { | |||
2917 | for (unsigned i = 0, e = CongruenceClasses.size(); i != e; ++i) { | |||
2918 | LLVM_DEBUG(dbgs() << "Congruence class " << CongruenceClasses[i]->getID()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Congruence class " << CongruenceClasses [i]->getID() << " has " << CongruenceClasses[i ]->size() << " members\n"; } } while (false) | |||
2919 | << " has " << CongruenceClasses[i]->size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Congruence class " << CongruenceClasses [i]->getID() << " has " << CongruenceClasses[i ]->size() << " members\n"; } } while (false) | |||
2920 | << " members\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Congruence class " << CongruenceClasses [i]->getID() << " has " << CongruenceClasses[i ]->size() << " members\n"; } } while (false); | |||
2921 | // Make sure we delete the congruence class (probably worth switching to | |||
2922 | // a unique_ptr at some point. | |||
2923 | delete CongruenceClasses[i]; | |||
2924 | CongruenceClasses[i] = nullptr; | |||
2925 | } | |||
2926 | ||||
2927 | // Destroy the value expressions | |||
2928 | SmallVector<Instruction *, 8> TempInst(AllTempInstructions.begin(), | |||
2929 | AllTempInstructions.end()); | |||
2930 | AllTempInstructions.clear(); | |||
2931 | ||||
2932 | // We have to drop all references for everything first, so there are no uses | |||
2933 | // left as we delete them. | |||
2934 | for (auto *I : TempInst) { | |||
2935 | I->dropAllReferences(); | |||
2936 | } | |||
2937 | ||||
2938 | while (!TempInst.empty()) { | |||
2939 | auto *I = TempInst.back(); | |||
2940 | TempInst.pop_back(); | |||
2941 | I->deleteValue(); | |||
2942 | } | |||
2943 | ||||
2944 | ValueToClass.clear(); | |||
2945 | ArgRecycler.clear(ExpressionAllocator); | |||
2946 | ExpressionAllocator.Reset(); | |||
2947 | CongruenceClasses.clear(); | |||
2948 | ExpressionToClass.clear(); | |||
2949 | ValueToExpression.clear(); | |||
2950 | RealToTemp.clear(); | |||
2951 | AdditionalUsers.clear(); | |||
2952 | ExpressionToPhiOfOps.clear(); | |||
2953 | TempToBlock.clear(); | |||
2954 | TempToMemory.clear(); | |||
2955 | PHINodeUses.clear(); | |||
2956 | OpSafeForPHIOfOps.clear(); | |||
2957 | ReachableBlocks.clear(); | |||
2958 | ReachableEdges.clear(); | |||
2959 | #ifndef NDEBUG | |||
2960 | ProcessedCount.clear(); | |||
2961 | #endif | |||
2962 | InstrDFS.clear(); | |||
2963 | InstructionsToErase.clear(); | |||
2964 | DFSToInstr.clear(); | |||
2965 | BlockInstRange.clear(); | |||
2966 | TouchedInstructions.clear(); | |||
2967 | MemoryAccessToClass.clear(); | |||
2968 | PredicateToUsers.clear(); | |||
2969 | MemoryToUsers.clear(); | |||
2970 | RevisitOnReachabilityChange.clear(); | |||
2971 | } | |||
2972 | ||||
2973 | // Assign local DFS number mapping to instructions, and leave space for Value | |||
2974 | // PHI's. | |||
2975 | std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B, | |||
2976 | unsigned Start) { | |||
2977 | unsigned End = Start; | |||
2978 | if (MemoryAccess *MemPhi = getMemoryAccess(B)) { | |||
2979 | InstrDFS[MemPhi] = End++; | |||
2980 | DFSToInstr.emplace_back(MemPhi); | |||
2981 | } | |||
2982 | ||||
2983 | // Then the real block goes next. | |||
2984 | for (auto &I : *B) { | |||
2985 | // There's no need to call isInstructionTriviallyDead more than once on | |||
2986 | // an instruction. Therefore, once we know that an instruction is dead | |||
2987 | // we change its DFS number so that it doesn't get value numbered. | |||
2988 | if (isInstructionTriviallyDead(&I, TLI)) { | |||
2989 | InstrDFS[&I] = 0; | |||
2990 | LLVM_DEBUG(dbgs() << "Skipping trivially dead instruction " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Skipping trivially dead instruction " << I << "\n"; } } while (false); | |||
2991 | markInstructionForDeletion(&I); | |||
2992 | continue; | |||
2993 | } | |||
2994 | if (isa<PHINode>(&I)) | |||
2995 | RevisitOnReachabilityChange[B].set(End); | |||
2996 | InstrDFS[&I] = End++; | |||
2997 | DFSToInstr.emplace_back(&I); | |||
2998 | } | |||
2999 | ||||
3000 | // All of the range functions taken half-open ranges (open on the end side). | |||
3001 | // So we do not subtract one from count, because at this point it is one | |||
3002 | // greater than the last instruction. | |||
3003 | return std::make_pair(Start, End); | |||
3004 | } | |||
3005 | ||||
3006 | void NewGVN::updateProcessedCount(const Value *V) { | |||
3007 | #ifndef NDEBUG | |||
3008 | if (ProcessedCount.count(V) == 0) { | |||
3009 | ProcessedCount.insert({V, 1}); | |||
3010 | } else { | |||
3011 | ++ProcessedCount[V]; | |||
3012 | assert(ProcessedCount[V] < 100 &&((ProcessedCount[V] < 100 && "Seem to have processed the same Value a lot" ) ? static_cast<void> (0) : __assert_fail ("ProcessedCount[V] < 100 && \"Seem to have processed the same Value a lot\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3013, __PRETTY_FUNCTION__)) | |||
3013 | "Seem to have processed the same Value a lot")((ProcessedCount[V] < 100 && "Seem to have processed the same Value a lot" ) ? static_cast<void> (0) : __assert_fail ("ProcessedCount[V] < 100 && \"Seem to have processed the same Value a lot\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3013, __PRETTY_FUNCTION__)); | |||
3014 | } | |||
3015 | #endif | |||
3016 | } | |||
3017 | ||||
3018 | // Evaluate MemoryPhi nodes symbolically, just like PHI nodes | |||
3019 | void NewGVN::valueNumberMemoryPhi(MemoryPhi *MP) { | |||
3020 | // If all the arguments are the same, the MemoryPhi has the same value as the | |||
3021 | // argument. Filter out unreachable blocks and self phis from our operands. | |||
3022 | // TODO: We could do cycle-checking on the memory phis to allow valueizing for | |||
3023 | // self-phi checking. | |||
3024 | const BasicBlock *PHIBlock = MP->getBlock(); | |||
3025 | auto Filtered = make_filter_range(MP->operands(), [&](const Use &U) { | |||
3026 | return cast<MemoryAccess>(U) != MP && | |||
3027 | !isMemoryAccessTOP(cast<MemoryAccess>(U)) && | |||
3028 | ReachableEdges.count({MP->getIncomingBlock(U), PHIBlock}); | |||
3029 | }); | |||
3030 | // If all that is left is nothing, our memoryphi is undef. We keep it as | |||
3031 | // InitialClass. Note: The only case this should happen is if we have at | |||
3032 | // least one self-argument. | |||
3033 | if (Filtered.begin() == Filtered.end()) { | |||
3034 | if (setMemoryClass(MP, TOPClass)) | |||
3035 | markMemoryUsersTouched(MP); | |||
3036 | return; | |||
3037 | } | |||
3038 | ||||
3039 | // Transform the remaining operands into operand leaders. | |||
3040 | // FIXME: mapped_iterator should have a range version. | |||
3041 | auto LookupFunc = [&](const Use &U) { | |||
3042 | return lookupMemoryLeader(cast<MemoryAccess>(U)); | |||
3043 | }; | |||
3044 | auto MappedBegin = map_iterator(Filtered.begin(), LookupFunc); | |||
3045 | auto MappedEnd = map_iterator(Filtered.end(), LookupFunc); | |||
3046 | ||||
3047 | // and now check if all the elements are equal. | |||
3048 | // Sadly, we can't use std::equals since these are random access iterators. | |||
3049 | const auto *AllSameValue = *MappedBegin; | |||
3050 | ++MappedBegin; | |||
3051 | bool AllEqual = std::all_of( | |||
3052 | MappedBegin, MappedEnd, | |||
3053 | [&AllSameValue](const MemoryAccess *V) { return V == AllSameValue; }); | |||
3054 | ||||
3055 | if (AllEqual) | |||
3056 | LLVM_DEBUG(dbgs() << "Memory Phi value numbered to " << *AllSameValuedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory Phi value numbered to " << *AllSameValue << "\n"; } } while (false) | |||
3057 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory Phi value numbered to " << *AllSameValue << "\n"; } } while (false); | |||
3058 | else | |||
3059 | LLVM_DEBUG(dbgs() << "Memory Phi value numbered to itself\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Memory Phi value numbered to itself\n" ; } } while (false); | |||
3060 | // If it's equal to something, it's in that class. Otherwise, it has to be in | |||
3061 | // a class where it is the leader (other things may be equivalent to it, but | |||
3062 | // it needs to start off in its own class, which means it must have been the | |||
3063 | // leader, and it can't have stopped being the leader because it was never | |||
3064 | // removed). | |||
3065 | CongruenceClass *CC = | |||
3066 | AllEqual ? getMemoryClass(AllSameValue) : ensureLeaderOfMemoryClass(MP); | |||
3067 | auto OldState = MemoryPhiState.lookup(MP); | |||
3068 | assert(OldState != MPS_Invalid && "Invalid memory phi state")((OldState != MPS_Invalid && "Invalid memory phi state" ) ? static_cast<void> (0) : __assert_fail ("OldState != MPS_Invalid && \"Invalid memory phi state\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3068, __PRETTY_FUNCTION__)); | |||
3069 | auto NewState = AllEqual ? MPS_Equivalent : MPS_Unique; | |||
3070 | MemoryPhiState[MP] = NewState; | |||
3071 | if (setMemoryClass(MP, CC) || OldState != NewState) | |||
3072 | markMemoryUsersTouched(MP); | |||
3073 | } | |||
3074 | ||||
3075 | // Value number a single instruction, symbolically evaluating, performing | |||
3076 | // congruence finding, and updating mappings. | |||
3077 | void NewGVN::valueNumberInstruction(Instruction *I) { | |||
3078 | LLVM_DEBUG(dbgs() << "Processing instruction " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Processing instruction " << *I << "\n"; } } while (false); | |||
3079 | if (!I->isTerminator()) { | |||
3080 | const Expression *Symbolized = nullptr; | |||
3081 | SmallPtrSet<Value *, 2> Visited; | |||
3082 | if (DebugCounter::shouldExecute(VNCounter)) { | |||
3083 | Symbolized = performSymbolicEvaluation(I, Visited); | |||
3084 | // Make a phi of ops if necessary | |||
3085 | if (Symbolized && !isa<ConstantExpression>(Symbolized) && | |||
3086 | !isa<VariableExpression>(Symbolized) && PHINodeUses.count(I)) { | |||
3087 | auto *PHIE = makePossiblePHIOfOps(I, Visited); | |||
3088 | // If we created a phi of ops, use it. | |||
3089 | // If we couldn't create one, make sure we don't leave one lying around | |||
3090 | if (PHIE) { | |||
3091 | Symbolized = PHIE; | |||
3092 | } else if (auto *Op = RealToTemp.lookup(I)) { | |||
3093 | removePhiOfOps(I, Op); | |||
3094 | } | |||
3095 | } | |||
3096 | } else { | |||
3097 | // Mark the instruction as unused so we don't value number it again. | |||
3098 | InstrDFS[I] = 0; | |||
3099 | } | |||
3100 | // If we couldn't come up with a symbolic expression, use the unknown | |||
3101 | // expression | |||
3102 | if (Symbolized == nullptr) | |||
3103 | Symbolized = createUnknownExpression(I); | |||
3104 | performCongruenceFinding(I, Symbolized); | |||
3105 | } else { | |||
3106 | // Handle terminators that return values. All of them produce values we | |||
3107 | // don't currently understand. We don't place non-value producing | |||
3108 | // terminators in a class. | |||
3109 | if (!I->getType()->isVoidTy()) { | |||
3110 | auto *Symbolized = createUnknownExpression(I); | |||
3111 | performCongruenceFinding(I, Symbolized); | |||
3112 | } | |||
3113 | processOutgoingEdges(I, I->getParent()); | |||
3114 | } | |||
3115 | } | |||
3116 | ||||
3117 | // Check if there is a path, using single or equal argument phi nodes, from | |||
3118 | // First to Second. | |||
3119 | bool NewGVN::singleReachablePHIPath( | |||
3120 | SmallPtrSet<const MemoryAccess *, 8> &Visited, const MemoryAccess *First, | |||
3121 | const MemoryAccess *Second) const { | |||
3122 | if (First == Second) | |||
3123 | return true; | |||
3124 | if (MSSA->isLiveOnEntryDef(First)) | |||
3125 | return false; | |||
3126 | ||||
3127 | // This is not perfect, but as we're just verifying here, we can live with | |||
3128 | // the loss of precision. The real solution would be that of doing strongly | |||
3129 | // connected component finding in this routine, and it's probably not worth | |||
3130 | // the complexity for the time being. So, we just keep a set of visited | |||
3131 | // MemoryAccess and return true when we hit a cycle. | |||
3132 | if (Visited.count(First)) | |||
3133 | return true; | |||
3134 | Visited.insert(First); | |||
3135 | ||||
3136 | const auto *EndDef = First; | |||
3137 | for (auto *ChainDef : optimized_def_chain(First)) { | |||
3138 | if (ChainDef == Second) | |||
3139 | return true; | |||
3140 | if (MSSA->isLiveOnEntryDef(ChainDef)) | |||
3141 | return false; | |||
3142 | EndDef = ChainDef; | |||
3143 | } | |||
3144 | auto *MP = cast<MemoryPhi>(EndDef); | |||
3145 | auto ReachableOperandPred = [&](const Use &U) { | |||
3146 | return ReachableEdges.count({MP->getIncomingBlock(U), MP->getBlock()}); | |||
3147 | }; | |||
3148 | auto FilteredPhiArgs = | |||
3149 | make_filter_range(MP->operands(), ReachableOperandPred); | |||
3150 | SmallVector<const Value *, 32> OperandList; | |||
3151 | llvm::copy(FilteredPhiArgs, std::back_inserter(OperandList)); | |||
3152 | bool Okay = is_splat(OperandList); | |||
3153 | if (Okay) | |||
3154 | return singleReachablePHIPath(Visited, cast<MemoryAccess>(OperandList[0]), | |||
3155 | Second); | |||
3156 | return false; | |||
3157 | } | |||
3158 | ||||
3159 | // Verify the that the memory equivalence table makes sense relative to the | |||
3160 | // congruence classes. Note that this checking is not perfect, and is currently | |||
3161 | // subject to very rare false negatives. It is only useful for | |||
3162 | // testing/debugging. | |||
3163 | void NewGVN::verifyMemoryCongruency() const { | |||
3164 | #ifndef NDEBUG | |||
3165 | // Verify that the memory table equivalence and memory member set match | |||
3166 | for (const auto *CC : CongruenceClasses) { | |||
3167 | if (CC == TOPClass || CC->isDead()) | |||
3168 | continue; | |||
3169 | if (CC->getStoreCount() != 0) { | |||
3170 | assert((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) &&(((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader ())) && "Any class with a store as a leader should have a " "representative stored value") ? static_cast<void> (0) : __assert_fail ("(CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) && \"Any class with a store as a leader should have a \" \"representative stored value\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3172, __PRETTY_FUNCTION__)) | |||
3171 | "Any class with a store as a leader should have a "(((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader ())) && "Any class with a store as a leader should have a " "representative stored value") ? static_cast<void> (0) : __assert_fail ("(CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) && \"Any class with a store as a leader should have a \" \"representative stored value\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3172, __PRETTY_FUNCTION__)) | |||
3172 | "representative stored value")(((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader ())) && "Any class with a store as a leader should have a " "representative stored value") ? static_cast<void> (0) : __assert_fail ("(CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) && \"Any class with a store as a leader should have a \" \"representative stored value\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3172, __PRETTY_FUNCTION__)); | |||
3173 | assert(CC->getMemoryLeader() &&((CC->getMemoryLeader() && "Any congruence class with a store should have a " "representative access") ? static_cast<void> (0) : __assert_fail ("CC->getMemoryLeader() && \"Any congruence class with a store should have a \" \"representative access\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3175, __PRETTY_FUNCTION__)) | |||
3174 | "Any congruence class with a store should have a "((CC->getMemoryLeader() && "Any congruence class with a store should have a " "representative access") ? static_cast<void> (0) : __assert_fail ("CC->getMemoryLeader() && \"Any congruence class with a store should have a \" \"representative access\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3175, __PRETTY_FUNCTION__)) | |||
3175 | "representative access")((CC->getMemoryLeader() && "Any congruence class with a store should have a " "representative access") ? static_cast<void> (0) : __assert_fail ("CC->getMemoryLeader() && \"Any congruence class with a store should have a \" \"representative access\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3175, __PRETTY_FUNCTION__)); | |||
3176 | } | |||
3177 | ||||
3178 | if (CC->getMemoryLeader()) | |||
3179 | assert(MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC &&((MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC && "Representative MemoryAccess does not appear to be reverse " "mapped properly") ? static_cast<void> (0) : __assert_fail ("MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC && \"Representative MemoryAccess does not appear to be reverse \" \"mapped properly\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3181, __PRETTY_FUNCTION__)) | |||
3180 | "Representative MemoryAccess does not appear to be reverse "((MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC && "Representative MemoryAccess does not appear to be reverse " "mapped properly") ? static_cast<void> (0) : __assert_fail ("MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC && \"Representative MemoryAccess does not appear to be reverse \" \"mapped properly\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3181, __PRETTY_FUNCTION__)) | |||
3181 | "mapped properly")((MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC && "Representative MemoryAccess does not appear to be reverse " "mapped properly") ? static_cast<void> (0) : __assert_fail ("MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC && \"Representative MemoryAccess does not appear to be reverse \" \"mapped properly\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3181, __PRETTY_FUNCTION__)); | |||
3182 | for (auto M : CC->memory()) | |||
3183 | assert(MemoryAccessToClass.lookup(M) == CC &&((MemoryAccessToClass.lookup(M) == CC && "Memory member does not appear to be reverse mapped properly" ) ? static_cast<void> (0) : __assert_fail ("MemoryAccessToClass.lookup(M) == CC && \"Memory member does not appear to be reverse mapped properly\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3184, __PRETTY_FUNCTION__)) | |||
3184 | "Memory member does not appear to be reverse mapped properly")((MemoryAccessToClass.lookup(M) == CC && "Memory member does not appear to be reverse mapped properly" ) ? static_cast<void> (0) : __assert_fail ("MemoryAccessToClass.lookup(M) == CC && \"Memory member does not appear to be reverse mapped properly\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3184, __PRETTY_FUNCTION__)); | |||
3185 | } | |||
3186 | ||||
3187 | // Anything equivalent in the MemoryAccess table should be in the same | |||
3188 | // congruence class. | |||
3189 | ||||
3190 | // Filter out the unreachable and trivially dead entries, because they may | |||
3191 | // never have been updated if the instructions were not processed. | |||
3192 | auto ReachableAccessPred = | |||
3193 | [&](const std::pair<const MemoryAccess *, CongruenceClass *> Pair) { | |||
3194 | bool Result = ReachableBlocks.count(Pair.first->getBlock()); | |||
3195 | if (!Result || MSSA->isLiveOnEntryDef(Pair.first) || | |||
3196 | MemoryToDFSNum(Pair.first) == 0) | |||
3197 | return false; | |||
3198 | if (auto *MemDef = dyn_cast<MemoryDef>(Pair.first)) | |||
3199 | return !isInstructionTriviallyDead(MemDef->getMemoryInst()); | |||
3200 | ||||
3201 | // We could have phi nodes which operands are all trivially dead, | |||
3202 | // so we don't process them. | |||
3203 | if (auto *MemPHI = dyn_cast<MemoryPhi>(Pair.first)) { | |||
3204 | for (auto &U : MemPHI->incoming_values()) { | |||
3205 | if (auto *I = dyn_cast<Instruction>(&*U)) { | |||
3206 | if (!isInstructionTriviallyDead(I)) | |||
3207 | return true; | |||
3208 | } | |||
3209 | } | |||
3210 | return false; | |||
3211 | } | |||
3212 | ||||
3213 | return true; | |||
3214 | }; | |||
3215 | ||||
3216 | auto Filtered = make_filter_range(MemoryAccessToClass, ReachableAccessPred); | |||
3217 | for (auto KV : Filtered) { | |||
3218 | if (auto *FirstMUD = dyn_cast<MemoryUseOrDef>(KV.first)) { | |||
3219 | auto *SecondMUD = dyn_cast<MemoryUseOrDef>(KV.second->getMemoryLeader()); | |||
3220 | if (FirstMUD && SecondMUD) { | |||
3221 | SmallPtrSet<const MemoryAccess *, 8> VisitedMAS; | |||
3222 | assert((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) ||(((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass .lookup(SecondMUD->getMemoryInst())) && "The instructions for these memory operations should have " "been in the same congruence class or reachable through" "a single argument phi" ) ? static_cast<void> (0) : __assert_fail ("(singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass.lookup(SecondMUD->getMemoryInst())) && \"The instructions for these memory operations should have \" \"been in the same congruence class or reachable through\" \"a single argument phi\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3227, __PRETTY_FUNCTION__)) | |||
3223 | ValueToClass.lookup(FirstMUD->getMemoryInst()) ==(((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass .lookup(SecondMUD->getMemoryInst())) && "The instructions for these memory operations should have " "been in the same congruence class or reachable through" "a single argument phi" ) ? static_cast<void> (0) : __assert_fail ("(singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass.lookup(SecondMUD->getMemoryInst())) && \"The instructions for these memory operations should have \" \"been in the same congruence class or reachable through\" \"a single argument phi\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3227, __PRETTY_FUNCTION__)) | |||
3224 | ValueToClass.lookup(SecondMUD->getMemoryInst())) &&(((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass .lookup(SecondMUD->getMemoryInst())) && "The instructions for these memory operations should have " "been in the same congruence class or reachable through" "a single argument phi" ) ? static_cast<void> (0) : __assert_fail ("(singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass.lookup(SecondMUD->getMemoryInst())) && \"The instructions for these memory operations should have \" \"been in the same congruence class or reachable through\" \"a single argument phi\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3227, __PRETTY_FUNCTION__)) | |||
3225 | "The instructions for these memory operations should have "(((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass .lookup(SecondMUD->getMemoryInst())) && "The instructions for these memory operations should have " "been in the same congruence class or reachable through" "a single argument phi" ) ? static_cast<void> (0) : __assert_fail ("(singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass.lookup(SecondMUD->getMemoryInst())) && \"The instructions for these memory operations should have \" \"been in the same congruence class or reachable through\" \"a single argument phi\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3227, __PRETTY_FUNCTION__)) | |||
3226 | "been in the same congruence class or reachable through"(((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass .lookup(SecondMUD->getMemoryInst())) && "The instructions for these memory operations should have " "been in the same congruence class or reachable through" "a single argument phi" ) ? static_cast<void> (0) : __assert_fail ("(singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass.lookup(SecondMUD->getMemoryInst())) && \"The instructions for these memory operations should have \" \"been in the same congruence class or reachable through\" \"a single argument phi\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3227, __PRETTY_FUNCTION__)) | |||
3227 | "a single argument phi")(((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass .lookup(SecondMUD->getMemoryInst())) && "The instructions for these memory operations should have " "been in the same congruence class or reachable through" "a single argument phi" ) ? static_cast<void> (0) : __assert_fail ("(singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) || ValueToClass.lookup(FirstMUD->getMemoryInst()) == ValueToClass.lookup(SecondMUD->getMemoryInst())) && \"The instructions for these memory operations should have \" \"been in the same congruence class or reachable through\" \"a single argument phi\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3227, __PRETTY_FUNCTION__)); | |||
3228 | } | |||
3229 | } else if (auto *FirstMP = dyn_cast<MemoryPhi>(KV.first)) { | |||
3230 | // We can only sanely verify that MemoryDefs in the operand list all have | |||
3231 | // the same class. | |||
3232 | auto ReachableOperandPred = [&](const Use &U) { | |||
3233 | return ReachableEdges.count( | |||
3234 | {FirstMP->getIncomingBlock(U), FirstMP->getBlock()}) && | |||
3235 | isa<MemoryDef>(U); | |||
3236 | ||||
3237 | }; | |||
3238 | // All arguments should in the same class, ignoring unreachable arguments | |||
3239 | auto FilteredPhiArgs = | |||
3240 | make_filter_range(FirstMP->operands(), ReachableOperandPred); | |||
3241 | SmallVector<const CongruenceClass *, 16> PhiOpClasses; | |||
3242 | std::transform(FilteredPhiArgs.begin(), FilteredPhiArgs.end(), | |||
3243 | std::back_inserter(PhiOpClasses), [&](const Use &U) { | |||
3244 | const MemoryDef *MD = cast<MemoryDef>(U); | |||
3245 | return ValueToClass.lookup(MD->getMemoryInst()); | |||
3246 | }); | |||
3247 | assert(is_splat(PhiOpClasses) &&((is_splat(PhiOpClasses) && "All MemoryPhi arguments should be in the same class" ) ? static_cast<void> (0) : __assert_fail ("is_splat(PhiOpClasses) && \"All MemoryPhi arguments should be in the same class\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3248, __PRETTY_FUNCTION__)) | |||
3248 | "All MemoryPhi arguments should be in the same class")((is_splat(PhiOpClasses) && "All MemoryPhi arguments should be in the same class" ) ? static_cast<void> (0) : __assert_fail ("is_splat(PhiOpClasses) && \"All MemoryPhi arguments should be in the same class\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3248, __PRETTY_FUNCTION__)); | |||
3249 | } | |||
3250 | } | |||
3251 | #endif | |||
3252 | } | |||
3253 | ||||
3254 | // Verify that the sparse propagation we did actually found the maximal fixpoint | |||
3255 | // We do this by storing the value to class mapping, touching all instructions, | |||
3256 | // and redoing the iteration to see if anything changed. | |||
3257 | void NewGVN::verifyIterationSettled(Function &F) { | |||
3258 | #ifndef NDEBUG | |||
3259 | LLVM_DEBUG(dbgs() << "Beginning iteration verification\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Beginning iteration verification\n" ; } } while (false); | |||
3260 | if (DebugCounter::isCounterSet(VNCounter)) | |||
3261 | DebugCounter::setCounterValue(VNCounter, StartingVNCounter); | |||
3262 | ||||
3263 | // Note that we have to store the actual classes, as we may change existing | |||
3264 | // classes during iteration. This is because our memory iteration propagation | |||
3265 | // is not perfect, and so may waste a little work. But it should generate | |||
3266 | // exactly the same congruence classes we have now, with different IDs. | |||
3267 | std::map<const Value *, CongruenceClass> BeforeIteration; | |||
3268 | ||||
3269 | for (auto &KV : ValueToClass) { | |||
3270 | if (auto *I = dyn_cast<Instruction>(KV.first)) | |||
3271 | // Skip unused/dead instructions. | |||
3272 | if (InstrToDFSNum(I) == 0) | |||
3273 | continue; | |||
3274 | BeforeIteration.insert({KV.first, *KV.second}); | |||
3275 | } | |||
3276 | ||||
3277 | TouchedInstructions.set(); | |||
3278 | TouchedInstructions.reset(0); | |||
3279 | iterateTouchedInstructions(); | |||
3280 | DenseSet<std::pair<const CongruenceClass *, const CongruenceClass *>> | |||
3281 | EqualClasses; | |||
3282 | for (const auto &KV : ValueToClass) { | |||
3283 | if (auto *I = dyn_cast<Instruction>(KV.first)) | |||
3284 | // Skip unused/dead instructions. | |||
3285 | if (InstrToDFSNum(I) == 0) | |||
3286 | continue; | |||
3287 | // We could sink these uses, but i think this adds a bit of clarity here as | |||
3288 | // to what we are comparing. | |||
3289 | auto *BeforeCC = &BeforeIteration.find(KV.first)->second; | |||
3290 | auto *AfterCC = KV.second; | |||
3291 | // Note that the classes can't change at this point, so we memoize the set | |||
3292 | // that are equal. | |||
3293 | if (!EqualClasses.count({BeforeCC, AfterCC})) { | |||
3294 | assert(BeforeCC->isEquivalentTo(AfterCC) &&((BeforeCC->isEquivalentTo(AfterCC) && "Value number changed after main loop completed!" ) ? static_cast<void> (0) : __assert_fail ("BeforeCC->isEquivalentTo(AfterCC) && \"Value number changed after main loop completed!\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3295, __PRETTY_FUNCTION__)) | |||
3295 | "Value number changed after main loop completed!")((BeforeCC->isEquivalentTo(AfterCC) && "Value number changed after main loop completed!" ) ? static_cast<void> (0) : __assert_fail ("BeforeCC->isEquivalentTo(AfterCC) && \"Value number changed after main loop completed!\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3295, __PRETTY_FUNCTION__)); | |||
3296 | EqualClasses.insert({BeforeCC, AfterCC}); | |||
3297 | } | |||
3298 | } | |||
3299 | #endif | |||
3300 | } | |||
3301 | ||||
3302 | // Verify that for each store expression in the expression to class mapping, | |||
3303 | // only the latest appears, and multiple ones do not appear. | |||
3304 | // Because loads do not use the stored value when doing equality with stores, | |||
3305 | // if we don't erase the old store expressions from the table, a load can find | |||
3306 | // a no-longer valid StoreExpression. | |||
3307 | void NewGVN::verifyStoreExpressions() const { | |||
3308 | #ifndef NDEBUG | |||
3309 | // This is the only use of this, and it's not worth defining a complicated | |||
3310 | // densemapinfo hash/equality function for it. | |||
3311 | std::set< | |||
3312 | std::pair<const Value *, | |||
3313 | std::tuple<const Value *, const CongruenceClass *, Value *>>> | |||
3314 | StoreExpressionSet; | |||
3315 | for (const auto &KV : ExpressionToClass) { | |||
3316 | if (auto *SE = dyn_cast<StoreExpression>(KV.first)) { | |||
3317 | // Make sure a version that will conflict with loads is not already there | |||
3318 | auto Res = StoreExpressionSet.insert( | |||
3319 | {SE->getOperand(0), std::make_tuple(SE->getMemoryLeader(), KV.second, | |||
3320 | SE->getStoredValue())}); | |||
3321 | bool Okay = Res.second; | |||
3322 | // It's okay to have the same expression already in there if it is | |||
3323 | // identical in nature. | |||
3324 | // This can happen when the leader of the stored value changes over time. | |||
3325 | if (!Okay) | |||
3326 | Okay = (std::get<1>(Res.first->second) == KV.second) && | |||
3327 | (lookupOperandLeader(std::get<2>(Res.first->second)) == | |||
3328 | lookupOperandLeader(SE->getStoredValue())); | |||
3329 | assert(Okay && "Stored expression conflict exists in expression table")((Okay && "Stored expression conflict exists in expression table" ) ? static_cast<void> (0) : __assert_fail ("Okay && \"Stored expression conflict exists in expression table\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3329, __PRETTY_FUNCTION__)); | |||
3330 | auto *ValueExpr = ValueToExpression.lookup(SE->getStoreInst()); | |||
3331 | assert(ValueExpr && ValueExpr->equals(*SE) &&((ValueExpr && ValueExpr->equals(*SE) && "StoreExpression in ExpressionToClass is not latest " "StoreExpression for value") ? static_cast<void> (0) : __assert_fail ("ValueExpr && ValueExpr->equals(*SE) && \"StoreExpression in ExpressionToClass is not latest \" \"StoreExpression for value\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3333, __PRETTY_FUNCTION__)) | |||
3332 | "StoreExpression in ExpressionToClass is not latest "((ValueExpr && ValueExpr->equals(*SE) && "StoreExpression in ExpressionToClass is not latest " "StoreExpression for value") ? static_cast<void> (0) : __assert_fail ("ValueExpr && ValueExpr->equals(*SE) && \"StoreExpression in ExpressionToClass is not latest \" \"StoreExpression for value\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3333, __PRETTY_FUNCTION__)) | |||
3333 | "StoreExpression for value")((ValueExpr && ValueExpr->equals(*SE) && "StoreExpression in ExpressionToClass is not latest " "StoreExpression for value") ? static_cast<void> (0) : __assert_fail ("ValueExpr && ValueExpr->equals(*SE) && \"StoreExpression in ExpressionToClass is not latest \" \"StoreExpression for value\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3333, __PRETTY_FUNCTION__)); | |||
3334 | } | |||
3335 | } | |||
3336 | #endif | |||
3337 | } | |||
3338 | ||||
3339 | // This is the main value numbering loop, it iterates over the initial touched | |||
3340 | // instruction set, propagating value numbers, marking things touched, etc, | |||
3341 | // until the set of touched instructions is completely empty. | |||
3342 | void NewGVN::iterateTouchedInstructions() { | |||
3343 | unsigned int Iterations = 0; | |||
3344 | // Figure out where touchedinstructions starts | |||
3345 | int FirstInstr = TouchedInstructions.find_first(); | |||
3346 | // Nothing set, nothing to iterate, just return. | |||
3347 | if (FirstInstr == -1) | |||
3348 | return; | |||
3349 | const BasicBlock *LastBlock = getBlockForValue(InstrFromDFSNum(FirstInstr)); | |||
3350 | while (TouchedInstructions.any()) { | |||
3351 | ++Iterations; | |||
3352 | // Walk through all the instructions in all the blocks in RPO. | |||
3353 | // TODO: As we hit a new block, we should push and pop equalities into a | |||
3354 | // table lookupOperandLeader can use, to catch things PredicateInfo | |||
3355 | // might miss, like edge-only equivalences. | |||
3356 | for (unsigned InstrNum : TouchedInstructions.set_bits()) { | |||
3357 | ||||
3358 | // This instruction was found to be dead. We don't bother looking | |||
3359 | // at it again. | |||
3360 | if (InstrNum == 0) { | |||
3361 | TouchedInstructions.reset(InstrNum); | |||
3362 | continue; | |||
3363 | } | |||
3364 | ||||
3365 | Value *V = InstrFromDFSNum(InstrNum); | |||
3366 | const BasicBlock *CurrBlock = getBlockForValue(V); | |||
3367 | ||||
3368 | // If we hit a new block, do reachability processing. | |||
3369 | if (CurrBlock != LastBlock) { | |||
3370 | LastBlock = CurrBlock; | |||
3371 | bool BlockReachable = ReachableBlocks.count(CurrBlock); | |||
3372 | const auto &CurrInstRange = BlockInstRange.lookup(CurrBlock); | |||
3373 | ||||
3374 | // If it's not reachable, erase any touched instructions and move on. | |||
3375 | if (!BlockReachable) { | |||
3376 | TouchedInstructions.reset(CurrInstRange.first, CurrInstRange.second); | |||
3377 | LLVM_DEBUG(dbgs() << "Skipping instructions in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Skipping instructions in block " << getBlockName(CurrBlock) << " because it is unreachable\n" ; } } while (false) | |||
3378 | << getBlockName(CurrBlock)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Skipping instructions in block " << getBlockName(CurrBlock) << " because it is unreachable\n" ; } } while (false) | |||
3379 | << " because it is unreachable\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Skipping instructions in block " << getBlockName(CurrBlock) << " because it is unreachable\n" ; } } while (false); | |||
3380 | continue; | |||
3381 | } | |||
3382 | updateProcessedCount(CurrBlock); | |||
3383 | } | |||
3384 | // Reset after processing (because we may mark ourselves as touched when | |||
3385 | // we propagate equalities). | |||
3386 | TouchedInstructions.reset(InstrNum); | |||
3387 | ||||
3388 | if (auto *MP = dyn_cast<MemoryPhi>(V)) { | |||
3389 | LLVM_DEBUG(dbgs() << "Processing MemoryPhi " << *MP << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Processing MemoryPhi " << *MP << "\n"; } } while (false); | |||
3390 | valueNumberMemoryPhi(MP); | |||
3391 | } else if (auto *I = dyn_cast<Instruction>(V)) { | |||
3392 | valueNumberInstruction(I); | |||
3393 | } else { | |||
3394 | llvm_unreachable("Should have been a MemoryPhi or Instruction")::llvm::llvm_unreachable_internal("Should have been a MemoryPhi or Instruction" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3394); | |||
3395 | } | |||
3396 | updateProcessedCount(V); | |||
3397 | } | |||
3398 | } | |||
3399 | NumGVNMaxIterations = std::max(NumGVNMaxIterations.getValue(), Iterations); | |||
3400 | } | |||
3401 | ||||
3402 | // This is the main transformation entry point. | |||
3403 | bool NewGVN::runGVN() { | |||
3404 | if (DebugCounter::isCounterSet(VNCounter)) | |||
3405 | StartingVNCounter = DebugCounter::getCounterValue(VNCounter); | |||
3406 | bool Changed = false; | |||
3407 | NumFuncArgs = F.arg_size(); | |||
3408 | MSSAWalker = MSSA->getWalker(); | |||
3409 | SingletonDeadExpression = new (ExpressionAllocator) DeadExpression(); | |||
3410 | ||||
3411 | // Count number of instructions for sizing of hash tables, and come | |||
3412 | // up with a global dfs numbering for instructions. | |||
3413 | unsigned ICount = 1; | |||
3414 | // Add an empty instruction to account for the fact that we start at 1 | |||
3415 | DFSToInstr.emplace_back(nullptr); | |||
3416 | // Note: We want ideal RPO traversal of the blocks, which is not quite the | |||
3417 | // same as dominator tree order, particularly with regard whether backedges | |||
3418 | // get visited first or second, given a block with multiple successors. | |||
3419 | // If we visit in the wrong order, we will end up performing N times as many | |||
3420 | // iterations. | |||
3421 | // The dominator tree does guarantee that, for a given dom tree node, it's | |||
3422 | // parent must occur before it in the RPO ordering. Thus, we only need to sort | |||
3423 | // the siblings. | |||
3424 | ReversePostOrderTraversal<Function *> RPOT(&F); | |||
3425 | unsigned Counter = 0; | |||
3426 | for (auto &B : RPOT) { | |||
3427 | auto *Node = DT->getNode(B); | |||
3428 | assert(Node && "RPO and Dominator tree should have same reachability")((Node && "RPO and Dominator tree should have same reachability" ) ? static_cast<void> (0) : __assert_fail ("Node && \"RPO and Dominator tree should have same reachability\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3428, __PRETTY_FUNCTION__)); | |||
3429 | RPOOrdering[Node] = ++Counter; | |||
3430 | } | |||
3431 | // Sort dominator tree children arrays into RPO. | |||
3432 | for (auto &B : RPOT) { | |||
3433 | auto *Node = DT->getNode(B); | |||
3434 | if (Node->getChildren().size() > 1) | |||
3435 | llvm::sort(Node->begin(), Node->end(), | |||
3436 | [&](const DomTreeNode *A, const DomTreeNode *B) { | |||
3437 | return RPOOrdering[A] < RPOOrdering[B]; | |||
3438 | }); | |||
3439 | } | |||
3440 | ||||
3441 | // Now a standard depth first ordering of the domtree is equivalent to RPO. | |||
3442 | for (auto DTN : depth_first(DT->getRootNode())) { | |||
3443 | BasicBlock *B = DTN->getBlock(); | |||
3444 | const auto &BlockRange = assignDFSNumbers(B, ICount); | |||
3445 | BlockInstRange.insert({B, BlockRange}); | |||
3446 | ICount += BlockRange.second - BlockRange.first; | |||
3447 | } | |||
3448 | initializeCongruenceClasses(F); | |||
3449 | ||||
3450 | TouchedInstructions.resize(ICount); | |||
3451 | // Ensure we don't end up resizing the expressionToClass map, as | |||
3452 | // that can be quite expensive. At most, we have one expression per | |||
3453 | // instruction. | |||
3454 | ExpressionToClass.reserve(ICount); | |||
3455 | ||||
3456 | // Initialize the touched instructions to include the entry block. | |||
3457 | const auto &InstRange = BlockInstRange.lookup(&F.getEntryBlock()); | |||
3458 | TouchedInstructions.set(InstRange.first, InstRange.second); | |||
3459 | LLVM_DEBUG(dbgs() << "Block " << getBlockName(&F.getEntryBlock())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Block " << getBlockName( &F.getEntryBlock()) << " marked reachable\n"; } } while (false) | |||
3460 | << " marked reachable\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Block " << getBlockName( &F.getEntryBlock()) << " marked reachable\n"; } } while (false); | |||
3461 | ReachableBlocks.insert(&F.getEntryBlock()); | |||
3462 | ||||
3463 | iterateTouchedInstructions(); | |||
3464 | verifyMemoryCongruency(); | |||
3465 | verifyIterationSettled(F); | |||
3466 | verifyStoreExpressions(); | |||
3467 | ||||
3468 | Changed |= eliminateInstructions(F); | |||
3469 | ||||
3470 | // Delete all instructions marked for deletion. | |||
3471 | for (Instruction *ToErase : InstructionsToErase) { | |||
3472 | if (!ToErase->use_empty()) | |||
3473 | ToErase->replaceAllUsesWith(UndefValue::get(ToErase->getType())); | |||
3474 | ||||
3475 | assert(ToErase->getParent() &&((ToErase->getParent() && "BB containing ToErase deleted unexpectedly!" ) ? static_cast<void> (0) : __assert_fail ("ToErase->getParent() && \"BB containing ToErase deleted unexpectedly!\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3476, __PRETTY_FUNCTION__)) | |||
3476 | "BB containing ToErase deleted unexpectedly!")((ToErase->getParent() && "BB containing ToErase deleted unexpectedly!" ) ? static_cast<void> (0) : __assert_fail ("ToErase->getParent() && \"BB containing ToErase deleted unexpectedly!\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3476, __PRETTY_FUNCTION__)); | |||
3477 | ToErase->eraseFromParent(); | |||
3478 | } | |||
3479 | Changed |= !InstructionsToErase.empty(); | |||
3480 | ||||
3481 | // Delete all unreachable blocks. | |||
3482 | auto UnreachableBlockPred = [&](const BasicBlock &BB) { | |||
3483 | return !ReachableBlocks.count(&BB); | |||
3484 | }; | |||
3485 | ||||
3486 | for (auto &BB : make_filter_range(F, UnreachableBlockPred)) { | |||
3487 | LLVM_DEBUG(dbgs() << "We believe block " << getBlockName(&BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "We believe block " << getBlockName (&BB) << " is unreachable\n"; } } while (false) | |||
3488 | << " is unreachable\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "We believe block " << getBlockName (&BB) << " is unreachable\n"; } } while (false); | |||
3489 | deleteInstructionsInBlock(&BB); | |||
3490 | Changed = true; | |||
3491 | } | |||
3492 | ||||
3493 | cleanupTables(); | |||
3494 | return Changed; | |||
3495 | } | |||
3496 | ||||
3497 | struct NewGVN::ValueDFS { | |||
3498 | int DFSIn = 0; | |||
3499 | int DFSOut = 0; | |||
3500 | int LocalNum = 0; | |||
3501 | ||||
3502 | // Only one of Def and U will be set. | |||
3503 | // The bool in the Def tells us whether the Def is the stored value of a | |||
3504 | // store. | |||
3505 | PointerIntPair<Value *, 1, bool> Def; | |||
3506 | Use *U = nullptr; | |||
3507 | ||||
3508 | bool operator<(const ValueDFS &Other) const { | |||
3509 | // It's not enough that any given field be less than - we have sets | |||
3510 | // of fields that need to be evaluated together to give a proper ordering. | |||
3511 | // For example, if you have; | |||
3512 | // DFS (1, 3) | |||
3513 | // Val 0 | |||
3514 | // DFS (1, 2) | |||
3515 | // Val 50 | |||
3516 | // We want the second to be less than the first, but if we just go field | |||
3517 | // by field, we will get to Val 0 < Val 50 and say the first is less than | |||
3518 | // the second. We only want it to be less than if the DFS orders are equal. | |||
3519 | // | |||
3520 | // Each LLVM instruction only produces one value, and thus the lowest-level | |||
3521 | // differentiator that really matters for the stack (and what we use as as a | |||
3522 | // replacement) is the local dfs number. | |||
3523 | // Everything else in the structure is instruction level, and only affects | |||
3524 | // the order in which we will replace operands of a given instruction. | |||
3525 | // | |||
3526 | // For a given instruction (IE things with equal dfsin, dfsout, localnum), | |||
3527 | // the order of replacement of uses does not matter. | |||
3528 | // IE given, | |||
3529 | // a = 5 | |||
3530 | // b = a + a | |||
3531 | // When you hit b, you will have two valuedfs with the same dfsin, out, and | |||
3532 | // localnum. | |||
3533 | // The .val will be the same as well. | |||
3534 | // The .u's will be different. | |||
3535 | // You will replace both, and it does not matter what order you replace them | |||
3536 | // in (IE whether you replace operand 2, then operand 1, or operand 1, then | |||
3537 | // operand 2). | |||
3538 | // Similarly for the case of same dfsin, dfsout, localnum, but different | |||
3539 | // .val's | |||
3540 | // a = 5 | |||
3541 | // b = 6 | |||
3542 | // c = a + b | |||
3543 | // in c, we will a valuedfs for a, and one for b,with everything the same | |||
3544 | // but .val and .u. | |||
3545 | // It does not matter what order we replace these operands in. | |||
3546 | // You will always end up with the same IR, and this is guaranteed. | |||
3547 | return std::tie(DFSIn, DFSOut, LocalNum, Def, U) < | |||
3548 | std::tie(Other.DFSIn, Other.DFSOut, Other.LocalNum, Other.Def, | |||
3549 | Other.U); | |||
3550 | } | |||
3551 | }; | |||
3552 | ||||
3553 | // This function converts the set of members for a congruence class from values, | |||
3554 | // to sets of defs and uses with associated DFS info. The total number of | |||
3555 | // reachable uses for each value is stored in UseCount, and instructions that | |||
3556 | // seem | |||
3557 | // dead (have no non-dead uses) are stored in ProbablyDead. | |||
3558 | void NewGVN::convertClassToDFSOrdered( | |||
3559 | const CongruenceClass &Dense, SmallVectorImpl<ValueDFS> &DFSOrderedSet, | |||
3560 | DenseMap<const Value *, unsigned int> &UseCounts, | |||
3561 | SmallPtrSetImpl<Instruction *> &ProbablyDead) const { | |||
3562 | for (auto D : Dense) { | |||
3563 | // First add the value. | |||
3564 | BasicBlock *BB = getBlockForValue(D); | |||
3565 | // Constants are handled prior to ever calling this function, so | |||
3566 | // we should only be left with instructions as members. | |||
3567 | assert(BB && "Should have figured out a basic block for value")((BB && "Should have figured out a basic block for value" ) ? static_cast<void> (0) : __assert_fail ("BB && \"Should have figured out a basic block for value\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3567, __PRETTY_FUNCTION__)); | |||
3568 | ValueDFS VDDef; | |||
3569 | DomTreeNode *DomNode = DT->getNode(BB); | |||
3570 | VDDef.DFSIn = DomNode->getDFSNumIn(); | |||
3571 | VDDef.DFSOut = DomNode->getDFSNumOut(); | |||
3572 | // If it's a store, use the leader of the value operand, if it's always | |||
3573 | // available, or the value operand. TODO: We could do dominance checks to | |||
3574 | // find a dominating leader, but not worth it ATM. | |||
3575 | if (auto *SI = dyn_cast<StoreInst>(D)) { | |||
3576 | auto Leader = lookupOperandLeader(SI->getValueOperand()); | |||
3577 | if (alwaysAvailable(Leader)) { | |||
3578 | VDDef.Def.setPointer(Leader); | |||
3579 | } else { | |||
3580 | VDDef.Def.setPointer(SI->getValueOperand()); | |||
3581 | VDDef.Def.setInt(true); | |||
3582 | } | |||
3583 | } else { | |||
3584 | VDDef.Def.setPointer(D); | |||
3585 | } | |||
3586 | assert(isa<Instruction>(D) &&((isa<Instruction>(D) && "The dense set member should always be an instruction" ) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(D) && \"The dense set member should always be an instruction\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3587, __PRETTY_FUNCTION__)) | |||
3587 | "The dense set member should always be an instruction")((isa<Instruction>(D) && "The dense set member should always be an instruction" ) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(D) && \"The dense set member should always be an instruction\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3587, __PRETTY_FUNCTION__)); | |||
3588 | Instruction *Def = cast<Instruction>(D); | |||
3589 | VDDef.LocalNum = InstrToDFSNum(D); | |||
3590 | DFSOrderedSet.push_back(VDDef); | |||
3591 | // If there is a phi node equivalent, add it | |||
3592 | if (auto *PN = RealToTemp.lookup(Def)) { | |||
3593 | auto *PHIE = | |||
3594 | dyn_cast_or_null<PHIExpression>(ValueToExpression.lookup(Def)); | |||
3595 | if (PHIE) { | |||
3596 | VDDef.Def.setInt(false); | |||
3597 | VDDef.Def.setPointer(PN); | |||
3598 | VDDef.LocalNum = 0; | |||
3599 | DFSOrderedSet.push_back(VDDef); | |||
3600 | } | |||
3601 | } | |||
3602 | ||||
3603 | unsigned int UseCount = 0; | |||
3604 | // Now add the uses. | |||
3605 | for (auto &U : Def->uses()) { | |||
3606 | if (auto *I = dyn_cast<Instruction>(U.getUser())) { | |||
3607 | // Don't try to replace into dead uses | |||
3608 | if (InstructionsToErase.count(I)) | |||
3609 | continue; | |||
3610 | ValueDFS VDUse; | |||
3611 | // Put the phi node uses in the incoming block. | |||
3612 | BasicBlock *IBlock; | |||
3613 | if (auto *P = dyn_cast<PHINode>(I)) { | |||
3614 | IBlock = P->getIncomingBlock(U); | |||
3615 | // Make phi node users appear last in the incoming block | |||
3616 | // they are from. | |||
3617 | VDUse.LocalNum = InstrDFS.size() + 1; | |||
3618 | } else { | |||
3619 | IBlock = getBlockForValue(I); | |||
3620 | VDUse.LocalNum = InstrToDFSNum(I); | |||
3621 | } | |||
3622 | ||||
3623 | // Skip uses in unreachable blocks, as we're going | |||
3624 | // to delete them. | |||
3625 | if (ReachableBlocks.count(IBlock) == 0) | |||
3626 | continue; | |||
3627 | ||||
3628 | DomTreeNode *DomNode = DT->getNode(IBlock); | |||
3629 | VDUse.DFSIn = DomNode->getDFSNumIn(); | |||
3630 | VDUse.DFSOut = DomNode->getDFSNumOut(); | |||
3631 | VDUse.U = &U; | |||
3632 | ++UseCount; | |||
3633 | DFSOrderedSet.emplace_back(VDUse); | |||
3634 | } | |||
3635 | } | |||
3636 | ||||
3637 | // If there are no uses, it's probably dead (but it may have side-effects, | |||
3638 | // so not definitely dead. Otherwise, store the number of uses so we can | |||
3639 | // track if it becomes dead later). | |||
3640 | if (UseCount == 0) | |||
3641 | ProbablyDead.insert(Def); | |||
3642 | else | |||
3643 | UseCounts[Def] = UseCount; | |||
3644 | } | |||
3645 | } | |||
3646 | ||||
3647 | // This function converts the set of members for a congruence class from values, | |||
3648 | // to the set of defs for loads and stores, with associated DFS info. | |||
3649 | void NewGVN::convertClassToLoadsAndStores( | |||
3650 | const CongruenceClass &Dense, | |||
3651 | SmallVectorImpl<ValueDFS> &LoadsAndStores) const { | |||
3652 | for (auto D : Dense) { | |||
3653 | if (!isa<LoadInst>(D) && !isa<StoreInst>(D)) | |||
3654 | continue; | |||
3655 | ||||
3656 | BasicBlock *BB = getBlockForValue(D); | |||
3657 | ValueDFS VD; | |||
3658 | DomTreeNode *DomNode = DT->getNode(BB); | |||
3659 | VD.DFSIn = DomNode->getDFSNumIn(); | |||
3660 | VD.DFSOut = DomNode->getDFSNumOut(); | |||
3661 | VD.Def.setPointer(D); | |||
3662 | ||||
3663 | // If it's an instruction, use the real local dfs number. | |||
3664 | if (auto *I = dyn_cast<Instruction>(D)) | |||
3665 | VD.LocalNum = InstrToDFSNum(I); | |||
3666 | else | |||
3667 | llvm_unreachable("Should have been an instruction")::llvm::llvm_unreachable_internal("Should have been an instruction" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3667); | |||
3668 | ||||
3669 | LoadsAndStores.emplace_back(VD); | |||
3670 | } | |||
3671 | } | |||
3672 | ||||
3673 | static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) { | |||
3674 | patchReplacementInstruction(I, Repl); | |||
3675 | I->replaceAllUsesWith(Repl); | |||
3676 | } | |||
3677 | ||||
3678 | void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) { | |||
3679 | LLVM_DEBUG(dbgs() << " BasicBlock Dead:" << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << " BasicBlock Dead:" << * BB; } } while (false); | |||
3680 | ++NumGVNBlocksDeleted; | |||
3681 | ||||
3682 | // Delete the instructions backwards, as it has a reduced likelihood of having | |||
3683 | // to update as many def-use and use-def chains. Start after the terminator. | |||
3684 | auto StartPoint = BB->rbegin(); | |||
3685 | ++StartPoint; | |||
3686 | // Note that we explicitly recalculate BB->rend() on each iteration, | |||
3687 | // as it may change when we remove the first instruction. | |||
3688 | for (BasicBlock::reverse_iterator I(StartPoint); I != BB->rend();) { | |||
3689 | Instruction &Inst = *I++; | |||
3690 | if (!Inst.use_empty()) | |||
3691 | Inst.replaceAllUsesWith(UndefValue::get(Inst.getType())); | |||
3692 | if (isa<LandingPadInst>(Inst)) | |||
3693 | continue; | |||
3694 | ||||
3695 | Inst.eraseFromParent(); | |||
3696 | ++NumGVNInstrDeleted; | |||
3697 | } | |||
3698 | // Now insert something that simplifycfg will turn into an unreachable. | |||
3699 | Type *Int8Ty = Type::getInt8Ty(BB->getContext()); | |||
3700 | new StoreInst(UndefValue::get(Int8Ty), | |||
3701 | Constant::getNullValue(Int8Ty->getPointerTo()), | |||
3702 | BB->getTerminator()); | |||
3703 | } | |||
3704 | ||||
3705 | void NewGVN::markInstructionForDeletion(Instruction *I) { | |||
3706 | LLVM_DEBUG(dbgs() << "Marking " << *I << " for deletion\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Marking " << *I << " for deletion\n"; } } while (false); | |||
3707 | InstructionsToErase.insert(I); | |||
3708 | } | |||
3709 | ||||
3710 | void NewGVN::replaceInstruction(Instruction *I, Value *V) { | |||
3711 | LLVM_DEBUG(dbgs() << "Replacing " << *I << " with " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Replacing " << *I << " with " << *V << "\n"; } } while (false); | |||
3712 | patchAndReplaceAllUsesWith(I, V); | |||
3713 | // We save the actual erasing to avoid invalidating memory | |||
3714 | // dependencies until we are done with everything. | |||
3715 | markInstructionForDeletion(I); | |||
3716 | } | |||
3717 | ||||
3718 | namespace { | |||
3719 | ||||
3720 | // This is a stack that contains both the value and dfs info of where | |||
3721 | // that value is valid. | |||
3722 | class ValueDFSStack { | |||
3723 | public: | |||
3724 | Value *back() const { return ValueStack.back(); } | |||
3725 | std::pair<int, int> dfs_back() const { return DFSStack.back(); } | |||
3726 | ||||
3727 | void push_back(Value *V, int DFSIn, int DFSOut) { | |||
3728 | ValueStack.emplace_back(V); | |||
3729 | DFSStack.emplace_back(DFSIn, DFSOut); | |||
3730 | } | |||
3731 | ||||
3732 | bool empty() const { return DFSStack.empty(); } | |||
3733 | ||||
3734 | bool isInScope(int DFSIn, int DFSOut) const { | |||
3735 | if (empty()) | |||
3736 | return false; | |||
3737 | return DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second; | |||
3738 | } | |||
3739 | ||||
3740 | void popUntilDFSScope(int DFSIn, int DFSOut) { | |||
3741 | ||||
3742 | // These two should always be in sync at this point. | |||
3743 | assert(ValueStack.size() == DFSStack.size() &&((ValueStack.size() == DFSStack.size() && "Mismatch between ValueStack and DFSStack" ) ? static_cast<void> (0) : __assert_fail ("ValueStack.size() == DFSStack.size() && \"Mismatch between ValueStack and DFSStack\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3744, __PRETTY_FUNCTION__)) | |||
3744 | "Mismatch between ValueStack and DFSStack")((ValueStack.size() == DFSStack.size() && "Mismatch between ValueStack and DFSStack" ) ? static_cast<void> (0) : __assert_fail ("ValueStack.size() == DFSStack.size() && \"Mismatch between ValueStack and DFSStack\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3744, __PRETTY_FUNCTION__)); | |||
3745 | while ( | |||
3746 | !DFSStack.empty() && | |||
3747 | !(DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second)) { | |||
3748 | DFSStack.pop_back(); | |||
3749 | ValueStack.pop_back(); | |||
3750 | } | |||
3751 | } | |||
3752 | ||||
3753 | private: | |||
3754 | SmallVector<Value *, 8> ValueStack; | |||
3755 | SmallVector<std::pair<int, int>, 8> DFSStack; | |||
3756 | }; | |||
3757 | ||||
3758 | } // end anonymous namespace | |||
3759 | ||||
3760 | // Given an expression, get the congruence class for it. | |||
3761 | CongruenceClass *NewGVN::getClassForExpression(const Expression *E) const { | |||
3762 | if (auto *VE = dyn_cast<VariableExpression>(E)) | |||
3763 | return ValueToClass.lookup(VE->getVariableValue()); | |||
3764 | else if (isa<DeadExpression>(E)) | |||
3765 | return TOPClass; | |||
3766 | return ExpressionToClass.lookup(E); | |||
3767 | } | |||
3768 | ||||
3769 | // Given a value and a basic block we are trying to see if it is available in, | |||
3770 | // see if the value has a leader available in that block. | |||
3771 | Value *NewGVN::findPHIOfOpsLeader(const Expression *E, | |||
3772 | const Instruction *OrigInst, | |||
3773 | const BasicBlock *BB) const { | |||
3774 | // It would already be constant if we could make it constant | |||
3775 | if (auto *CE = dyn_cast<ConstantExpression>(E)) | |||
3776 | return CE->getConstantValue(); | |||
3777 | if (auto *VE = dyn_cast<VariableExpression>(E)) { | |||
3778 | auto *V = VE->getVariableValue(); | |||
3779 | if (alwaysAvailable(V) || DT->dominates(getBlockForValue(V), BB)) | |||
3780 | return VE->getVariableValue(); | |||
3781 | } | |||
3782 | ||||
3783 | auto *CC = getClassForExpression(E); | |||
3784 | if (!CC) | |||
3785 | return nullptr; | |||
3786 | if (alwaysAvailable(CC->getLeader())) | |||
3787 | return CC->getLeader(); | |||
3788 | ||||
3789 | for (auto Member : *CC) { | |||
3790 | auto *MemberInst = dyn_cast<Instruction>(Member); | |||
3791 | if (MemberInst == OrigInst) | |||
3792 | continue; | |||
3793 | // Anything that isn't an instruction is always available. | |||
3794 | if (!MemberInst) | |||
3795 | return Member; | |||
3796 | if (DT->dominates(getBlockForValue(MemberInst), BB)) | |||
3797 | return Member; | |||
3798 | } | |||
3799 | return nullptr; | |||
3800 | } | |||
3801 | ||||
3802 | bool NewGVN::eliminateInstructions(Function &F) { | |||
3803 | // This is a non-standard eliminator. The normal way to eliminate is | |||
3804 | // to walk the dominator tree in order, keeping track of available | |||
3805 | // values, and eliminating them. However, this is mildly | |||
3806 | // pointless. It requires doing lookups on every instruction, | |||
3807 | // regardless of whether we will ever eliminate it. For | |||
3808 | // instructions part of most singleton congruence classes, we know we | |||
3809 | // will never eliminate them. | |||
3810 | ||||
3811 | // Instead, this eliminator looks at the congruence classes directly, sorts | |||
3812 | // them into a DFS ordering of the dominator tree, and then we just | |||
3813 | // perform elimination straight on the sets by walking the congruence | |||
3814 | // class member uses in order, and eliminate the ones dominated by the | |||
3815 | // last member. This is worst case O(E log E) where E = number of | |||
3816 | // instructions in a single congruence class. In theory, this is all | |||
3817 | // instructions. In practice, it is much faster, as most instructions are | |||
3818 | // either in singleton congruence classes or can't possibly be eliminated | |||
3819 | // anyway (if there are no overlapping DFS ranges in class). | |||
3820 | // When we find something not dominated, it becomes the new leader | |||
3821 | // for elimination purposes. | |||
3822 | // TODO: If we wanted to be faster, We could remove any members with no | |||
3823 | // overlapping ranges while sorting, as we will never eliminate anything | |||
3824 | // with those members, as they don't dominate anything else in our set. | |||
3825 | ||||
3826 | bool AnythingReplaced = false; | |||
3827 | ||||
3828 | // Since we are going to walk the domtree anyway, and we can't guarantee the | |||
3829 | // DFS numbers are updated, we compute some ourselves. | |||
3830 | DT->updateDFSNumbers(); | |||
3831 | ||||
3832 | // Go through all of our phi nodes, and kill the arguments associated with | |||
3833 | // unreachable edges. | |||
3834 | auto ReplaceUnreachablePHIArgs = [&](PHINode *PHI, BasicBlock *BB) { | |||
3835 | for (auto &Operand : PHI->incoming_values()) | |||
3836 | if (!ReachableEdges.count({PHI->getIncomingBlock(Operand), BB})) { | |||
3837 | LLVM_DEBUG(dbgs() << "Replacing incoming value of " << PHIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Replacing incoming value of " << PHI << " for block " << getBlockName(PHI->getIncomingBlock (Operand)) << " with undef due to it being unreachable\n" ; } } while (false) | |||
3838 | << " for block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Replacing incoming value of " << PHI << " for block " << getBlockName(PHI->getIncomingBlock (Operand)) << " with undef due to it being unreachable\n" ; } } while (false) | |||
3839 | << getBlockName(PHI->getIncomingBlock(Operand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Replacing incoming value of " << PHI << " for block " << getBlockName(PHI->getIncomingBlock (Operand)) << " with undef due to it being unreachable\n" ; } } while (false) | |||
3840 | << " with undef due to it being unreachable\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Replacing incoming value of " << PHI << " for block " << getBlockName(PHI->getIncomingBlock (Operand)) << " with undef due to it being unreachable\n" ; } } while (false); | |||
3841 | Operand.set(UndefValue::get(PHI->getType())); | |||
3842 | } | |||
3843 | }; | |||
3844 | // Replace unreachable phi arguments. | |||
3845 | // At this point, RevisitOnReachabilityChange only contains: | |||
3846 | // | |||
3847 | // 1. PHIs | |||
3848 | // 2. Temporaries that will convert to PHIs | |||
3849 | // 3. Operations that are affected by an unreachable edge but do not fit into | |||
3850 | // 1 or 2 (rare). | |||
3851 | // So it is a slight overshoot of what we want. We could make it exact by | |||
3852 | // using two SparseBitVectors per block. | |||
3853 | DenseMap<const BasicBlock *, unsigned> ReachablePredCount; | |||
3854 | for (auto &KV : ReachableEdges) | |||
3855 | ReachablePredCount[KV.getEnd()]++; | |||
3856 | for (auto &BBPair : RevisitOnReachabilityChange) { | |||
3857 | for (auto InstNum : BBPair.second) { | |||
3858 | auto *Inst = InstrFromDFSNum(InstNum); | |||
3859 | auto *PHI = dyn_cast<PHINode>(Inst); | |||
3860 | PHI = PHI ? PHI : dyn_cast_or_null<PHINode>(RealToTemp.lookup(Inst)); | |||
3861 | if (!PHI) | |||
3862 | continue; | |||
3863 | auto *BB = BBPair.first; | |||
3864 | if (ReachablePredCount.lookup(BB) != PHI->getNumIncomingValues()) | |||
3865 | ReplaceUnreachablePHIArgs(PHI, BB); | |||
3866 | } | |||
3867 | } | |||
3868 | ||||
3869 | // Map to store the use counts | |||
3870 | DenseMap<const Value *, unsigned int> UseCounts; | |||
3871 | for (auto *CC : reverse(CongruenceClasses)) { | |||
3872 | LLVM_DEBUG(dbgs() << "Eliminating in congruence class " << CC->getID()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Eliminating in congruence class " << CC->getID() << "\n"; } } while (false) | |||
3873 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Eliminating in congruence class " << CC->getID() << "\n"; } } while (false); | |||
3874 | // Track the equivalent store info so we can decide whether to try | |||
3875 | // dead store elimination. | |||
3876 | SmallVector<ValueDFS, 8> PossibleDeadStores; | |||
3877 | SmallPtrSet<Instruction *, 8> ProbablyDead; | |||
3878 | if (CC->isDead() || CC->empty()) | |||
3879 | continue; | |||
3880 | // Everything still in the TOP class is unreachable or dead. | |||
3881 | if (CC == TOPClass) { | |||
3882 | for (auto M : *CC) { | |||
3883 | auto *VTE = ValueToExpression.lookup(M); | |||
3884 | if (VTE && isa<DeadExpression>(VTE)) | |||
3885 | markInstructionForDeletion(cast<Instruction>(M)); | |||
3886 | assert((!ReachableBlocks.count(cast<Instruction>(M)->getParent()) ||(((!ReachableBlocks.count(cast<Instruction>(M)->getParent ()) || InstructionsToErase.count(cast<Instruction>(M))) && "Everything in TOP should be unreachable or dead at this " "point") ? static_cast<void> (0) : __assert_fail ("(!ReachableBlocks.count(cast<Instruction>(M)->getParent()) || InstructionsToErase.count(cast<Instruction>(M))) && \"Everything in TOP should be unreachable or dead at this \" \"point\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3889, __PRETTY_FUNCTION__)) | |||
3887 | InstructionsToErase.count(cast<Instruction>(M))) &&(((!ReachableBlocks.count(cast<Instruction>(M)->getParent ()) || InstructionsToErase.count(cast<Instruction>(M))) && "Everything in TOP should be unreachable or dead at this " "point") ? static_cast<void> (0) : __assert_fail ("(!ReachableBlocks.count(cast<Instruction>(M)->getParent()) || InstructionsToErase.count(cast<Instruction>(M))) && \"Everything in TOP should be unreachable or dead at this \" \"point\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3889, __PRETTY_FUNCTION__)) | |||
3888 | "Everything in TOP should be unreachable or dead at this "(((!ReachableBlocks.count(cast<Instruction>(M)->getParent ()) || InstructionsToErase.count(cast<Instruction>(M))) && "Everything in TOP should be unreachable or dead at this " "point") ? static_cast<void> (0) : __assert_fail ("(!ReachableBlocks.count(cast<Instruction>(M)->getParent()) || InstructionsToErase.count(cast<Instruction>(M))) && \"Everything in TOP should be unreachable or dead at this \" \"point\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3889, __PRETTY_FUNCTION__)) | |||
3889 | "point")(((!ReachableBlocks.count(cast<Instruction>(M)->getParent ()) || InstructionsToErase.count(cast<Instruction>(M))) && "Everything in TOP should be unreachable or dead at this " "point") ? static_cast<void> (0) : __assert_fail ("(!ReachableBlocks.count(cast<Instruction>(M)->getParent()) || InstructionsToErase.count(cast<Instruction>(M))) && \"Everything in TOP should be unreachable or dead at this \" \"point\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3889, __PRETTY_FUNCTION__)); | |||
3890 | } | |||
3891 | continue; | |||
3892 | } | |||
3893 | ||||
3894 | assert(CC->getLeader() && "We should have had a leader")((CC->getLeader() && "We should have had a leader" ) ? static_cast<void> (0) : __assert_fail ("CC->getLeader() && \"We should have had a leader\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3894, __PRETTY_FUNCTION__)); | |||
3895 | // If this is a leader that is always available, and it's a | |||
3896 | // constant or has no equivalences, just replace everything with | |||
3897 | // it. We then update the congruence class with whatever members | |||
3898 | // are left. | |||
3899 | Value *Leader = | |||
3900 | CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader(); | |||
3901 | if (alwaysAvailable(Leader)) { | |||
3902 | CongruenceClass::MemberSet MembersLeft; | |||
3903 | for (auto M : *CC) { | |||
3904 | Value *Member = M; | |||
3905 | // Void things have no uses we can replace. | |||
3906 | if (Member == Leader || !isa<Instruction>(Member) || | |||
3907 | Member->getType()->isVoidTy()) { | |||
3908 | MembersLeft.insert(Member); | |||
3909 | continue; | |||
3910 | } | |||
3911 | LLVM_DEBUG(dbgs() << "Found replacement " << *(Leader) << " for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Found replacement " << * (Leader) << " for " << *Member << "\n"; } } while (false) | |||
3912 | << *Member << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Found replacement " << * (Leader) << " for " << *Member << "\n"; } } while (false); | |||
3913 | auto *I = cast<Instruction>(Member); | |||
3914 | assert(Leader != I && "About to accidentally remove our leader")((Leader != I && "About to accidentally remove our leader" ) ? static_cast<void> (0) : __assert_fail ("Leader != I && \"About to accidentally remove our leader\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 3914, __PRETTY_FUNCTION__)); | |||
3915 | replaceInstruction(I, Leader); | |||
3916 | AnythingReplaced = true; | |||
3917 | } | |||
3918 | CC->swap(MembersLeft); | |||
3919 | } else { | |||
3920 | // If this is a singleton, we can skip it. | |||
3921 | if (CC->size() != 1 || RealToTemp.count(Leader)) { | |||
3922 | // This is a stack because equality replacement/etc may place | |||
3923 | // constants in the middle of the member list, and we want to use | |||
3924 | // those constant values in preference to the current leader, over | |||
3925 | // the scope of those constants. | |||
3926 | ValueDFSStack EliminationStack; | |||
3927 | ||||
3928 | // Convert the members to DFS ordered sets and then merge them. | |||
3929 | SmallVector<ValueDFS, 8> DFSOrderedSet; | |||
3930 | convertClassToDFSOrdered(*CC, DFSOrderedSet, UseCounts, ProbablyDead); | |||
3931 | ||||
3932 | // Sort the whole thing. | |||
3933 | llvm::sort(DFSOrderedSet); | |||
3934 | for (auto &VD : DFSOrderedSet) { | |||
3935 | int MemberDFSIn = VD.DFSIn; | |||
3936 | int MemberDFSOut = VD.DFSOut; | |||
3937 | Value *Def = VD.Def.getPointer(); | |||
3938 | bool FromStore = VD.Def.getInt(); | |||
3939 | Use *U = VD.U; | |||
3940 | // We ignore void things because we can't get a value from them. | |||
3941 | if (Def && Def->getType()->isVoidTy()) | |||
3942 | continue; | |||
3943 | auto *DefInst = dyn_cast_or_null<Instruction>(Def); | |||
3944 | if (DefInst && AllTempInstructions.count(DefInst)) { | |||
3945 | auto *PN = cast<PHINode>(DefInst); | |||
3946 | ||||
3947 | // If this is a value phi and that's the expression we used, insert | |||
3948 | // it into the program | |||
3949 | // remove from temp instruction list. | |||
3950 | AllTempInstructions.erase(PN); | |||
3951 | auto *DefBlock = getBlockForValue(Def); | |||
3952 | LLVM_DEBUG(dbgs() << "Inserting fully real phi of ops" << *Defdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Inserting fully real phi of ops" << *Def << " into block " << getBlockName( getBlockForValue(Def)) << "\n"; } } while (false) | |||
3953 | << " into block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Inserting fully real phi of ops" << *Def << " into block " << getBlockName( getBlockForValue(Def)) << "\n"; } } while (false) | |||
3954 | << getBlockName(getBlockForValue(Def)) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Inserting fully real phi of ops" << *Def << " into block " << getBlockName( getBlockForValue(Def)) << "\n"; } } while (false); | |||
3955 | PN->insertBefore(&DefBlock->front()); | |||
3956 | Def = PN; | |||
3957 | NumGVNPHIOfOpsEliminations++; | |||
3958 | } | |||
3959 | ||||
3960 | if (EliminationStack.empty()) { | |||
3961 | LLVM_DEBUG(dbgs() << "Elimination Stack is empty\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Elimination Stack is empty\n"; } } while (false); | |||
3962 | } else { | |||
3963 | LLVM_DEBUG(dbgs() << "Elimination Stack Top DFS numbers are ("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Elimination Stack Top DFS numbers are (" << EliminationStack.dfs_back().first << "," << EliminationStack.dfs_back().second << ")\n"; } } while (false) | |||
3964 | << EliminationStack.dfs_back().first << ","do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Elimination Stack Top DFS numbers are (" << EliminationStack.dfs_back().first << "," << EliminationStack.dfs_back().second << ")\n"; } } while (false) | |||
3965 | << EliminationStack.dfs_back().second << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Elimination Stack Top DFS numbers are (" << EliminationStack.dfs_back().first << "," << EliminationStack.dfs_back().second << ")\n"; } } while (false); | |||
3966 | } | |||
3967 | ||||
3968 | LLVM_DEBUG(dbgs() << "Current DFS numbers are (" << MemberDFSIn << ","do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Current DFS numbers are (" << MemberDFSIn << "," << MemberDFSOut << ")\n" ; } } while (false) | |||
3969 | << MemberDFSOut << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Current DFS numbers are (" << MemberDFSIn << "," << MemberDFSOut << ")\n" ; } } while (false); | |||
3970 | // First, we see if we are out of scope or empty. If so, | |||
3971 | // and there equivalences, we try to replace the top of | |||
3972 | // stack with equivalences (if it's on the stack, it must | |||
3973 | // not have been eliminated yet). | |||
3974 | // Then we synchronize to our current scope, by | |||
3975 | // popping until we are back within a DFS scope that | |||
3976 | // dominates the current member. | |||
3977 | // Then, what happens depends on a few factors | |||
3978 | // If the stack is now empty, we need to push | |||
3979 | // If we have a constant or a local equivalence we want to | |||
3980 | // start using, we also push. | |||
3981 | // Otherwise, we walk along, processing members who are | |||
3982 | // dominated by this scope, and eliminate them. | |||
3983 | bool ShouldPush = Def && EliminationStack.empty(); | |||
3984 | bool OutOfScope = | |||
3985 | !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut); | |||
3986 | ||||
3987 | if (OutOfScope || ShouldPush) { | |||
3988 | // Sync to our current scope. | |||
3989 | EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut); | |||
3990 | bool ShouldPush = Def && EliminationStack.empty(); | |||
3991 | if (ShouldPush) { | |||
3992 | EliminationStack.push_back(Def, MemberDFSIn, MemberDFSOut); | |||
3993 | } | |||
3994 | } | |||
3995 | ||||
3996 | // Skip the Def's, we only want to eliminate on their uses. But mark | |||
3997 | // dominated defs as dead. | |||
3998 | if (Def) { | |||
3999 | // For anything in this case, what and how we value number | |||
4000 | // guarantees that any side-effets that would have occurred (ie | |||
4001 | // throwing, etc) can be proven to either still occur (because it's | |||
4002 | // dominated by something that has the same side-effects), or never | |||
4003 | // occur. Otherwise, we would not have been able to prove it value | |||
4004 | // equivalent to something else. For these things, we can just mark | |||
4005 | // it all dead. Note that this is different from the "ProbablyDead" | |||
4006 | // set, which may not be dominated by anything, and thus, are only | |||
4007 | // easy to prove dead if they are also side-effect free. Note that | |||
4008 | // because stores are put in terms of the stored value, we skip | |||
4009 | // stored values here. If the stored value is really dead, it will | |||
4010 | // still be marked for deletion when we process it in its own class. | |||
4011 | if (!EliminationStack.empty() && Def != EliminationStack.back() && | |||
4012 | isa<Instruction>(Def) && !FromStore) | |||
4013 | markInstructionForDeletion(cast<Instruction>(Def)); | |||
4014 | continue; | |||
4015 | } | |||
4016 | // At this point, we know it is a Use we are trying to possibly | |||
4017 | // replace. | |||
4018 | ||||
4019 | assert(isa<Instruction>(U->get()) &&((isa<Instruction>(U->get()) && "Current def should have been an instruction" ) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(U->get()) && \"Current def should have been an instruction\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 4020, __PRETTY_FUNCTION__)) | |||
4020 | "Current def should have been an instruction")((isa<Instruction>(U->get()) && "Current def should have been an instruction" ) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(U->get()) && \"Current def should have been an instruction\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 4020, __PRETTY_FUNCTION__)); | |||
4021 | assert(isa<Instruction>(U->getUser()) &&((isa<Instruction>(U->getUser()) && "Current user should have been an instruction" ) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(U->getUser()) && \"Current user should have been an instruction\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 4022, __PRETTY_FUNCTION__)) | |||
4022 | "Current user should have been an instruction")((isa<Instruction>(U->getUser()) && "Current user should have been an instruction" ) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(U->getUser()) && \"Current user should have been an instruction\"" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 4022, __PRETTY_FUNCTION__)); | |||
4023 | ||||
4024 | // If the thing we are replacing into is already marked to be dead, | |||
4025 | // this use is dead. Note that this is true regardless of whether | |||
4026 | // we have anything dominating the use or not. We do this here | |||
4027 | // because we are already walking all the uses anyway. | |||
4028 | Instruction *InstUse = cast<Instruction>(U->getUser()); | |||
4029 | if (InstructionsToErase.count(InstUse)) { | |||
4030 | auto &UseCount = UseCounts[U->get()]; | |||
4031 | if (--UseCount == 0) { | |||
4032 | ProbablyDead.insert(cast<Instruction>(U->get())); | |||
4033 | } | |||
4034 | } | |||
4035 | ||||
4036 | // If we get to this point, and the stack is empty we must have a use | |||
4037 | // with nothing we can use to eliminate this use, so just skip it. | |||
4038 | if (EliminationStack.empty()) | |||
4039 | continue; | |||
4040 | ||||
4041 | Value *DominatingLeader = EliminationStack.back(); | |||
4042 | ||||
4043 | auto *II = dyn_cast<IntrinsicInst>(DominatingLeader); | |||
4044 | bool isSSACopy = II && II->getIntrinsicID() == Intrinsic::ssa_copy; | |||
4045 | if (isSSACopy) | |||
4046 | DominatingLeader = II->getOperand(0); | |||
4047 | ||||
4048 | // Don't replace our existing users with ourselves. | |||
4049 | if (U->get() == DominatingLeader) | |||
4050 | continue; | |||
4051 | LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Found replacement " << * DominatingLeader << " for " << *U->get() << " in " << *(U->getUser()) << "\n"; } } while ( false) | |||
4052 | << "Found replacement " << *DominatingLeader << " for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Found replacement " << * DominatingLeader << " for " << *U->get() << " in " << *(U->getUser()) << "\n"; } } while ( false) | |||
4053 | << *U->get() << " in " << *(U->getUser()) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Found replacement " << * DominatingLeader << " for " << *U->get() << " in " << *(U->getUser()) << "\n"; } } while ( false); | |||
4054 | ||||
4055 | // If we replaced something in an instruction, handle the patching of | |||
4056 | // metadata. Skip this if we are replacing predicateinfo with its | |||
4057 | // original operand, as we already know we can just drop it. | |||
4058 | auto *ReplacedInst = cast<Instruction>(U->get()); | |||
4059 | auto *PI = PredInfo->getPredicateInfoFor(ReplacedInst); | |||
4060 | if (!PI || DominatingLeader != PI->OriginalOp) | |||
4061 | patchReplacementInstruction(ReplacedInst, DominatingLeader); | |||
4062 | U->set(DominatingLeader); | |||
4063 | // This is now a use of the dominating leader, which means if the | |||
4064 | // dominating leader was dead, it's now live! | |||
4065 | auto &LeaderUseCount = UseCounts[DominatingLeader]; | |||
4066 | // It's about to be alive again. | |||
4067 | if (LeaderUseCount == 0 && isa<Instruction>(DominatingLeader)) | |||
4068 | ProbablyDead.erase(cast<Instruction>(DominatingLeader)); | |||
4069 | // For copy instructions, we use their operand as a leader, | |||
4070 | // which means we remove a user of the copy and it may become dead. | |||
4071 | if (isSSACopy) { | |||
4072 | unsigned &IIUseCount = UseCounts[II]; | |||
4073 | if (--IIUseCount == 0) | |||
4074 | ProbablyDead.insert(II); | |||
4075 | } | |||
4076 | ++LeaderUseCount; | |||
4077 | AnythingReplaced = true; | |||
4078 | } | |||
4079 | } | |||
4080 | } | |||
4081 | ||||
4082 | // At this point, anything still in the ProbablyDead set is actually dead if | |||
4083 | // would be trivially dead. | |||
4084 | for (auto *I : ProbablyDead) | |||
4085 | if (wouldInstructionBeTriviallyDead(I)) | |||
4086 | markInstructionForDeletion(I); | |||
4087 | ||||
4088 | // Cleanup the congruence class. | |||
4089 | CongruenceClass::MemberSet MembersLeft; | |||
4090 | for (auto *Member : *CC) | |||
4091 | if (!isa<Instruction>(Member) || | |||
4092 | !InstructionsToErase.count(cast<Instruction>(Member))) | |||
4093 | MembersLeft.insert(Member); | |||
4094 | CC->swap(MembersLeft); | |||
4095 | ||||
4096 | // If we have possible dead stores to look at, try to eliminate them. | |||
4097 | if (CC->getStoreCount() > 0) { | |||
4098 | convertClassToLoadsAndStores(*CC, PossibleDeadStores); | |||
4099 | llvm::sort(PossibleDeadStores); | |||
4100 | ValueDFSStack EliminationStack; | |||
4101 | for (auto &VD : PossibleDeadStores) { | |||
4102 | int MemberDFSIn = VD.DFSIn; | |||
4103 | int MemberDFSOut = VD.DFSOut; | |||
4104 | Instruction *Member = cast<Instruction>(VD.Def.getPointer()); | |||
4105 | if (EliminationStack.empty() || | |||
4106 | !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut)) { | |||
4107 | // Sync to our current scope. | |||
4108 | EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut); | |||
4109 | if (EliminationStack.empty()) { | |||
4110 | EliminationStack.push_back(Member, MemberDFSIn, MemberDFSOut); | |||
4111 | continue; | |||
4112 | } | |||
4113 | } | |||
4114 | // We already did load elimination, so nothing to do here. | |||
4115 | if (isa<LoadInst>(Member)) | |||
4116 | continue; | |||
4117 | assert(!EliminationStack.empty())((!EliminationStack.empty()) ? static_cast<void> (0) : __assert_fail ("!EliminationStack.empty()", "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 4117, __PRETTY_FUNCTION__)); | |||
4118 | Instruction *Leader = cast<Instruction>(EliminationStack.back()); | |||
4119 | (void)Leader; | |||
4120 | assert(DT->dominates(Leader->getParent(), Member->getParent()))((DT->dominates(Leader->getParent(), Member->getParent ())) ? static_cast<void> (0) : __assert_fail ("DT->dominates(Leader->getParent(), Member->getParent())" , "/build/llvm-toolchain-snapshot-9~svn362543/lib/Transforms/Scalar/NewGVN.cpp" , 4120, __PRETTY_FUNCTION__)); | |||
4121 | // Member is dominater by Leader, and thus dead | |||
4122 | LLVM_DEBUG(dbgs() << "Marking dead store " << *Memberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Marking dead store " << * Member << " that is dominated by " << *Leader << "\n"; } } while (false) | |||
4123 | << " that is dominated by " << *Leader << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("newgvn")) { dbgs() << "Marking dead store " << * Member << " that is dominated by " << *Leader << "\n"; } } while (false); | |||
4124 | markInstructionForDeletion(Member); | |||
4125 | CC->erase(Member); | |||
4126 | ++NumGVNDeadStores; | |||
4127 | } | |||
4128 | } | |||
4129 | } | |||
4130 | return AnythingReplaced; | |||
4131 | } | |||
4132 | ||||
4133 | // This function provides global ranking of operations so that we can place them | |||
4134 | // in a canonical order. Note that rank alone is not necessarily enough for a | |||
4135 | // complete ordering, as constants all have the same rank. However, generally, | |||
4136 | // we will simplify an operation with all constants so that it doesn't matter | |||
4137 | // what order they appear in. | |||
4138 | unsigned int NewGVN::getRank(const Value *V) const { | |||
4139 | // Prefer constants to undef to anything else | |||
4140 | // Undef is a constant, have to check it first. | |||
4141 | // Prefer smaller constants to constantexprs | |||
4142 | if (isa<ConstantExpr>(V)) | |||
4143 | return 2; | |||
4144 | if (isa<UndefValue>(V)) | |||
4145 | return 1; | |||
4146 | if (isa<Constant>(V)) | |||
4147 | return 0; | |||
4148 | else if (auto *A = dyn_cast<Argument>(V)) | |||
4149 | return 3 + A->getArgNo(); | |||
4150 | ||||
4151 | // Need to shift the instruction DFS by number of arguments + 3 to account for | |||
4152 | // the constant and argument ranking above. | |||
4153 | unsigned Result = InstrToDFSNum(V); | |||
4154 | if (Result > 0) | |||
4155 | return 4 + NumFuncArgs + Result; | |||
4156 | // Unreachable or something else, just return a really large number. | |||
4157 | return ~0; | |||
4158 | } | |||
4159 | ||||
4160 | // This is a function that says whether two commutative operations should | |||
4161 | // have their order swapped when canonicalizing. | |||
4162 | bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const { | |||
4163 | // Because we only care about a total ordering, and don't rewrite expressions | |||
4164 | // in this order, we order by rank, which will give a strict weak ordering to | |||
4165 | // everything but constants, and then we order by pointer address. | |||
4166 | return std::make_pair(getRank(A), A) > std::make_pair(getRank(B), B); | |||
4167 | } | |||
4168 | ||||
4169 | namespace { | |||
4170 | ||||
4171 | class NewGVNLegacyPass : public FunctionPass { | |||
4172 | public: | |||
4173 | // Pass identification, replacement for typeid. | |||
4174 | static char ID; | |||
4175 | ||||
4176 | NewGVNLegacyPass() : FunctionPass(ID) { | |||
4177 | initializeNewGVNLegacyPassPass(*PassRegistry::getPassRegistry()); | |||
4178 | } | |||
4179 | ||||
4180 | bool runOnFunction(Function &F) override; | |||
4181 | ||||
4182 | private: | |||
4183 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
4184 | AU.addRequired<AssumptionCacheTracker>(); | |||
4185 | AU.addRequired<DominatorTreeWrapperPass>(); | |||
4186 | AU.addRequired<TargetLibraryInfoWrapperPass>(); | |||
4187 | AU.addRequired<MemorySSAWrapperPass>(); | |||
4188 | AU.addRequired<AAResultsWrapperPass>(); | |||
4189 | AU.addPreserved<DominatorTreeWrapperPass>(); | |||
4190 | AU.addPreserved<GlobalsAAWrapperPass>(); | |||
4191 | } | |||
4192 | }; | |||
4193 | ||||
4194 | } // end anonymous namespace | |||
4195 | ||||
4196 | bool NewGVNLegacyPass::runOnFunction(Function &F) { | |||
4197 | if (skipFunction(F)) | |||
4198 | return false; | |||
4199 | return NewGVN(F, &getAnalysis<DominatorTreeWrapperPass>().getDomTree(), | |||
4200 | &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F), | |||
4201 | &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(), | |||
4202 | &getAnalysis<AAResultsWrapperPass>().getAAResults(), | |||
4203 | &getAnalysis<MemorySSAWrapperPass>().getMSSA(), | |||
4204 | F.getParent()->getDataLayout()) | |||
4205 | .runGVN(); | |||
4206 | } | |||
4207 | ||||
4208 | char NewGVNLegacyPass::ID = 0; | |||
4209 | ||||
4210 | INITIALIZE_PASS_BEGIN(NewGVNLegacyPass, "newgvn", "Global Value Numbering",static void *initializeNewGVNLegacyPassPassOnce(PassRegistry & Registry) { | |||
4211 | false, false)static void *initializeNewGVNLegacyPassPassOnce(PassRegistry & Registry) { | |||
4212 | INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry); | |||
4213 | INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)initializeMemorySSAWrapperPassPass(Registry); | |||
4214 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); | |||
4215 | INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry); | |||
4216 | INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry); | |||
4217 | INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)initializeGlobalsAAWrapperPassPass(Registry); | |||
4218 | INITIALIZE_PASS_END(NewGVNLegacyPass, "newgvn", "Global Value Numbering", false,PassInfo *PI = new PassInfo( "Global Value Numbering", "newgvn" , &NewGVNLegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <NewGVNLegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeNewGVNLegacyPassPassFlag ; void llvm::initializeNewGVNLegacyPassPass(PassRegistry & Registry) { llvm::call_once(InitializeNewGVNLegacyPassPassFlag , initializeNewGVNLegacyPassPassOnce, std::ref(Registry)); } | |||
4219 | false)PassInfo *PI = new PassInfo( "Global Value Numbering", "newgvn" , &NewGVNLegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <NewGVNLegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeNewGVNLegacyPassPassFlag ; void llvm::initializeNewGVNLegacyPassPass(PassRegistry & Registry) { llvm::call_once(InitializeNewGVNLegacyPassPassFlag , initializeNewGVNLegacyPassPassOnce, std::ref(Registry)); } | |||
4220 | ||||
4221 | // createGVNPass - The public interface to this file. | |||
4222 | FunctionPass *llvm::createNewGVNPass() { return new NewGVNLegacyPass(); } | |||
4223 | ||||
4224 | PreservedAnalyses NewGVNPass::run(Function &F, AnalysisManager<Function> &AM) { | |||
4225 | // Apparently the order in which we get these results matter for | |||
4226 | // the old GVN (see Chandler's comment in GVN.cpp). I'll keep | |||
4227 | // the same order here, just in case. | |||
4228 | auto &AC = AM.getResult<AssumptionAnalysis>(F); | |||
4229 | auto &DT = AM.getResult<DominatorTreeAnalysis>(F); | |||
4230 | auto &TLI = AM.getResult<TargetLibraryAnalysis>(F); | |||
4231 | auto &AA = AM.getResult<AAManager>(F); | |||
4232 | auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA(); | |||
4233 | bool Changed = | |||
4234 | NewGVN(F, &DT, &AC, &TLI, &AA, &MSSA, F.getParent()->getDataLayout()) | |||
4235 | .runGVN(); | |||
4236 | if (!Changed) | |||
4237 | return PreservedAnalyses::all(); | |||
4238 | PreservedAnalyses PA; | |||
4239 | PA.preserve<DominatorTreeAnalysis>(); | |||
4240 | PA.preserve<GlobalsAA>(); | |||
4241 | return PA; | |||
4242 | } |