File: | lib/Transforms/Utils/SimplifyCFG.cpp |
Warning: | line 2358, column 27 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===// | |||
2 | // | |||
3 | // The LLVM Compiler Infrastructure | |||
4 | // | |||
5 | // This file is distributed under the University of Illinois Open Source | |||
6 | // License. See LICENSE.TXT for details. | |||
7 | // | |||
8 | //===----------------------------------------------------------------------===// | |||
9 | // | |||
10 | // Peephole optimize the CFG. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "llvm/ADT/APInt.h" | |||
15 | #include "llvm/ADT/ArrayRef.h" | |||
16 | #include "llvm/ADT/DenseMap.h" | |||
17 | #include "llvm/ADT/Optional.h" | |||
18 | #include "llvm/ADT/STLExtras.h" | |||
19 | #include "llvm/ADT/SetOperations.h" | |||
20 | #include "llvm/ADT/SetVector.h" | |||
21 | #include "llvm/ADT/SmallPtrSet.h" | |||
22 | #include "llvm/ADT/SmallSet.h" | |||
23 | #include "llvm/ADT/SmallVector.h" | |||
24 | #include "llvm/ADT/Statistic.h" | |||
25 | #include "llvm/ADT/StringRef.h" | |||
26 | #include "llvm/Analysis/AssumptionCache.h" | |||
27 | #include "llvm/Analysis/ConstantFolding.h" | |||
28 | #include "llvm/Analysis/EHPersonalities.h" | |||
29 | #include "llvm/Analysis/InstructionSimplify.h" | |||
30 | #include "llvm/Analysis/TargetTransformInfo.h" | |||
31 | #include "llvm/Analysis/ValueTracking.h" | |||
32 | #include "llvm/IR/Attributes.h" | |||
33 | #include "llvm/IR/BasicBlock.h" | |||
34 | #include "llvm/IR/CFG.h" | |||
35 | #include "llvm/IR/CallSite.h" | |||
36 | #include "llvm/IR/Constant.h" | |||
37 | #include "llvm/IR/ConstantRange.h" | |||
38 | #include "llvm/IR/Constants.h" | |||
39 | #include "llvm/IR/DataLayout.h" | |||
40 | #include "llvm/IR/DerivedTypes.h" | |||
41 | #include "llvm/IR/Function.h" | |||
42 | #include "llvm/IR/GlobalValue.h" | |||
43 | #include "llvm/IR/GlobalVariable.h" | |||
44 | #include "llvm/IR/IRBuilder.h" | |||
45 | #include "llvm/IR/InstrTypes.h" | |||
46 | #include "llvm/IR/Instruction.h" | |||
47 | #include "llvm/IR/Instructions.h" | |||
48 | #include "llvm/IR/IntrinsicInst.h" | |||
49 | #include "llvm/IR/Intrinsics.h" | |||
50 | #include "llvm/IR/LLVMContext.h" | |||
51 | #include "llvm/IR/MDBuilder.h" | |||
52 | #include "llvm/IR/Metadata.h" | |||
53 | #include "llvm/IR/Module.h" | |||
54 | #include "llvm/IR/NoFolder.h" | |||
55 | #include "llvm/IR/Operator.h" | |||
56 | #include "llvm/IR/PatternMatch.h" | |||
57 | #include "llvm/IR/Type.h" | |||
58 | #include "llvm/IR/Use.h" | |||
59 | #include "llvm/IR/User.h" | |||
60 | #include "llvm/IR/Value.h" | |||
61 | #include "llvm/Support/Casting.h" | |||
62 | #include "llvm/Support/CommandLine.h" | |||
63 | #include "llvm/Support/Debug.h" | |||
64 | #include "llvm/Support/ErrorHandling.h" | |||
65 | #include "llvm/Support/KnownBits.h" | |||
66 | #include "llvm/Support/MathExtras.h" | |||
67 | #include "llvm/Support/raw_ostream.h" | |||
68 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | |||
69 | #include "llvm/Transforms/Utils/Local.h" | |||
70 | #include "llvm/Transforms/Utils/ValueMapper.h" | |||
71 | #include <algorithm> | |||
72 | #include <cassert> | |||
73 | #include <climits> | |||
74 | #include <cstddef> | |||
75 | #include <cstdint> | |||
76 | #include <iterator> | |||
77 | #include <map> | |||
78 | #include <set> | |||
79 | #include <tuple> | |||
80 | #include <utility> | |||
81 | #include <vector> | |||
82 | ||||
83 | using namespace llvm; | |||
84 | using namespace PatternMatch; | |||
85 | ||||
86 | #define DEBUG_TYPE"simplifycfg" "simplifycfg" | |||
87 | ||||
88 | // Chosen as 2 so as to be cheap, but still to have enough power to fold | |||
89 | // a select, so the "clamp" idiom (of a min followed by a max) will be caught. | |||
90 | // To catch this, we need to fold a compare and a select, hence '2' being the | |||
91 | // minimum reasonable default. | |||
92 | static cl::opt<unsigned> PHINodeFoldingThreshold( | |||
93 | "phi-node-folding-threshold", cl::Hidden, cl::init(2), | |||
94 | cl::desc( | |||
95 | "Control the amount of phi node folding to perform (default = 2)")); | |||
96 | ||||
97 | static cl::opt<bool> DupRet( | |||
98 | "simplifycfg-dup-ret", cl::Hidden, cl::init(false), | |||
99 | cl::desc("Duplicate return instructions into unconditional branches")); | |||
100 | ||||
101 | static cl::opt<bool> | |||
102 | SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true), | |||
103 | cl::desc("Sink common instructions down to the end block")); | |||
104 | ||||
105 | static cl::opt<bool> HoistCondStores( | |||
106 | "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true), | |||
107 | cl::desc("Hoist conditional stores if an unconditional store precedes")); | |||
108 | ||||
109 | static cl::opt<bool> MergeCondStores( | |||
110 | "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true), | |||
111 | cl::desc("Hoist conditional stores even if an unconditional store does not " | |||
112 | "precede - hoist multiple conditional stores into a single " | |||
113 | "predicated store")); | |||
114 | ||||
115 | static cl::opt<bool> MergeCondStoresAggressively( | |||
116 | "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false), | |||
117 | cl::desc("When merging conditional stores, do so even if the resultant " | |||
118 | "basic blocks are unlikely to be if-converted as a result")); | |||
119 | ||||
120 | static cl::opt<bool> SpeculateOneExpensiveInst( | |||
121 | "speculate-one-expensive-inst", cl::Hidden, cl::init(true), | |||
122 | cl::desc("Allow exactly one expensive instruction to be speculatively " | |||
123 | "executed")); | |||
124 | ||||
125 | static cl::opt<unsigned> MaxSpeculationDepth( | |||
126 | "max-speculation-depth", cl::Hidden, cl::init(10), | |||
127 | cl::desc("Limit maximum recursion depth when calculating costs of " | |||
128 | "speculatively executed instructions")); | |||
129 | ||||
130 | STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps")static llvm::Statistic NumBitMaps = {"simplifycfg", "NumBitMaps" , "Number of switch instructions turned into bitmaps", {0}, { false}}; | |||
131 | STATISTIC(NumLinearMaps,static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps" , "Number of switch instructions turned into linear mapping", {0}, {false}} | |||
132 | "Number of switch instructions turned into linear mapping")static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps" , "Number of switch instructions turned into linear mapping", {0}, {false}}; | |||
133 | STATISTIC(NumLookupTables,static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables" , "Number of switch instructions turned into lookup tables", { 0}, {false}} | |||
134 | "Number of switch instructions turned into lookup tables")static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables" , "Number of switch instructions turned into lookup tables", { 0}, {false}}; | |||
135 | STATISTIC(static llvm::Statistic NumLookupTablesHoles = {"simplifycfg", "NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)" , {0}, {false}} | |||
136 | NumLookupTablesHoles,static llvm::Statistic NumLookupTablesHoles = {"simplifycfg", "NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)" , {0}, {false}} | |||
137 | "Number of switch instructions turned into lookup tables (holes checked)")static llvm::Statistic NumLookupTablesHoles = {"simplifycfg", "NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)" , {0}, {false}}; | |||
138 | STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares")static llvm::Statistic NumTableCmpReuses = {"simplifycfg", "NumTableCmpReuses" , "Number of reused switch table lookup compares", {0}, {false }}; | |||
139 | STATISTIC(NumSinkCommons,static llvm::Statistic NumSinkCommons = {"simplifycfg", "NumSinkCommons" , "Number of common instructions sunk down to the end block", {0}, {false}} | |||
140 | "Number of common instructions sunk down to the end block")static llvm::Statistic NumSinkCommons = {"simplifycfg", "NumSinkCommons" , "Number of common instructions sunk down to the end block", {0}, {false}}; | |||
141 | STATISTIC(NumSpeculations, "Number of speculative executed instructions")static llvm::Statistic NumSpeculations = {"simplifycfg", "NumSpeculations" , "Number of speculative executed instructions", {0}, {false} }; | |||
142 | ||||
143 | namespace { | |||
144 | ||||
145 | // The first field contains the value that the switch produces when a certain | |||
146 | // case group is selected, and the second field is a vector containing the | |||
147 | // cases composing the case group. | |||
148 | using SwitchCaseResultVectorTy = | |||
149 | SmallVector<std::pair<Constant *, SmallVector<ConstantInt *, 4>>, 2>; | |||
150 | ||||
151 | // The first field contains the phi node that generates a result of the switch | |||
152 | // and the second field contains the value generated for a certain case in the | |||
153 | // switch for that PHI. | |||
154 | using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>; | |||
155 | ||||
156 | /// ValueEqualityComparisonCase - Represents a case of a switch. | |||
157 | struct ValueEqualityComparisonCase { | |||
158 | ConstantInt *Value; | |||
159 | BasicBlock *Dest; | |||
160 | ||||
161 | ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest) | |||
162 | : Value(Value), Dest(Dest) {} | |||
163 | ||||
164 | bool operator<(ValueEqualityComparisonCase RHS) const { | |||
165 | // Comparing pointers is ok as we only rely on the order for uniquing. | |||
166 | return Value < RHS.Value; | |||
167 | } | |||
168 | ||||
169 | bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; } | |||
170 | }; | |||
171 | ||||
172 | class SimplifyCFGOpt { | |||
173 | const TargetTransformInfo &TTI; | |||
174 | const DataLayout &DL; | |||
175 | SmallPtrSetImpl<BasicBlock *> *LoopHeaders; | |||
176 | const SimplifyCFGOptions &Options; | |||
177 | ||||
178 | Value *isValueEqualityComparison(TerminatorInst *TI); | |||
179 | BasicBlock *GetValueEqualityComparisonCases( | |||
180 | TerminatorInst *TI, std::vector<ValueEqualityComparisonCase> &Cases); | |||
181 | bool SimplifyEqualityComparisonWithOnlyPredecessor(TerminatorInst *TI, | |||
182 | BasicBlock *Pred, | |||
183 | IRBuilder<> &Builder); | |||
184 | bool FoldValueComparisonIntoPredecessors(TerminatorInst *TI, | |||
185 | IRBuilder<> &Builder); | |||
186 | ||||
187 | bool SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder); | |||
188 | bool SimplifyResume(ResumeInst *RI, IRBuilder<> &Builder); | |||
189 | bool SimplifySingleResume(ResumeInst *RI); | |||
190 | bool SimplifyCommonResume(ResumeInst *RI); | |||
191 | bool SimplifyCleanupReturn(CleanupReturnInst *RI); | |||
192 | bool SimplifyUnreachable(UnreachableInst *UI); | |||
193 | bool SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder); | |||
194 | bool SimplifyIndirectBr(IndirectBrInst *IBI); | |||
195 | bool SimplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder); | |||
196 | bool SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder); | |||
197 | ||||
198 | public: | |||
199 | SimplifyCFGOpt(const TargetTransformInfo &TTI, const DataLayout &DL, | |||
200 | SmallPtrSetImpl<BasicBlock *> *LoopHeaders, | |||
201 | const SimplifyCFGOptions &Opts) | |||
202 | : TTI(TTI), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) {} | |||
203 | ||||
204 | bool run(BasicBlock *BB); | |||
205 | }; | |||
206 | ||||
207 | } // end anonymous namespace | |||
208 | ||||
209 | /// Return true if it is safe to merge these two | |||
210 | /// terminator instructions together. | |||
211 | static bool | |||
212 | SafeToMergeTerminators(TerminatorInst *SI1, TerminatorInst *SI2, | |||
213 | SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) { | |||
214 | if (SI1 == SI2) | |||
215 | return false; // Can't merge with self! | |||
216 | ||||
217 | // It is not safe to merge these two switch instructions if they have a common | |||
218 | // successor, and if that successor has a PHI node, and if *that* PHI node has | |||
219 | // conflicting incoming values from the two switch blocks. | |||
220 | BasicBlock *SI1BB = SI1->getParent(); | |||
221 | BasicBlock *SI2BB = SI2->getParent(); | |||
222 | ||||
223 | SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB)); | |||
224 | bool Fail = false; | |||
225 | for (BasicBlock *Succ : successors(SI2BB)) | |||
226 | if (SI1Succs.count(Succ)) | |||
227 | for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) { | |||
228 | PHINode *PN = cast<PHINode>(BBI); | |||
229 | if (PN->getIncomingValueForBlock(SI1BB) != | |||
230 | PN->getIncomingValueForBlock(SI2BB)) { | |||
231 | if (FailBlocks) | |||
232 | FailBlocks->insert(Succ); | |||
233 | Fail = true; | |||
234 | } | |||
235 | } | |||
236 | ||||
237 | return !Fail; | |||
238 | } | |||
239 | ||||
240 | /// Return true if it is safe and profitable to merge these two terminator | |||
241 | /// instructions together, where SI1 is an unconditional branch. PhiNodes will | |||
242 | /// store all PHI nodes in common successors. | |||
243 | static bool | |||
244 | isProfitableToFoldUnconditional(BranchInst *SI1, BranchInst *SI2, | |||
245 | Instruction *Cond, | |||
246 | SmallVectorImpl<PHINode *> &PhiNodes) { | |||
247 | if (SI1 == SI2) | |||
248 | return false; // Can't merge with self! | |||
249 | assert(SI1->isUnconditional() && SI2->isConditional())(static_cast <bool> (SI1->isUnconditional() && SI2->isConditional()) ? void (0) : __assert_fail ("SI1->isUnconditional() && SI2->isConditional()" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 249, __extension__ __PRETTY_FUNCTION__)); | |||
250 | ||||
251 | // We fold the unconditional branch if we can easily update all PHI nodes in | |||
252 | // common successors: | |||
253 | // 1> We have a constant incoming value for the conditional branch; | |||
254 | // 2> We have "Cond" as the incoming value for the unconditional branch; | |||
255 | // 3> SI2->getCondition() and Cond have same operands. | |||
256 | CmpInst *Ci2 = dyn_cast<CmpInst>(SI2->getCondition()); | |||
257 | if (!Ci2) | |||
258 | return false; | |||
259 | if (!(Cond->getOperand(0) == Ci2->getOperand(0) && | |||
260 | Cond->getOperand(1) == Ci2->getOperand(1)) && | |||
261 | !(Cond->getOperand(0) == Ci2->getOperand(1) && | |||
262 | Cond->getOperand(1) == Ci2->getOperand(0))) | |||
263 | return false; | |||
264 | ||||
265 | BasicBlock *SI1BB = SI1->getParent(); | |||
266 | BasicBlock *SI2BB = SI2->getParent(); | |||
267 | SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB)); | |||
268 | for (BasicBlock *Succ : successors(SI2BB)) | |||
269 | if (SI1Succs.count(Succ)) | |||
270 | for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) { | |||
271 | PHINode *PN = cast<PHINode>(BBI); | |||
272 | if (PN->getIncomingValueForBlock(SI1BB) != Cond || | |||
273 | !isa<ConstantInt>(PN->getIncomingValueForBlock(SI2BB))) | |||
274 | return false; | |||
275 | PhiNodes.push_back(PN); | |||
276 | } | |||
277 | return true; | |||
278 | } | |||
279 | ||||
280 | /// Update PHI nodes in Succ to indicate that there will now be entries in it | |||
281 | /// from the 'NewPred' block. The values that will be flowing into the PHI nodes | |||
282 | /// will be the same as those coming in from ExistPred, an existing predecessor | |||
283 | /// of Succ. | |||
284 | static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred, | |||
285 | BasicBlock *ExistPred) { | |||
286 | for (PHINode &PN : Succ->phis()) | |||
287 | PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred); | |||
288 | } | |||
289 | ||||
290 | /// Compute an abstract "cost" of speculating the given instruction, | |||
291 | /// which is assumed to be safe to speculate. TCC_Free means cheap, | |||
292 | /// TCC_Basic means less cheap, and TCC_Expensive means prohibitively | |||
293 | /// expensive. | |||
294 | static unsigned ComputeSpeculationCost(const User *I, | |||
295 | const TargetTransformInfo &TTI) { | |||
296 | assert(isSafeToSpeculativelyExecute(I) &&(static_cast <bool> (isSafeToSpeculativelyExecute(I) && "Instruction is not safe to speculatively execute!") ? void ( 0) : __assert_fail ("isSafeToSpeculativelyExecute(I) && \"Instruction is not safe to speculatively execute!\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 297, __extension__ __PRETTY_FUNCTION__)) | |||
297 | "Instruction is not safe to speculatively execute!")(static_cast <bool> (isSafeToSpeculativelyExecute(I) && "Instruction is not safe to speculatively execute!") ? void ( 0) : __assert_fail ("isSafeToSpeculativelyExecute(I) && \"Instruction is not safe to speculatively execute!\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 297, __extension__ __PRETTY_FUNCTION__)); | |||
298 | return TTI.getUserCost(I); | |||
299 | } | |||
300 | ||||
301 | /// If we have a merge point of an "if condition" as accepted above, | |||
302 | /// return true if the specified value dominates the block. We | |||
303 | /// don't handle the true generality of domination here, just a special case | |||
304 | /// which works well enough for us. | |||
305 | /// | |||
306 | /// If AggressiveInsts is non-null, and if V does not dominate BB, we check to | |||
307 | /// see if V (which must be an instruction) and its recursive operands | |||
308 | /// that do not dominate BB have a combined cost lower than CostRemaining and | |||
309 | /// are non-trapping. If both are true, the instruction is inserted into the | |||
310 | /// set and true is returned. | |||
311 | /// | |||
312 | /// The cost for most non-trapping instructions is defined as 1 except for | |||
313 | /// Select whose cost is 2. | |||
314 | /// | |||
315 | /// After this function returns, CostRemaining is decreased by the cost of | |||
316 | /// V plus its non-dominating operands. If that cost is greater than | |||
317 | /// CostRemaining, false is returned and CostRemaining is undefined. | |||
318 | static bool DominatesMergePoint(Value *V, BasicBlock *BB, | |||
319 | SmallPtrSetImpl<Instruction *> *AggressiveInsts, | |||
320 | unsigned &CostRemaining, | |||
321 | const TargetTransformInfo &TTI, | |||
322 | unsigned Depth = 0) { | |||
323 | // It is possible to hit a zero-cost cycle (phi/gep instructions for example), | |||
324 | // so limit the recursion depth. | |||
325 | // TODO: While this recursion limit does prevent pathological behavior, it | |||
326 | // would be better to track visited instructions to avoid cycles. | |||
327 | if (Depth == MaxSpeculationDepth) | |||
328 | return false; | |||
329 | ||||
330 | Instruction *I = dyn_cast<Instruction>(V); | |||
331 | if (!I) { | |||
332 | // Non-instructions all dominate instructions, but not all constantexprs | |||
333 | // can be executed unconditionally. | |||
334 | if (ConstantExpr *C = dyn_cast<ConstantExpr>(V)) | |||
335 | if (C->canTrap()) | |||
336 | return false; | |||
337 | return true; | |||
338 | } | |||
339 | BasicBlock *PBB = I->getParent(); | |||
340 | ||||
341 | // We don't want to allow weird loops that might have the "if condition" in | |||
342 | // the bottom of this block. | |||
343 | if (PBB == BB) | |||
344 | return false; | |||
345 | ||||
346 | // If this instruction is defined in a block that contains an unconditional | |||
347 | // branch to BB, then it must be in the 'conditional' part of the "if | |||
348 | // statement". If not, it definitely dominates the region. | |||
349 | BranchInst *BI = dyn_cast<BranchInst>(PBB->getTerminator()); | |||
350 | if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB) | |||
351 | return true; | |||
352 | ||||
353 | // If we aren't allowing aggressive promotion anymore, then don't consider | |||
354 | // instructions in the 'if region'. | |||
355 | if (!AggressiveInsts) | |||
356 | return false; | |||
357 | ||||
358 | // If we have seen this instruction before, don't count it again. | |||
359 | if (AggressiveInsts->count(I)) | |||
360 | return true; | |||
361 | ||||
362 | // Okay, it looks like the instruction IS in the "condition". Check to | |||
363 | // see if it's a cheap instruction to unconditionally compute, and if it | |||
364 | // only uses stuff defined outside of the condition. If so, hoist it out. | |||
365 | if (!isSafeToSpeculativelyExecute(I)) | |||
366 | return false; | |||
367 | ||||
368 | unsigned Cost = ComputeSpeculationCost(I, TTI); | |||
369 | ||||
370 | // Allow exactly one instruction to be speculated regardless of its cost | |||
371 | // (as long as it is safe to do so). | |||
372 | // This is intended to flatten the CFG even if the instruction is a division | |||
373 | // or other expensive operation. The speculation of an expensive instruction | |||
374 | // is expected to be undone in CodeGenPrepare if the speculation has not | |||
375 | // enabled further IR optimizations. | |||
376 | if (Cost > CostRemaining && | |||
377 | (!SpeculateOneExpensiveInst || !AggressiveInsts->empty() || Depth > 0)) | |||
378 | return false; | |||
379 | ||||
380 | // Avoid unsigned wrap. | |||
381 | CostRemaining = (Cost > CostRemaining) ? 0 : CostRemaining - Cost; | |||
382 | ||||
383 | // Okay, we can only really hoist these out if their operands do | |||
384 | // not take us over the cost threshold. | |||
385 | for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i) | |||
386 | if (!DominatesMergePoint(*i, BB, AggressiveInsts, CostRemaining, TTI, | |||
387 | Depth + 1)) | |||
388 | return false; | |||
389 | // Okay, it's safe to do this! Remember this instruction. | |||
390 | AggressiveInsts->insert(I); | |||
391 | return true; | |||
392 | } | |||
393 | ||||
394 | /// Extract ConstantInt from value, looking through IntToPtr | |||
395 | /// and PointerNullValue. Return NULL if value is not a constant int. | |||
396 | static ConstantInt *GetConstantInt(Value *V, const DataLayout &DL) { | |||
397 | // Normal constant int. | |||
398 | ConstantInt *CI = dyn_cast<ConstantInt>(V); | |||
399 | if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy()) | |||
400 | return CI; | |||
401 | ||||
402 | // This is some kind of pointer constant. Turn it into a pointer-sized | |||
403 | // ConstantInt if possible. | |||
404 | IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType())); | |||
405 | ||||
406 | // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*). | |||
407 | if (isa<ConstantPointerNull>(V)) | |||
408 | return ConstantInt::get(PtrTy, 0); | |||
409 | ||||
410 | // IntToPtr const int. | |||
411 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) | |||
412 | if (CE->getOpcode() == Instruction::IntToPtr) | |||
413 | if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) { | |||
414 | // The constant is very likely to have the right type already. | |||
415 | if (CI->getType() == PtrTy) | |||
416 | return CI; | |||
417 | else | |||
418 | return cast<ConstantInt>( | |||
419 | ConstantExpr::getIntegerCast(CI, PtrTy, /*isSigned=*/false)); | |||
420 | } | |||
421 | return nullptr; | |||
422 | } | |||
423 | ||||
424 | namespace { | |||
425 | ||||
426 | /// Given a chain of or (||) or and (&&) comparison of a value against a | |||
427 | /// constant, this will try to recover the information required for a switch | |||
428 | /// structure. | |||
429 | /// It will depth-first traverse the chain of comparison, seeking for patterns | |||
430 | /// like %a == 12 or %a < 4 and combine them to produce a set of integer | |||
431 | /// representing the different cases for the switch. | |||
432 | /// Note that if the chain is composed of '||' it will build the set of elements | |||
433 | /// that matches the comparisons (i.e. any of this value validate the chain) | |||
434 | /// while for a chain of '&&' it will build the set elements that make the test | |||
435 | /// fail. | |||
436 | struct ConstantComparesGatherer { | |||
437 | const DataLayout &DL; | |||
438 | ||||
439 | /// Value found for the switch comparison | |||
440 | Value *CompValue = nullptr; | |||
441 | ||||
442 | /// Extra clause to be checked before the switch | |||
443 | Value *Extra = nullptr; | |||
444 | ||||
445 | /// Set of integers to match in switch | |||
446 | SmallVector<ConstantInt *, 8> Vals; | |||
447 | ||||
448 | /// Number of comparisons matched in the and/or chain | |||
449 | unsigned UsedICmps = 0; | |||
450 | ||||
451 | /// Construct and compute the result for the comparison instruction Cond | |||
452 | ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) { | |||
453 | gather(Cond); | |||
454 | } | |||
455 | ||||
456 | ConstantComparesGatherer(const ConstantComparesGatherer &) = delete; | |||
457 | ConstantComparesGatherer & | |||
458 | operator=(const ConstantComparesGatherer &) = delete; | |||
459 | ||||
460 | private: | |||
461 | /// Try to set the current value used for the comparison, it succeeds only if | |||
462 | /// it wasn't set before or if the new value is the same as the old one | |||
463 | bool setValueOnce(Value *NewVal) { | |||
464 | if (CompValue && CompValue != NewVal) | |||
465 | return false; | |||
466 | CompValue = NewVal; | |||
467 | return (CompValue != nullptr); | |||
468 | } | |||
469 | ||||
470 | /// Try to match Instruction "I" as a comparison against a constant and | |||
471 | /// populates the array Vals with the set of values that match (or do not | |||
472 | /// match depending on isEQ). | |||
473 | /// Return false on failure. On success, the Value the comparison matched | |||
474 | /// against is placed in CompValue. | |||
475 | /// If CompValue is already set, the function is expected to fail if a match | |||
476 | /// is found but the value compared to is different. | |||
477 | bool matchInstruction(Instruction *I, bool isEQ) { | |||
478 | // If this is an icmp against a constant, handle this as one of the cases. | |||
479 | ICmpInst *ICI; | |||
480 | ConstantInt *C; | |||
481 | if (!((ICI = dyn_cast<ICmpInst>(I)) && | |||
482 | (C = GetConstantInt(I->getOperand(1), DL)))) { | |||
483 | return false; | |||
484 | } | |||
485 | ||||
486 | Value *RHSVal; | |||
487 | const APInt *RHSC; | |||
488 | ||||
489 | // Pattern match a special case | |||
490 | // (x & ~2^z) == y --> x == y || x == y|2^z | |||
491 | // This undoes a transformation done by instcombine to fuse 2 compares. | |||
492 | if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) { | |||
493 | // It's a little bit hard to see why the following transformations are | |||
494 | // correct. Here is a CVC3 program to verify them for 64-bit values: | |||
495 | ||||
496 | /* | |||
497 | ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63); | |||
498 | x : BITVECTOR(64); | |||
499 | y : BITVECTOR(64); | |||
500 | z : BITVECTOR(64); | |||
501 | mask : BITVECTOR(64) = BVSHL(ONE, z); | |||
502 | QUERY( (y & ~mask = y) => | |||
503 | ((x & ~mask = y) <=> (x = y OR x = (y | mask))) | |||
504 | ); | |||
505 | QUERY( (y | mask = y) => | |||
506 | ((x | mask = y) <=> (x = y OR x = (y & ~mask))) | |||
507 | ); | |||
508 | */ | |||
509 | ||||
510 | // Please note that each pattern must be a dual implication (<--> or | |||
511 | // iff). One directional implication can create spurious matches. If the | |||
512 | // implication is only one-way, an unsatisfiable condition on the left | |||
513 | // side can imply a satisfiable condition on the right side. Dual | |||
514 | // implication ensures that satisfiable conditions are transformed to | |||
515 | // other satisfiable conditions and unsatisfiable conditions are | |||
516 | // transformed to other unsatisfiable conditions. | |||
517 | ||||
518 | // Here is a concrete example of a unsatisfiable condition on the left | |||
519 | // implying a satisfiable condition on the right: | |||
520 | // | |||
521 | // mask = (1 << z) | |||
522 | // (x & ~mask) == y --> (x == y || x == (y | mask)) | |||
523 | // | |||
524 | // Substituting y = 3, z = 0 yields: | |||
525 | // (x & -2) == 3 --> (x == 3 || x == 2) | |||
526 | ||||
527 | // Pattern match a special case: | |||
528 | /* | |||
529 | QUERY( (y & ~mask = y) => | |||
530 | ((x & ~mask = y) <=> (x = y OR x = (y | mask))) | |||
531 | ); | |||
532 | */ | |||
533 | if (match(ICI->getOperand(0), | |||
534 | m_And(m_Value(RHSVal), m_APInt(RHSC)))) { | |||
535 | APInt Mask = ~*RHSC; | |||
536 | if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) { | |||
537 | // If we already have a value for the switch, it has to match! | |||
538 | if (!setValueOnce(RHSVal)) | |||
539 | return false; | |||
540 | ||||
541 | Vals.push_back(C); | |||
542 | Vals.push_back( | |||
543 | ConstantInt::get(C->getContext(), | |||
544 | C->getValue() | Mask)); | |||
545 | UsedICmps++; | |||
546 | return true; | |||
547 | } | |||
548 | } | |||
549 | ||||
550 | // Pattern match a special case: | |||
551 | /* | |||
552 | QUERY( (y | mask = y) => | |||
553 | ((x | mask = y) <=> (x = y OR x = (y & ~mask))) | |||
554 | ); | |||
555 | */ | |||
556 | if (match(ICI->getOperand(0), | |||
557 | m_Or(m_Value(RHSVal), m_APInt(RHSC)))) { | |||
558 | APInt Mask = *RHSC; | |||
559 | if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) { | |||
560 | // If we already have a value for the switch, it has to match! | |||
561 | if (!setValueOnce(RHSVal)) | |||
562 | return false; | |||
563 | ||||
564 | Vals.push_back(C); | |||
565 | Vals.push_back(ConstantInt::get(C->getContext(), | |||
566 | C->getValue() & ~Mask)); | |||
567 | UsedICmps++; | |||
568 | return true; | |||
569 | } | |||
570 | } | |||
571 | ||||
572 | // If we already have a value for the switch, it has to match! | |||
573 | if (!setValueOnce(ICI->getOperand(0))) | |||
574 | return false; | |||
575 | ||||
576 | UsedICmps++; | |||
577 | Vals.push_back(C); | |||
578 | return ICI->getOperand(0); | |||
579 | } | |||
580 | ||||
581 | // If we have "x ult 3", for example, then we can add 0,1,2 to the set. | |||
582 | ConstantRange Span = ConstantRange::makeAllowedICmpRegion( | |||
583 | ICI->getPredicate(), C->getValue()); | |||
584 | ||||
585 | // Shift the range if the compare is fed by an add. This is the range | |||
586 | // compare idiom as emitted by instcombine. | |||
587 | Value *CandidateVal = I->getOperand(0); | |||
588 | if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) { | |||
589 | Span = Span.subtract(*RHSC); | |||
590 | CandidateVal = RHSVal; | |||
591 | } | |||
592 | ||||
593 | // If this is an and/!= check, then we are looking to build the set of | |||
594 | // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into | |||
595 | // x != 0 && x != 1. | |||
596 | if (!isEQ) | |||
597 | Span = Span.inverse(); | |||
598 | ||||
599 | // If there are a ton of values, we don't want to make a ginormous switch. | |||
600 | if (Span.isSizeLargerThan(8) || Span.isEmptySet()) { | |||
601 | return false; | |||
602 | } | |||
603 | ||||
604 | // If we already have a value for the switch, it has to match! | |||
605 | if (!setValueOnce(CandidateVal)) | |||
606 | return false; | |||
607 | ||||
608 | // Add all values from the range to the set | |||
609 | for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp) | |||
610 | Vals.push_back(ConstantInt::get(I->getContext(), Tmp)); | |||
611 | ||||
612 | UsedICmps++; | |||
613 | return true; | |||
614 | } | |||
615 | ||||
616 | /// Given a potentially 'or'd or 'and'd together collection of icmp | |||
617 | /// eq/ne/lt/gt instructions that compare a value against a constant, extract | |||
618 | /// the value being compared, and stick the list constants into the Vals | |||
619 | /// vector. | |||
620 | /// One "Extra" case is allowed to differ from the other. | |||
621 | void gather(Value *V) { | |||
622 | Instruction *I = dyn_cast<Instruction>(V); | |||
623 | bool isEQ = (I->getOpcode() == Instruction::Or); | |||
624 | ||||
625 | // Keep a stack (SmallVector for efficiency) for depth-first traversal | |||
626 | SmallVector<Value *, 8> DFT; | |||
627 | SmallPtrSet<Value *, 8> Visited; | |||
628 | ||||
629 | // Initialize | |||
630 | Visited.insert(V); | |||
631 | DFT.push_back(V); | |||
632 | ||||
633 | while (!DFT.empty()) { | |||
634 | V = DFT.pop_back_val(); | |||
635 | ||||
636 | if (Instruction *I = dyn_cast<Instruction>(V)) { | |||
637 | // If it is a || (or && depending on isEQ), process the operands. | |||
638 | if (I->getOpcode() == (isEQ ? Instruction::Or : Instruction::And)) { | |||
639 | if (Visited.insert(I->getOperand(1)).second) | |||
640 | DFT.push_back(I->getOperand(1)); | |||
641 | if (Visited.insert(I->getOperand(0)).second) | |||
642 | DFT.push_back(I->getOperand(0)); | |||
643 | continue; | |||
644 | } | |||
645 | ||||
646 | // Try to match the current instruction | |||
647 | if (matchInstruction(I, isEQ)) | |||
648 | // Match succeed, continue the loop | |||
649 | continue; | |||
650 | } | |||
651 | ||||
652 | // One element of the sequence of || (or &&) could not be match as a | |||
653 | // comparison against the same value as the others. | |||
654 | // We allow only one "Extra" case to be checked before the switch | |||
655 | if (!Extra) { | |||
656 | Extra = V; | |||
657 | continue; | |||
658 | } | |||
659 | // Failed to parse a proper sequence, abort now | |||
660 | CompValue = nullptr; | |||
661 | break; | |||
662 | } | |||
663 | } | |||
664 | }; | |||
665 | ||||
666 | } // end anonymous namespace | |||
667 | ||||
668 | static void EraseTerminatorInstAndDCECond(TerminatorInst *TI) { | |||
669 | Instruction *Cond = nullptr; | |||
670 | if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { | |||
671 | Cond = dyn_cast<Instruction>(SI->getCondition()); | |||
672 | } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { | |||
673 | if (BI->isConditional()) | |||
674 | Cond = dyn_cast<Instruction>(BI->getCondition()); | |||
675 | } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) { | |||
676 | Cond = dyn_cast<Instruction>(IBI->getAddress()); | |||
677 | } | |||
678 | ||||
679 | TI->eraseFromParent(); | |||
680 | if (Cond) | |||
681 | RecursivelyDeleteTriviallyDeadInstructions(Cond); | |||
682 | } | |||
683 | ||||
684 | /// Return true if the specified terminator checks | |||
685 | /// to see if a value is equal to constant integer value. | |||
686 | Value *SimplifyCFGOpt::isValueEqualityComparison(TerminatorInst *TI) { | |||
687 | Value *CV = nullptr; | |||
688 | if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { | |||
689 | // Do not permit merging of large switch instructions into their | |||
690 | // predecessors unless there is only one predecessor. | |||
691 | if (SI->getNumSuccessors() * std::distance(pred_begin(SI->getParent()), | |||
692 | pred_end(SI->getParent())) <= | |||
693 | 128) | |||
694 | CV = SI->getCondition(); | |||
695 | } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) | |||
696 | if (BI->isConditional() && BI->getCondition()->hasOneUse()) | |||
697 | if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) { | |||
698 | if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL)) | |||
699 | CV = ICI->getOperand(0); | |||
700 | } | |||
701 | ||||
702 | // Unwrap any lossless ptrtoint cast. | |||
703 | if (CV) { | |||
704 | if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) { | |||
705 | Value *Ptr = PTII->getPointerOperand(); | |||
706 | if (PTII->getType() == DL.getIntPtrType(Ptr->getType())) | |||
707 | CV = Ptr; | |||
708 | } | |||
709 | } | |||
710 | return CV; | |||
711 | } | |||
712 | ||||
713 | /// Given a value comparison instruction, | |||
714 | /// decode all of the 'cases' that it represents and return the 'default' block. | |||
715 | BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases( | |||
716 | TerminatorInst *TI, std::vector<ValueEqualityComparisonCase> &Cases) { | |||
717 | if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { | |||
718 | Cases.reserve(SI->getNumCases()); | |||
719 | for (auto Case : SI->cases()) | |||
720 | Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(), | |||
721 | Case.getCaseSuccessor())); | |||
722 | return SI->getDefaultDest(); | |||
723 | } | |||
724 | ||||
725 | BranchInst *BI = cast<BranchInst>(TI); | |||
726 | ICmpInst *ICI = cast<ICmpInst>(BI->getCondition()); | |||
727 | BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE); | |||
728 | Cases.push_back(ValueEqualityComparisonCase( | |||
729 | GetConstantInt(ICI->getOperand(1), DL), Succ)); | |||
730 | return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ); | |||
731 | } | |||
732 | ||||
733 | /// Given a vector of bb/value pairs, remove any entries | |||
734 | /// in the list that match the specified block. | |||
735 | static void | |||
736 | EliminateBlockCases(BasicBlock *BB, | |||
737 | std::vector<ValueEqualityComparisonCase> &Cases) { | |||
738 | Cases.erase(std::remove(Cases.begin(), Cases.end(), BB), Cases.end()); | |||
739 | } | |||
740 | ||||
741 | /// Return true if there are any keys in C1 that exist in C2 as well. | |||
742 | static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1, | |||
743 | std::vector<ValueEqualityComparisonCase> &C2) { | |||
744 | std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2; | |||
745 | ||||
746 | // Make V1 be smaller than V2. | |||
747 | if (V1->size() > V2->size()) | |||
748 | std::swap(V1, V2); | |||
749 | ||||
750 | if (V1->empty()) | |||
751 | return false; | |||
752 | if (V1->size() == 1) { | |||
753 | // Just scan V2. | |||
754 | ConstantInt *TheVal = (*V1)[0].Value; | |||
755 | for (unsigned i = 0, e = V2->size(); i != e; ++i) | |||
756 | if (TheVal == (*V2)[i].Value) | |||
757 | return true; | |||
758 | } | |||
759 | ||||
760 | // Otherwise, just sort both lists and compare element by element. | |||
761 | array_pod_sort(V1->begin(), V1->end()); | |||
762 | array_pod_sort(V2->begin(), V2->end()); | |||
763 | unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size(); | |||
764 | while (i1 != e1 && i2 != e2) { | |||
765 | if ((*V1)[i1].Value == (*V2)[i2].Value) | |||
766 | return true; | |||
767 | if ((*V1)[i1].Value < (*V2)[i2].Value) | |||
768 | ++i1; | |||
769 | else | |||
770 | ++i2; | |||
771 | } | |||
772 | return false; | |||
773 | } | |||
774 | ||||
775 | // Set branch weights on SwitchInst. This sets the metadata if there is at | |||
776 | // least one non-zero weight. | |||
777 | static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights) { | |||
778 | // Check that there is at least one non-zero weight. Otherwise, pass | |||
779 | // nullptr to setMetadata which will erase the existing metadata. | |||
780 | MDNode *N = nullptr; | |||
781 | if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; })) | |||
782 | N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights); | |||
783 | SI->setMetadata(LLVMContext::MD_prof, N); | |||
784 | } | |||
785 | ||||
786 | // Similar to the above, but for branch and select instructions that take | |||
787 | // exactly 2 weights. | |||
788 | static void setBranchWeights(Instruction *I, uint32_t TrueWeight, | |||
789 | uint32_t FalseWeight) { | |||
790 | assert(isa<BranchInst>(I) || isa<SelectInst>(I))(static_cast <bool> (isa<BranchInst>(I) || isa< SelectInst>(I)) ? void (0) : __assert_fail ("isa<BranchInst>(I) || isa<SelectInst>(I)" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 790, __extension__ __PRETTY_FUNCTION__)); | |||
791 | // Check that there is at least one non-zero weight. Otherwise, pass | |||
792 | // nullptr to setMetadata which will erase the existing metadata. | |||
793 | MDNode *N = nullptr; | |||
794 | if (TrueWeight || FalseWeight) | |||
795 | N = MDBuilder(I->getParent()->getContext()) | |||
796 | .createBranchWeights(TrueWeight, FalseWeight); | |||
797 | I->setMetadata(LLVMContext::MD_prof, N); | |||
798 | } | |||
799 | ||||
800 | /// If TI is known to be a terminator instruction and its block is known to | |||
801 | /// only have a single predecessor block, check to see if that predecessor is | |||
802 | /// also a value comparison with the same value, and if that comparison | |||
803 | /// determines the outcome of this comparison. If so, simplify TI. This does a | |||
804 | /// very limited form of jump threading. | |||
805 | bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor( | |||
806 | TerminatorInst *TI, BasicBlock *Pred, IRBuilder<> &Builder) { | |||
807 | Value *PredVal = isValueEqualityComparison(Pred->getTerminator()); | |||
808 | if (!PredVal) | |||
809 | return false; // Not a value comparison in predecessor. | |||
810 | ||||
811 | Value *ThisVal = isValueEqualityComparison(TI); | |||
812 | assert(ThisVal && "This isn't a value comparison!!")(static_cast <bool> (ThisVal && "This isn't a value comparison!!" ) ? void (0) : __assert_fail ("ThisVal && \"This isn't a value comparison!!\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 812, __extension__ __PRETTY_FUNCTION__)); | |||
813 | if (ThisVal != PredVal) | |||
814 | return false; // Different predicates. | |||
815 | ||||
816 | // TODO: Preserve branch weight metadata, similarly to how | |||
817 | // FoldValueComparisonIntoPredecessors preserves it. | |||
818 | ||||
819 | // Find out information about when control will move from Pred to TI's block. | |||
820 | std::vector<ValueEqualityComparisonCase> PredCases; | |||
821 | BasicBlock *PredDef = | |||
822 | GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases); | |||
823 | EliminateBlockCases(PredDef, PredCases); // Remove default from cases. | |||
824 | ||||
825 | // Find information about how control leaves this block. | |||
826 | std::vector<ValueEqualityComparisonCase> ThisCases; | |||
827 | BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases); | |||
828 | EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases. | |||
829 | ||||
830 | // If TI's block is the default block from Pred's comparison, potentially | |||
831 | // simplify TI based on this knowledge. | |||
832 | if (PredDef == TI->getParent()) { | |||
833 | // If we are here, we know that the value is none of those cases listed in | |||
834 | // PredCases. If there are any cases in ThisCases that are in PredCases, we | |||
835 | // can simplify TI. | |||
836 | if (!ValuesOverlap(PredCases, ThisCases)) | |||
837 | return false; | |||
838 | ||||
839 | if (isa<BranchInst>(TI)) { | |||
840 | // Okay, one of the successors of this condbr is dead. Convert it to a | |||
841 | // uncond br. | |||
842 | assert(ThisCases.size() == 1 && "Branch can only have one case!")(static_cast <bool> (ThisCases.size() == 1 && "Branch can only have one case!" ) ? void (0) : __assert_fail ("ThisCases.size() == 1 && \"Branch can only have one case!\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 842, __extension__ __PRETTY_FUNCTION__)); | |||
843 | // Insert the new branch. | |||
844 | Instruction *NI = Builder.CreateBr(ThisDef); | |||
845 | (void)NI; | |||
846 | ||||
847 | // Remove PHI node entries for the dead edge. | |||
848 | ThisCases[0].Dest->removePredecessor(TI->getParent()); | |||
849 | ||||
850 | DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Threading pred instr: " << *Pred->getTerminator() << "Through successor TI: " << *TI << "Leaving: " << *NI << "\n"; } } while (false) | |||
851 | << "Through successor TI: " << *TI << "Leaving: " << *NIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Threading pred instr: " << *Pred->getTerminator() << "Through successor TI: " << *TI << "Leaving: " << *NI << "\n"; } } while (false) | |||
852 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Threading pred instr: " << *Pred->getTerminator() << "Through successor TI: " << *TI << "Leaving: " << *NI << "\n"; } } while (false); | |||
853 | ||||
854 | EraseTerminatorInstAndDCECond(TI); | |||
855 | return true; | |||
856 | } | |||
857 | ||||
858 | SwitchInst *SI = cast<SwitchInst>(TI); | |||
859 | // Okay, TI has cases that are statically dead, prune them away. | |||
860 | SmallPtrSet<Constant *, 16> DeadCases; | |||
861 | for (unsigned i = 0, e = PredCases.size(); i != e; ++i) | |||
862 | DeadCases.insert(PredCases[i].Value); | |||
863 | ||||
864 | DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Threading pred instr: " << *Pred->getTerminator() << "Through successor TI: " << *TI; } } while (false) | |||
865 | << "Through successor TI: " << *TI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Threading pred instr: " << *Pred->getTerminator() << "Through successor TI: " << *TI; } } while (false); | |||
866 | ||||
867 | // Collect branch weights into a vector. | |||
868 | SmallVector<uint32_t, 8> Weights; | |||
869 | MDNode *MD = SI->getMetadata(LLVMContext::MD_prof); | |||
870 | bool HasWeight = MD && (MD->getNumOperands() == 2 + SI->getNumCases()); | |||
871 | if (HasWeight) | |||
872 | for (unsigned MD_i = 1, MD_e = MD->getNumOperands(); MD_i < MD_e; | |||
873 | ++MD_i) { | |||
874 | ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(MD_i)); | |||
875 | Weights.push_back(CI->getValue().getZExtValue()); | |||
876 | } | |||
877 | for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) { | |||
878 | --i; | |||
879 | if (DeadCases.count(i->getCaseValue())) { | |||
880 | if (HasWeight) { | |||
881 | std::swap(Weights[i->getCaseIndex() + 1], Weights.back()); | |||
882 | Weights.pop_back(); | |||
883 | } | |||
884 | i->getCaseSuccessor()->removePredecessor(TI->getParent()); | |||
885 | SI->removeCase(i); | |||
886 | } | |||
887 | } | |||
888 | if (HasWeight && Weights.size() >= 2) | |||
889 | setBranchWeights(SI, Weights); | |||
890 | ||||
891 | DEBUG(dbgs() << "Leaving: " << *TI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Leaving: " << *TI << "\n"; } } while (false); | |||
892 | return true; | |||
893 | } | |||
894 | ||||
895 | // Otherwise, TI's block must correspond to some matched value. Find out | |||
896 | // which value (or set of values) this is. | |||
897 | ConstantInt *TIV = nullptr; | |||
898 | BasicBlock *TIBB = TI->getParent(); | |||
899 | for (unsigned i = 0, e = PredCases.size(); i != e; ++i) | |||
900 | if (PredCases[i].Dest == TIBB) { | |||
901 | if (TIV) | |||
902 | return false; // Cannot handle multiple values coming to this block. | |||
903 | TIV = PredCases[i].Value; | |||
904 | } | |||
905 | assert(TIV && "No edge from pred to succ?")(static_cast <bool> (TIV && "No edge from pred to succ?" ) ? void (0) : __assert_fail ("TIV && \"No edge from pred to succ?\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 905, __extension__ __PRETTY_FUNCTION__)); | |||
906 | ||||
907 | // Okay, we found the one constant that our value can be if we get into TI's | |||
908 | // BB. Find out which successor will unconditionally be branched to. | |||
909 | BasicBlock *TheRealDest = nullptr; | |||
910 | for (unsigned i = 0, e = ThisCases.size(); i != e; ++i) | |||
911 | if (ThisCases[i].Value == TIV) { | |||
912 | TheRealDest = ThisCases[i].Dest; | |||
913 | break; | |||
914 | } | |||
915 | ||||
916 | // If not handled by any explicit cases, it is handled by the default case. | |||
917 | if (!TheRealDest) | |||
918 | TheRealDest = ThisDef; | |||
919 | ||||
920 | // Remove PHI node entries for dead edges. | |||
921 | BasicBlock *CheckEdge = TheRealDest; | |||
922 | for (BasicBlock *Succ : successors(TIBB)) | |||
923 | if (Succ != CheckEdge) | |||
924 | Succ->removePredecessor(TIBB); | |||
925 | else | |||
926 | CheckEdge = nullptr; | |||
927 | ||||
928 | // Insert the new branch. | |||
929 | Instruction *NI = Builder.CreateBr(TheRealDest); | |||
930 | (void)NI; | |||
931 | ||||
932 | DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Threading pred instr: " << *Pred->getTerminator() << "Through successor TI: " << *TI << "Leaving: " << *NI << "\n"; } } while (false) | |||
933 | << "Through successor TI: " << *TI << "Leaving: " << *NIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Threading pred instr: " << *Pred->getTerminator() << "Through successor TI: " << *TI << "Leaving: " << *NI << "\n"; } } while (false) | |||
934 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Threading pred instr: " << *Pred->getTerminator() << "Through successor TI: " << *TI << "Leaving: " << *NI << "\n"; } } while (false); | |||
935 | ||||
936 | EraseTerminatorInstAndDCECond(TI); | |||
937 | return true; | |||
938 | } | |||
939 | ||||
940 | namespace { | |||
941 | ||||
942 | /// This class implements a stable ordering of constant | |||
943 | /// integers that does not depend on their address. This is important for | |||
944 | /// applications that sort ConstantInt's to ensure uniqueness. | |||
945 | struct ConstantIntOrdering { | |||
946 | bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const { | |||
947 | return LHS->getValue().ult(RHS->getValue()); | |||
948 | } | |||
949 | }; | |||
950 | ||||
951 | } // end anonymous namespace | |||
952 | ||||
953 | static int ConstantIntSortPredicate(ConstantInt *const *P1, | |||
954 | ConstantInt *const *P2) { | |||
955 | const ConstantInt *LHS = *P1; | |||
956 | const ConstantInt *RHS = *P2; | |||
957 | if (LHS == RHS) | |||
958 | return 0; | |||
959 | return LHS->getValue().ult(RHS->getValue()) ? 1 : -1; | |||
960 | } | |||
961 | ||||
962 | static inline bool HasBranchWeights(const Instruction *I) { | |||
963 | MDNode *ProfMD = I->getMetadata(LLVMContext::MD_prof); | |||
964 | if (ProfMD && ProfMD->getOperand(0)) | |||
965 | if (MDString *MDS = dyn_cast<MDString>(ProfMD->getOperand(0))) | |||
966 | return MDS->getString().equals("branch_weights"); | |||
967 | ||||
968 | return false; | |||
969 | } | |||
970 | ||||
971 | /// Get Weights of a given TerminatorInst, the default weight is at the front | |||
972 | /// of the vector. If TI is a conditional eq, we need to swap the branch-weight | |||
973 | /// metadata. | |||
974 | static void GetBranchWeights(TerminatorInst *TI, | |||
975 | SmallVectorImpl<uint64_t> &Weights) { | |||
976 | MDNode *MD = TI->getMetadata(LLVMContext::MD_prof); | |||
977 | assert(MD)(static_cast <bool> (MD) ? void (0) : __assert_fail ("MD" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 977, __extension__ __PRETTY_FUNCTION__)); | |||
978 | for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) { | |||
979 | ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(i)); | |||
980 | Weights.push_back(CI->getValue().getZExtValue()); | |||
981 | } | |||
982 | ||||
983 | // If TI is a conditional eq, the default case is the false case, | |||
984 | // and the corresponding branch-weight data is at index 2. We swap the | |||
985 | // default weight to be the first entry. | |||
986 | if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { | |||
987 | assert(Weights.size() == 2)(static_cast <bool> (Weights.size() == 2) ? void (0) : __assert_fail ("Weights.size() == 2", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 987, __extension__ __PRETTY_FUNCTION__)); | |||
988 | ICmpInst *ICI = cast<ICmpInst>(BI->getCondition()); | |||
989 | if (ICI->getPredicate() == ICmpInst::ICMP_EQ) | |||
990 | std::swap(Weights.front(), Weights.back()); | |||
991 | } | |||
992 | } | |||
993 | ||||
994 | /// Keep halving the weights until all can fit in uint32_t. | |||
995 | static void FitWeights(MutableArrayRef<uint64_t> Weights) { | |||
996 | uint64_t Max = *std::max_element(Weights.begin(), Weights.end()); | |||
997 | if (Max > UINT_MAX(2147483647 *2U +1U)) { | |||
998 | unsigned Offset = 32 - countLeadingZeros(Max); | |||
999 | for (uint64_t &I : Weights) | |||
1000 | I >>= Offset; | |||
1001 | } | |||
1002 | } | |||
1003 | ||||
1004 | /// The specified terminator is a value equality comparison instruction | |||
1005 | /// (either a switch or a branch on "X == c"). | |||
1006 | /// See if any of the predecessors of the terminator block are value comparisons | |||
1007 | /// on the same value. If so, and if safe to do so, fold them together. | |||
1008 | bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI, | |||
1009 | IRBuilder<> &Builder) { | |||
1010 | BasicBlock *BB = TI->getParent(); | |||
1011 | Value *CV = isValueEqualityComparison(TI); // CondVal | |||
1012 | assert(CV && "Not a comparison?")(static_cast <bool> (CV && "Not a comparison?") ? void (0) : __assert_fail ("CV && \"Not a comparison?\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 1012, __extension__ __PRETTY_FUNCTION__)); | |||
1013 | bool Changed = false; | |||
1014 | ||||
1015 | SmallVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB)); | |||
1016 | while (!Preds.empty()) { | |||
1017 | BasicBlock *Pred = Preds.pop_back_val(); | |||
1018 | ||||
1019 | // See if the predecessor is a comparison with the same value. | |||
1020 | TerminatorInst *PTI = Pred->getTerminator(); | |||
1021 | Value *PCV = isValueEqualityComparison(PTI); // PredCondVal | |||
1022 | ||||
1023 | if (PCV == CV && TI != PTI) { | |||
1024 | SmallSetVector<BasicBlock*, 4> FailBlocks; | |||
1025 | if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) { | |||
1026 | for (auto *Succ : FailBlocks) { | |||
1027 | if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split")) | |||
1028 | return false; | |||
1029 | } | |||
1030 | } | |||
1031 | ||||
1032 | // Figure out which 'cases' to copy from SI to PSI. | |||
1033 | std::vector<ValueEqualityComparisonCase> BBCases; | |||
1034 | BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases); | |||
1035 | ||||
1036 | std::vector<ValueEqualityComparisonCase> PredCases; | |||
1037 | BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases); | |||
1038 | ||||
1039 | // Based on whether the default edge from PTI goes to BB or not, fill in | |||
1040 | // PredCases and PredDefault with the new switch cases we would like to | |||
1041 | // build. | |||
1042 | SmallVector<BasicBlock *, 8> NewSuccessors; | |||
1043 | ||||
1044 | // Update the branch weight metadata along the way | |||
1045 | SmallVector<uint64_t, 8> Weights; | |||
1046 | bool PredHasWeights = HasBranchWeights(PTI); | |||
1047 | bool SuccHasWeights = HasBranchWeights(TI); | |||
1048 | ||||
1049 | if (PredHasWeights) { | |||
1050 | GetBranchWeights(PTI, Weights); | |||
1051 | // branch-weight metadata is inconsistent here. | |||
1052 | if (Weights.size() != 1 + PredCases.size()) | |||
1053 | PredHasWeights = SuccHasWeights = false; | |||
1054 | } else if (SuccHasWeights) | |||
1055 | // If there are no predecessor weights but there are successor weights, | |||
1056 | // populate Weights with 1, which will later be scaled to the sum of | |||
1057 | // successor's weights | |||
1058 | Weights.assign(1 + PredCases.size(), 1); | |||
1059 | ||||
1060 | SmallVector<uint64_t, 8> SuccWeights; | |||
1061 | if (SuccHasWeights) { | |||
1062 | GetBranchWeights(TI, SuccWeights); | |||
1063 | // branch-weight metadata is inconsistent here. | |||
1064 | if (SuccWeights.size() != 1 + BBCases.size()) | |||
1065 | PredHasWeights = SuccHasWeights = false; | |||
1066 | } else if (PredHasWeights) | |||
1067 | SuccWeights.assign(1 + BBCases.size(), 1); | |||
1068 | ||||
1069 | if (PredDefault == BB) { | |||
1070 | // If this is the default destination from PTI, only the edges in TI | |||
1071 | // that don't occur in PTI, or that branch to BB will be activated. | |||
1072 | std::set<ConstantInt *, ConstantIntOrdering> PTIHandled; | |||
1073 | for (unsigned i = 0, e = PredCases.size(); i != e; ++i) | |||
1074 | if (PredCases[i].Dest != BB) | |||
1075 | PTIHandled.insert(PredCases[i].Value); | |||
1076 | else { | |||
1077 | // The default destination is BB, we don't need explicit targets. | |||
1078 | std::swap(PredCases[i], PredCases.back()); | |||
1079 | ||||
1080 | if (PredHasWeights || SuccHasWeights) { | |||
1081 | // Increase weight for the default case. | |||
1082 | Weights[0] += Weights[i + 1]; | |||
1083 | std::swap(Weights[i + 1], Weights.back()); | |||
1084 | Weights.pop_back(); | |||
1085 | } | |||
1086 | ||||
1087 | PredCases.pop_back(); | |||
1088 | --i; | |||
1089 | --e; | |||
1090 | } | |||
1091 | ||||
1092 | // Reconstruct the new switch statement we will be building. | |||
1093 | if (PredDefault != BBDefault) { | |||
1094 | PredDefault->removePredecessor(Pred); | |||
1095 | PredDefault = BBDefault; | |||
1096 | NewSuccessors.push_back(BBDefault); | |||
1097 | } | |||
1098 | ||||
1099 | unsigned CasesFromPred = Weights.size(); | |||
1100 | uint64_t ValidTotalSuccWeight = 0; | |||
1101 | for (unsigned i = 0, e = BBCases.size(); i != e; ++i) | |||
1102 | if (!PTIHandled.count(BBCases[i].Value) && | |||
1103 | BBCases[i].Dest != BBDefault) { | |||
1104 | PredCases.push_back(BBCases[i]); | |||
1105 | NewSuccessors.push_back(BBCases[i].Dest); | |||
1106 | if (SuccHasWeights || PredHasWeights) { | |||
1107 | // The default weight is at index 0, so weight for the ith case | |||
1108 | // should be at index i+1. Scale the cases from successor by | |||
1109 | // PredDefaultWeight (Weights[0]). | |||
1110 | Weights.push_back(Weights[0] * SuccWeights[i + 1]); | |||
1111 | ValidTotalSuccWeight += SuccWeights[i + 1]; | |||
1112 | } | |||
1113 | } | |||
1114 | ||||
1115 | if (SuccHasWeights || PredHasWeights) { | |||
1116 | ValidTotalSuccWeight += SuccWeights[0]; | |||
1117 | // Scale the cases from predecessor by ValidTotalSuccWeight. | |||
1118 | for (unsigned i = 1; i < CasesFromPred; ++i) | |||
1119 | Weights[i] *= ValidTotalSuccWeight; | |||
1120 | // Scale the default weight by SuccDefaultWeight (SuccWeights[0]). | |||
1121 | Weights[0] *= SuccWeights[0]; | |||
1122 | } | |||
1123 | } else { | |||
1124 | // If this is not the default destination from PSI, only the edges | |||
1125 | // in SI that occur in PSI with a destination of BB will be | |||
1126 | // activated. | |||
1127 | std::set<ConstantInt *, ConstantIntOrdering> PTIHandled; | |||
1128 | std::map<ConstantInt *, uint64_t> WeightsForHandled; | |||
1129 | for (unsigned i = 0, e = PredCases.size(); i != e; ++i) | |||
1130 | if (PredCases[i].Dest == BB) { | |||
1131 | PTIHandled.insert(PredCases[i].Value); | |||
1132 | ||||
1133 | if (PredHasWeights || SuccHasWeights) { | |||
1134 | WeightsForHandled[PredCases[i].Value] = Weights[i + 1]; | |||
1135 | std::swap(Weights[i + 1], Weights.back()); | |||
1136 | Weights.pop_back(); | |||
1137 | } | |||
1138 | ||||
1139 | std::swap(PredCases[i], PredCases.back()); | |||
1140 | PredCases.pop_back(); | |||
1141 | --i; | |||
1142 | --e; | |||
1143 | } | |||
1144 | ||||
1145 | // Okay, now we know which constants were sent to BB from the | |||
1146 | // predecessor. Figure out where they will all go now. | |||
1147 | for (unsigned i = 0, e = BBCases.size(); i != e; ++i) | |||
1148 | if (PTIHandled.count(BBCases[i].Value)) { | |||
1149 | // If this is one we are capable of getting... | |||
1150 | if (PredHasWeights || SuccHasWeights) | |||
1151 | Weights.push_back(WeightsForHandled[BBCases[i].Value]); | |||
1152 | PredCases.push_back(BBCases[i]); | |||
1153 | NewSuccessors.push_back(BBCases[i].Dest); | |||
1154 | PTIHandled.erase( | |||
1155 | BBCases[i].Value); // This constant is taken care of | |||
1156 | } | |||
1157 | ||||
1158 | // If there are any constants vectored to BB that TI doesn't handle, | |||
1159 | // they must go to the default destination of TI. | |||
1160 | for (ConstantInt *I : PTIHandled) { | |||
1161 | if (PredHasWeights || SuccHasWeights) | |||
1162 | Weights.push_back(WeightsForHandled[I]); | |||
1163 | PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault)); | |||
1164 | NewSuccessors.push_back(BBDefault); | |||
1165 | } | |||
1166 | } | |||
1167 | ||||
1168 | // Okay, at this point, we know which new successor Pred will get. Make | |||
1169 | // sure we update the number of entries in the PHI nodes for these | |||
1170 | // successors. | |||
1171 | for (BasicBlock *NewSuccessor : NewSuccessors) | |||
1172 | AddPredecessorToBlock(NewSuccessor, Pred, BB); | |||
1173 | ||||
1174 | Builder.SetInsertPoint(PTI); | |||
1175 | // Convert pointer to int before we switch. | |||
1176 | if (CV->getType()->isPointerTy()) { | |||
1177 | CV = Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()), | |||
1178 | "magicptr"); | |||
1179 | } | |||
1180 | ||||
1181 | // Now that the successors are updated, create the new Switch instruction. | |||
1182 | SwitchInst *NewSI = | |||
1183 | Builder.CreateSwitch(CV, PredDefault, PredCases.size()); | |||
1184 | NewSI->setDebugLoc(PTI->getDebugLoc()); | |||
1185 | for (ValueEqualityComparisonCase &V : PredCases) | |||
1186 | NewSI->addCase(V.Value, V.Dest); | |||
1187 | ||||
1188 | if (PredHasWeights || SuccHasWeights) { | |||
1189 | // Halve the weights if any of them cannot fit in an uint32_t | |||
1190 | FitWeights(Weights); | |||
1191 | ||||
1192 | SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end()); | |||
1193 | ||||
1194 | setBranchWeights(NewSI, MDWeights); | |||
1195 | } | |||
1196 | ||||
1197 | EraseTerminatorInstAndDCECond(PTI); | |||
1198 | ||||
1199 | // Okay, last check. If BB is still a successor of PSI, then we must | |||
1200 | // have an infinite loop case. If so, add an infinitely looping block | |||
1201 | // to handle the case to preserve the behavior of the code. | |||
1202 | BasicBlock *InfLoopBlock = nullptr; | |||
1203 | for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i) | |||
1204 | if (NewSI->getSuccessor(i) == BB) { | |||
1205 | if (!InfLoopBlock) { | |||
1206 | // Insert it at the end of the function, because it's either code, | |||
1207 | // or it won't matter if it's hot. :) | |||
1208 | InfLoopBlock = BasicBlock::Create(BB->getContext(), "infloop", | |||
1209 | BB->getParent()); | |||
1210 | BranchInst::Create(InfLoopBlock, InfLoopBlock); | |||
1211 | } | |||
1212 | NewSI->setSuccessor(i, InfLoopBlock); | |||
1213 | } | |||
1214 | ||||
1215 | Changed = true; | |||
1216 | } | |||
1217 | } | |||
1218 | return Changed; | |||
1219 | } | |||
1220 | ||||
1221 | // If we would need to insert a select that uses the value of this invoke | |||
1222 | // (comments in HoistThenElseCodeToIf explain why we would need to do this), we | |||
1223 | // can't hoist the invoke, as there is nowhere to put the select in this case. | |||
1224 | static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2, | |||
1225 | Instruction *I1, Instruction *I2) { | |||
1226 | for (BasicBlock *Succ : successors(BB1)) { | |||
1227 | for (const PHINode &PN : Succ->phis()) { | |||
1228 | Value *BB1V = PN.getIncomingValueForBlock(BB1); | |||
1229 | Value *BB2V = PN.getIncomingValueForBlock(BB2); | |||
1230 | if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) { | |||
1231 | return false; | |||
1232 | } | |||
1233 | } | |||
1234 | } | |||
1235 | return true; | |||
1236 | } | |||
1237 | ||||
1238 | static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I); | |||
1239 | ||||
1240 | /// Given a conditional branch that goes to BB1 and BB2, hoist any common code | |||
1241 | /// in the two blocks up into the branch block. The caller of this function | |||
1242 | /// guarantees that BI's block dominates BB1 and BB2. | |||
1243 | static bool HoistThenElseCodeToIf(BranchInst *BI, | |||
1244 | const TargetTransformInfo &TTI) { | |||
1245 | // This does very trivial matching, with limited scanning, to find identical | |||
1246 | // instructions in the two blocks. In particular, we don't want to get into | |||
1247 | // O(M*N) situations here where M and N are the sizes of BB1 and BB2. As | |||
1248 | // such, we currently just scan for obviously identical instructions in an | |||
1249 | // identical order. | |||
1250 | BasicBlock *BB1 = BI->getSuccessor(0); // The true destination. | |||
1251 | BasicBlock *BB2 = BI->getSuccessor(1); // The false destination | |||
1252 | ||||
1253 | BasicBlock::iterator BB1_Itr = BB1->begin(); | |||
1254 | BasicBlock::iterator BB2_Itr = BB2->begin(); | |||
1255 | ||||
1256 | Instruction *I1 = &*BB1_Itr++, *I2 = &*BB2_Itr++; | |||
1257 | // Skip debug info if it is not identical. | |||
1258 | DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1); | |||
1259 | DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2); | |||
1260 | if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) { | |||
1261 | while (isa<DbgInfoIntrinsic>(I1)) | |||
1262 | I1 = &*BB1_Itr++; | |||
1263 | while (isa<DbgInfoIntrinsic>(I2)) | |||
1264 | I2 = &*BB2_Itr++; | |||
1265 | } | |||
1266 | if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) || | |||
1267 | (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))) | |||
1268 | return false; | |||
1269 | ||||
1270 | BasicBlock *BIParent = BI->getParent(); | |||
1271 | ||||
1272 | bool Changed = false; | |||
1273 | do { | |||
1274 | // If we are hoisting the terminator instruction, don't move one (making a | |||
1275 | // broken BB), instead clone it, and remove BI. | |||
1276 | if (isa<TerminatorInst>(I1)) | |||
1277 | goto HoistTerminator; | |||
1278 | ||||
1279 | // If we're going to hoist a call, make sure that the two instructions we're | |||
1280 | // commoning/hoisting are both marked with musttail, or neither of them is | |||
1281 | // marked as such. Otherwise, we might end up in a situation where we hoist | |||
1282 | // from a block where the terminator is a `ret` to a block where the terminator | |||
1283 | // is a `br`, and `musttail` calls expect to be followed by a return. | |||
1284 | auto *C1 = dyn_cast<CallInst>(I1); | |||
1285 | auto *C2 = dyn_cast<CallInst>(I2); | |||
1286 | if (C1 && C2) | |||
1287 | if (C1->isMustTailCall() != C2->isMustTailCall()) | |||
1288 | return Changed; | |||
1289 | ||||
1290 | if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2)) | |||
1291 | return Changed; | |||
1292 | ||||
1293 | // For a normal instruction, we just move one to right before the branch, | |||
1294 | // then replace all uses of the other with the first. Finally, we remove | |||
1295 | // the now redundant second instruction. | |||
1296 | BIParent->getInstList().splice(BI->getIterator(), BB1->getInstList(), I1); | |||
1297 | if (!I2->use_empty()) | |||
1298 | I2->replaceAllUsesWith(I1); | |||
1299 | I1->andIRFlags(I2); | |||
1300 | unsigned KnownIDs[] = {LLVMContext::MD_tbaa, | |||
1301 | LLVMContext::MD_range, | |||
1302 | LLVMContext::MD_fpmath, | |||
1303 | LLVMContext::MD_invariant_load, | |||
1304 | LLVMContext::MD_nonnull, | |||
1305 | LLVMContext::MD_invariant_group, | |||
1306 | LLVMContext::MD_align, | |||
1307 | LLVMContext::MD_dereferenceable, | |||
1308 | LLVMContext::MD_dereferenceable_or_null, | |||
1309 | LLVMContext::MD_mem_parallel_loop_access}; | |||
1310 | combineMetadata(I1, I2, KnownIDs); | |||
1311 | ||||
1312 | // I1 and I2 are being combined into a single instruction. Its debug | |||
1313 | // location is the merged locations of the original instructions. | |||
1314 | I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc()); | |||
1315 | ||||
1316 | I2->eraseFromParent(); | |||
1317 | Changed = true; | |||
1318 | ||||
1319 | I1 = &*BB1_Itr++; | |||
1320 | I2 = &*BB2_Itr++; | |||
1321 | // Skip debug info if it is not identical. | |||
1322 | DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1); | |||
1323 | DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2); | |||
1324 | if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) { | |||
1325 | while (isa<DbgInfoIntrinsic>(I1)) | |||
1326 | I1 = &*BB1_Itr++; | |||
1327 | while (isa<DbgInfoIntrinsic>(I2)) | |||
1328 | I2 = &*BB2_Itr++; | |||
1329 | } | |||
1330 | } while (I1->isIdenticalToWhenDefined(I2)); | |||
1331 | ||||
1332 | return true; | |||
1333 | ||||
1334 | HoistTerminator: | |||
1335 | // It may not be possible to hoist an invoke. | |||
1336 | if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) | |||
1337 | return Changed; | |||
1338 | ||||
1339 | for (BasicBlock *Succ : successors(BB1)) { | |||
1340 | for (PHINode &PN : Succ->phis()) { | |||
1341 | Value *BB1V = PN.getIncomingValueForBlock(BB1); | |||
1342 | Value *BB2V = PN.getIncomingValueForBlock(BB2); | |||
1343 | if (BB1V == BB2V) | |||
1344 | continue; | |||
1345 | ||||
1346 | // Check for passingValueIsAlwaysUndefined here because we would rather | |||
1347 | // eliminate undefined control flow then converting it to a select. | |||
1348 | if (passingValueIsAlwaysUndefined(BB1V, &PN) || | |||
1349 | passingValueIsAlwaysUndefined(BB2V, &PN)) | |||
1350 | return Changed; | |||
1351 | ||||
1352 | if (isa<ConstantExpr>(BB1V) && !isSafeToSpeculativelyExecute(BB1V)) | |||
1353 | return Changed; | |||
1354 | if (isa<ConstantExpr>(BB2V) && !isSafeToSpeculativelyExecute(BB2V)) | |||
1355 | return Changed; | |||
1356 | } | |||
1357 | } | |||
1358 | ||||
1359 | // Okay, it is safe to hoist the terminator. | |||
1360 | Instruction *NT = I1->clone(); | |||
1361 | BIParent->getInstList().insert(BI->getIterator(), NT); | |||
1362 | if (!NT->getType()->isVoidTy()) { | |||
1363 | I1->replaceAllUsesWith(NT); | |||
1364 | I2->replaceAllUsesWith(NT); | |||
1365 | NT->takeName(I1); | |||
1366 | } | |||
1367 | ||||
1368 | IRBuilder<NoFolder> Builder(NT); | |||
1369 | // Hoisting one of the terminators from our successor is a great thing. | |||
1370 | // Unfortunately, the successors of the if/else blocks may have PHI nodes in | |||
1371 | // them. If they do, all PHI entries for BB1/BB2 must agree for all PHI | |||
1372 | // nodes, so we insert select instruction to compute the final result. | |||
1373 | std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects; | |||
1374 | for (BasicBlock *Succ : successors(BB1)) { | |||
1375 | for (PHINode &PN : Succ->phis()) { | |||
1376 | Value *BB1V = PN.getIncomingValueForBlock(BB1); | |||
1377 | Value *BB2V = PN.getIncomingValueForBlock(BB2); | |||
1378 | if (BB1V == BB2V) | |||
1379 | continue; | |||
1380 | ||||
1381 | // These values do not agree. Insert a select instruction before NT | |||
1382 | // that determines the right value. | |||
1383 | SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)]; | |||
1384 | if (!SI) | |||
1385 | SI = cast<SelectInst>( | |||
1386 | Builder.CreateSelect(BI->getCondition(), BB1V, BB2V, | |||
1387 | BB1V->getName() + "." + BB2V->getName(), BI)); | |||
1388 | ||||
1389 | // Make the PHI node use the select for all incoming values for BB1/BB2 | |||
1390 | for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) | |||
1391 | if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2) | |||
1392 | PN.setIncomingValue(i, SI); | |||
1393 | } | |||
1394 | } | |||
1395 | ||||
1396 | // Update any PHI nodes in our new successors. | |||
1397 | for (BasicBlock *Succ : successors(BB1)) | |||
1398 | AddPredecessorToBlock(Succ, BIParent, BB1); | |||
1399 | ||||
1400 | EraseTerminatorInstAndDCECond(BI); | |||
1401 | return true; | |||
1402 | } | |||
1403 | ||||
1404 | // All instructions in Insts belong to different blocks that all unconditionally | |||
1405 | // branch to a common successor. Analyze each instruction and return true if it | |||
1406 | // would be possible to sink them into their successor, creating one common | |||
1407 | // instruction instead. For every value that would be required to be provided by | |||
1408 | // PHI node (because an operand varies in each input block), add to PHIOperands. | |||
1409 | static bool canSinkInstructions( | |||
1410 | ArrayRef<Instruction *> Insts, | |||
1411 | DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) { | |||
1412 | // Prune out obviously bad instructions to move. Any non-store instruction | |||
1413 | // must have exactly one use, and we check later that use is by a single, | |||
1414 | // common PHI instruction in the successor. | |||
1415 | for (auto *I : Insts) { | |||
1416 | // These instructions may change or break semantics if moved. | |||
1417 | if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) || | |||
1418 | I->getType()->isTokenTy()) | |||
1419 | return false; | |||
1420 | ||||
1421 | // Conservatively return false if I is an inline-asm instruction. Sinking | |||
1422 | // and merging inline-asm instructions can potentially create arguments | |||
1423 | // that cannot satisfy the inline-asm constraints. | |||
1424 | if (const auto *C = dyn_cast<CallInst>(I)) | |||
1425 | if (C->isInlineAsm()) | |||
1426 | return false; | |||
1427 | ||||
1428 | // Everything must have only one use too, apart from stores which | |||
1429 | // have no uses. | |||
1430 | if (!isa<StoreInst>(I) && !I->hasOneUse()) | |||
1431 | return false; | |||
1432 | } | |||
1433 | ||||
1434 | const Instruction *I0 = Insts.front(); | |||
1435 | for (auto *I : Insts) | |||
1436 | if (!I->isSameOperationAs(I0)) | |||
1437 | return false; | |||
1438 | ||||
1439 | // All instructions in Insts are known to be the same opcode. If they aren't | |||
1440 | // stores, check the only user of each is a PHI or in the same block as the | |||
1441 | // instruction, because if a user is in the same block as an instruction | |||
1442 | // we're contemplating sinking, it must already be determined to be sinkable. | |||
1443 | if (!isa<StoreInst>(I0)) { | |||
1444 | auto *PNUse = dyn_cast<PHINode>(*I0->user_begin()); | |||
1445 | auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0); | |||
1446 | if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool { | |||
1447 | auto *U = cast<Instruction>(*I->user_begin()); | |||
1448 | return (PNUse && | |||
1449 | PNUse->getParent() == Succ && | |||
1450 | PNUse->getIncomingValueForBlock(I->getParent()) == I) || | |||
1451 | U->getParent() == I->getParent(); | |||
1452 | })) | |||
1453 | return false; | |||
1454 | } | |||
1455 | ||||
1456 | // Because SROA can't handle speculating stores of selects, try not | |||
1457 | // to sink loads or stores of allocas when we'd have to create a PHI for | |||
1458 | // the address operand. Also, because it is likely that loads or stores | |||
1459 | // of allocas will disappear when Mem2Reg/SROA is run, don't sink them. | |||
1460 | // This can cause code churn which can have unintended consequences down | |||
1461 | // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244. | |||
1462 | // FIXME: This is a workaround for a deficiency in SROA - see | |||
1463 | // https://llvm.org/bugs/show_bug.cgi?id=30188 | |||
1464 | if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) { | |||
1465 | return isa<AllocaInst>(I->getOperand(1)); | |||
1466 | })) | |||
1467 | return false; | |||
1468 | if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) { | |||
1469 | return isa<AllocaInst>(I->getOperand(0)); | |||
1470 | })) | |||
1471 | return false; | |||
1472 | ||||
1473 | for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) { | |||
1474 | if (I0->getOperand(OI)->getType()->isTokenTy()) | |||
1475 | // Don't touch any operand of token type. | |||
1476 | return false; | |||
1477 | ||||
1478 | auto SameAsI0 = [&I0, OI](const Instruction *I) { | |||
1479 | assert(I->getNumOperands() == I0->getNumOperands())(static_cast <bool> (I->getNumOperands() == I0->getNumOperands ()) ? void (0) : __assert_fail ("I->getNumOperands() == I0->getNumOperands()" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 1479, __extension__ __PRETTY_FUNCTION__)); | |||
1480 | return I->getOperand(OI) == I0->getOperand(OI); | |||
1481 | }; | |||
1482 | if (!all_of(Insts, SameAsI0)) { | |||
1483 | if (!canReplaceOperandWithVariable(I0, OI)) | |||
1484 | // We can't create a PHI from this GEP. | |||
1485 | return false; | |||
1486 | // Don't create indirect calls! The called value is the final operand. | |||
1487 | if ((isa<CallInst>(I0) || isa<InvokeInst>(I0)) && OI == OE - 1) { | |||
1488 | // FIXME: if the call was *already* indirect, we should do this. | |||
1489 | return false; | |||
1490 | } | |||
1491 | for (auto *I : Insts) | |||
1492 | PHIOperands[I].push_back(I->getOperand(OI)); | |||
1493 | } | |||
1494 | } | |||
1495 | return true; | |||
1496 | } | |||
1497 | ||||
1498 | // Assuming canSinkLastInstruction(Blocks) has returned true, sink the last | |||
1499 | // instruction of every block in Blocks to their common successor, commoning | |||
1500 | // into one instruction. | |||
1501 | static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) { | |||
1502 | auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0); | |||
1503 | ||||
1504 | // canSinkLastInstruction returning true guarantees that every block has at | |||
1505 | // least one non-terminator instruction. | |||
1506 | SmallVector<Instruction*,4> Insts; | |||
1507 | for (auto *BB : Blocks) { | |||
1508 | Instruction *I = BB->getTerminator(); | |||
1509 | do { | |||
1510 | I = I->getPrevNode(); | |||
1511 | } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front()); | |||
1512 | if (!isa<DbgInfoIntrinsic>(I)) | |||
1513 | Insts.push_back(I); | |||
1514 | } | |||
1515 | ||||
1516 | // The only checking we need to do now is that all users of all instructions | |||
1517 | // are the same PHI node. canSinkLastInstruction should have checked this but | |||
1518 | // it is slightly over-aggressive - it gets confused by commutative instructions | |||
1519 | // so double-check it here. | |||
1520 | Instruction *I0 = Insts.front(); | |||
1521 | if (!isa<StoreInst>(I0)) { | |||
1522 | auto *PNUse = dyn_cast<PHINode>(*I0->user_begin()); | |||
1523 | if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool { | |||
1524 | auto *U = cast<Instruction>(*I->user_begin()); | |||
1525 | return U == PNUse; | |||
1526 | })) | |||
1527 | return false; | |||
1528 | } | |||
1529 | ||||
1530 | // We don't need to do any more checking here; canSinkLastInstruction should | |||
1531 | // have done it all for us. | |||
1532 | SmallVector<Value*, 4> NewOperands; | |||
1533 | for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) { | |||
1534 | // This check is different to that in canSinkLastInstruction. There, we | |||
1535 | // cared about the global view once simplifycfg (and instcombine) have | |||
1536 | // completed - it takes into account PHIs that become trivially | |||
1537 | // simplifiable. However here we need a more local view; if an operand | |||
1538 | // differs we create a PHI and rely on instcombine to clean up the very | |||
1539 | // small mess we may make. | |||
1540 | bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) { | |||
1541 | return I->getOperand(O) != I0->getOperand(O); | |||
1542 | }); | |||
1543 | if (!NeedPHI) { | |||
1544 | NewOperands.push_back(I0->getOperand(O)); | |||
1545 | continue; | |||
1546 | } | |||
1547 | ||||
1548 | // Create a new PHI in the successor block and populate it. | |||
1549 | auto *Op = I0->getOperand(O); | |||
1550 | assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!")(static_cast <bool> (!Op->getType()->isTokenTy() && "Can't PHI tokens!") ? void (0) : __assert_fail ("!Op->getType()->isTokenTy() && \"Can't PHI tokens!\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 1550, __extension__ __PRETTY_FUNCTION__)); | |||
1551 | auto *PN = PHINode::Create(Op->getType(), Insts.size(), | |||
1552 | Op->getName() + ".sink", &BBEnd->front()); | |||
1553 | for (auto *I : Insts) | |||
1554 | PN->addIncoming(I->getOperand(O), I->getParent()); | |||
1555 | NewOperands.push_back(PN); | |||
1556 | } | |||
1557 | ||||
1558 | // Arbitrarily use I0 as the new "common" instruction; remap its operands | |||
1559 | // and move it to the start of the successor block. | |||
1560 | for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) | |||
1561 | I0->getOperandUse(O).set(NewOperands[O]); | |||
1562 | I0->moveBefore(&*BBEnd->getFirstInsertionPt()); | |||
1563 | ||||
1564 | // Update metadata and IR flags, and merge debug locations. | |||
1565 | for (auto *I : Insts) | |||
1566 | if (I != I0) { | |||
1567 | // The debug location for the "common" instruction is the merged locations | |||
1568 | // of all the commoned instructions. We start with the original location | |||
1569 | // of the "common" instruction and iteratively merge each location in the | |||
1570 | // loop below. | |||
1571 | // This is an N-way merge, which will be inefficient if I0 is a CallInst. | |||
1572 | // However, as N-way merge for CallInst is rare, so we use simplified API | |||
1573 | // instead of using complex API for N-way merge. | |||
1574 | I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc()); | |||
1575 | combineMetadataForCSE(I0, I); | |||
1576 | I0->andIRFlags(I); | |||
1577 | } | |||
1578 | ||||
1579 | if (!isa<StoreInst>(I0)) { | |||
1580 | // canSinkLastInstruction checked that all instructions were used by | |||
1581 | // one and only one PHI node. Find that now, RAUW it to our common | |||
1582 | // instruction and nuke it. | |||
1583 | assert(I0->hasOneUse())(static_cast <bool> (I0->hasOneUse()) ? void (0) : __assert_fail ("I0->hasOneUse()", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 1583, __extension__ __PRETTY_FUNCTION__)); | |||
1584 | auto *PN = cast<PHINode>(*I0->user_begin()); | |||
1585 | PN->replaceAllUsesWith(I0); | |||
1586 | PN->eraseFromParent(); | |||
1587 | } | |||
1588 | ||||
1589 | // Finally nuke all instructions apart from the common instruction. | |||
1590 | for (auto *I : Insts) | |||
1591 | if (I != I0) | |||
1592 | I->eraseFromParent(); | |||
1593 | ||||
1594 | return true; | |||
1595 | } | |||
1596 | ||||
1597 | namespace { | |||
1598 | ||||
1599 | // LockstepReverseIterator - Iterates through instructions | |||
1600 | // in a set of blocks in reverse order from the first non-terminator. | |||
1601 | // For example (assume all blocks have size n): | |||
1602 | // LockstepReverseIterator I([B1, B2, B3]); | |||
1603 | // *I-- = [B1[n], B2[n], B3[n]]; | |||
1604 | // *I-- = [B1[n-1], B2[n-1], B3[n-1]]; | |||
1605 | // *I-- = [B1[n-2], B2[n-2], B3[n-2]]; | |||
1606 | // ... | |||
1607 | class LockstepReverseIterator { | |||
1608 | ArrayRef<BasicBlock*> Blocks; | |||
1609 | SmallVector<Instruction*,4> Insts; | |||
1610 | bool Fail; | |||
1611 | ||||
1612 | public: | |||
1613 | LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) : Blocks(Blocks) { | |||
1614 | reset(); | |||
1615 | } | |||
1616 | ||||
1617 | void reset() { | |||
1618 | Fail = false; | |||
1619 | Insts.clear(); | |||
1620 | for (auto *BB : Blocks) { | |||
1621 | Instruction *Inst = BB->getTerminator(); | |||
1622 | for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);) | |||
1623 | Inst = Inst->getPrevNode(); | |||
1624 | if (!Inst) { | |||
1625 | // Block wasn't big enough. | |||
1626 | Fail = true; | |||
1627 | return; | |||
1628 | } | |||
1629 | Insts.push_back(Inst); | |||
1630 | } | |||
1631 | } | |||
1632 | ||||
1633 | bool isValid() const { | |||
1634 | return !Fail; | |||
1635 | } | |||
1636 | ||||
1637 | void operator--() { | |||
1638 | if (Fail) | |||
1639 | return; | |||
1640 | for (auto *&Inst : Insts) { | |||
1641 | for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);) | |||
1642 | Inst = Inst->getPrevNode(); | |||
1643 | // Already at beginning of block. | |||
1644 | if (!Inst) { | |||
1645 | Fail = true; | |||
1646 | return; | |||
1647 | } | |||
1648 | } | |||
1649 | } | |||
1650 | ||||
1651 | ArrayRef<Instruction*> operator * () const { | |||
1652 | return Insts; | |||
1653 | } | |||
1654 | }; | |||
1655 | ||||
1656 | } // end anonymous namespace | |||
1657 | ||||
1658 | /// Check whether BB's predecessors end with unconditional branches. If it is | |||
1659 | /// true, sink any common code from the predecessors to BB. | |||
1660 | /// We also allow one predecessor to end with conditional branch (but no more | |||
1661 | /// than one). | |||
1662 | static bool SinkCommonCodeFromPredecessors(BasicBlock *BB) { | |||
1663 | // We support two situations: | |||
1664 | // (1) all incoming arcs are unconditional | |||
1665 | // (2) one incoming arc is conditional | |||
1666 | // | |||
1667 | // (2) is very common in switch defaults and | |||
1668 | // else-if patterns; | |||
1669 | // | |||
1670 | // if (a) f(1); | |||
1671 | // else if (b) f(2); | |||
1672 | // | |||
1673 | // produces: | |||
1674 | // | |||
1675 | // [if] | |||
1676 | // / \ | |||
1677 | // [f(1)] [if] | |||
1678 | // | | \ | |||
1679 | // | | | | |||
1680 | // | [f(2)]| | |||
1681 | // \ | / | |||
1682 | // [ end ] | |||
1683 | // | |||
1684 | // [end] has two unconditional predecessor arcs and one conditional. The | |||
1685 | // conditional refers to the implicit empty 'else' arc. This conditional | |||
1686 | // arc can also be caused by an empty default block in a switch. | |||
1687 | // | |||
1688 | // In this case, we attempt to sink code from all *unconditional* arcs. | |||
1689 | // If we can sink instructions from these arcs (determined during the scan | |||
1690 | // phase below) we insert a common successor for all unconditional arcs and | |||
1691 | // connect that to [end], to enable sinking: | |||
1692 | // | |||
1693 | // [if] | |||
1694 | // / \ | |||
1695 | // [x(1)] [if] | |||
1696 | // | | \ | |||
1697 | // | | \ | |||
1698 | // | [x(2)] | | |||
1699 | // \ / | | |||
1700 | // [sink.split] | | |||
1701 | // \ / | |||
1702 | // [ end ] | |||
1703 | // | |||
1704 | SmallVector<BasicBlock*,4> UnconditionalPreds; | |||
1705 | Instruction *Cond = nullptr; | |||
1706 | for (auto *B : predecessors(BB)) { | |||
1707 | auto *T = B->getTerminator(); | |||
1708 | if (isa<BranchInst>(T) && cast<BranchInst>(T)->isUnconditional()) | |||
1709 | UnconditionalPreds.push_back(B); | |||
1710 | else if ((isa<BranchInst>(T) || isa<SwitchInst>(T)) && !Cond) | |||
1711 | Cond = T; | |||
1712 | else | |||
1713 | return false; | |||
1714 | } | |||
1715 | if (UnconditionalPreds.size() < 2) | |||
1716 | return false; | |||
1717 | ||||
1718 | bool Changed = false; | |||
1719 | // We take a two-step approach to tail sinking. First we scan from the end of | |||
1720 | // each block upwards in lockstep. If the n'th instruction from the end of each | |||
1721 | // block can be sunk, those instructions are added to ValuesToSink and we | |||
1722 | // carry on. If we can sink an instruction but need to PHI-merge some operands | |||
1723 | // (because they're not identical in each instruction) we add these to | |||
1724 | // PHIOperands. | |||
1725 | unsigned ScanIdx = 0; | |||
1726 | SmallPtrSet<Value*,4> InstructionsToSink; | |||
1727 | DenseMap<Instruction*, SmallVector<Value*,4>> PHIOperands; | |||
1728 | LockstepReverseIterator LRI(UnconditionalPreds); | |||
1729 | while (LRI.isValid() && | |||
1730 | canSinkInstructions(*LRI, PHIOperands)) { | |||
1731 | DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0] << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0] << "\n"; } } while (false); | |||
1732 | InstructionsToSink.insert((*LRI).begin(), (*LRI).end()); | |||
1733 | ++ScanIdx; | |||
1734 | --LRI; | |||
1735 | } | |||
1736 | ||||
1737 | auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) { | |||
1738 | unsigned NumPHIdValues = 0; | |||
1739 | for (auto *I : *LRI) | |||
1740 | for (auto *V : PHIOperands[I]) | |||
1741 | if (InstructionsToSink.count(V) == 0) | |||
1742 | ++NumPHIdValues; | |||
1743 | DEBUG(dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n"; } } while (false); | |||
1744 | unsigned NumPHIInsts = NumPHIdValues / UnconditionalPreds.size(); | |||
1745 | if ((NumPHIdValues % UnconditionalPreds.size()) != 0) | |||
1746 | NumPHIInsts++; | |||
1747 | ||||
1748 | return NumPHIInsts <= 1; | |||
1749 | }; | |||
1750 | ||||
1751 | if (ScanIdx > 0 && Cond) { | |||
1752 | // Check if we would actually sink anything first! This mutates the CFG and | |||
1753 | // adds an extra block. The goal in doing this is to allow instructions that | |||
1754 | // couldn't be sunk before to be sunk - obviously, speculatable instructions | |||
1755 | // (such as trunc, add) can be sunk and predicated already. So we check that | |||
1756 | // we're going to sink at least one non-speculatable instruction. | |||
1757 | LRI.reset(); | |||
1758 | unsigned Idx = 0; | |||
1759 | bool Profitable = false; | |||
1760 | while (ProfitableToSinkInstruction(LRI) && Idx < ScanIdx) { | |||
1761 | if (!isSafeToSpeculativelyExecute((*LRI)[0])) { | |||
1762 | Profitable = true; | |||
1763 | break; | |||
1764 | } | |||
1765 | --LRI; | |||
1766 | ++Idx; | |||
1767 | } | |||
1768 | if (!Profitable) | |||
1769 | return false; | |||
1770 | ||||
1771 | DEBUG(dbgs() << "SINK: Splitting edge\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SINK: Splitting edge\n"; } } while (false); | |||
1772 | // We have a conditional edge and we're going to sink some instructions. | |||
1773 | // Insert a new block postdominating all blocks we're going to sink from. | |||
1774 | if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split")) | |||
1775 | // Edges couldn't be split. | |||
1776 | return false; | |||
1777 | Changed = true; | |||
1778 | } | |||
1779 | ||||
1780 | // Now that we've analyzed all potential sinking candidates, perform the | |||
1781 | // actual sink. We iteratively sink the last non-terminator of the source | |||
1782 | // blocks into their common successor unless doing so would require too | |||
1783 | // many PHI instructions to be generated (currently only one PHI is allowed | |||
1784 | // per sunk instruction). | |||
1785 | // | |||
1786 | // We can use InstructionsToSink to discount values needing PHI-merging that will | |||
1787 | // actually be sunk in a later iteration. This allows us to be more | |||
1788 | // aggressive in what we sink. This does allow a false positive where we | |||
1789 | // sink presuming a later value will also be sunk, but stop half way through | |||
1790 | // and never actually sink it which means we produce more PHIs than intended. | |||
1791 | // This is unlikely in practice though. | |||
1792 | for (unsigned SinkIdx = 0; SinkIdx != ScanIdx; ++SinkIdx) { | |||
1793 | DEBUG(dbgs() << "SINK: Sink: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds [0]->getTerminator()->getPrevNode() << "\n"; } } while (false) | |||
1794 | << *UnconditionalPreds[0]->getTerminator()->getPrevNode()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds [0]->getTerminator()->getPrevNode() << "\n"; } } while (false) | |||
1795 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds [0]->getTerminator()->getPrevNode() << "\n"; } } while (false); | |||
1796 | ||||
1797 | // Because we've sunk every instruction in turn, the current instruction to | |||
1798 | // sink is always at index 0. | |||
1799 | LRI.reset(); | |||
1800 | if (!ProfitableToSinkInstruction(LRI)) { | |||
1801 | // Too many PHIs would be created. | |||
1802 | DEBUG(dbgs() << "SINK: stopping here, too many PHIs would be created!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SINK: stopping here, too many PHIs would be created!\n" ; } } while (false); | |||
1803 | break; | |||
1804 | } | |||
1805 | ||||
1806 | if (!sinkLastInstruction(UnconditionalPreds)) | |||
1807 | return Changed; | |||
1808 | NumSinkCommons++; | |||
1809 | Changed = true; | |||
1810 | } | |||
1811 | return Changed; | |||
1812 | } | |||
1813 | ||||
1814 | /// \brief Determine if we can hoist sink a sole store instruction out of a | |||
1815 | /// conditional block. | |||
1816 | /// | |||
1817 | /// We are looking for code like the following: | |||
1818 | /// BrBB: | |||
1819 | /// store i32 %add, i32* %arrayidx2 | |||
1820 | /// ... // No other stores or function calls (we could be calling a memory | |||
1821 | /// ... // function). | |||
1822 | /// %cmp = icmp ult %x, %y | |||
1823 | /// br i1 %cmp, label %EndBB, label %ThenBB | |||
1824 | /// ThenBB: | |||
1825 | /// store i32 %add5, i32* %arrayidx2 | |||
1826 | /// br label EndBB | |||
1827 | /// EndBB: | |||
1828 | /// ... | |||
1829 | /// We are going to transform this into: | |||
1830 | /// BrBB: | |||
1831 | /// store i32 %add, i32* %arrayidx2 | |||
1832 | /// ... // | |||
1833 | /// %cmp = icmp ult %x, %y | |||
1834 | /// %add.add5 = select i1 %cmp, i32 %add, %add5 | |||
1835 | /// store i32 %add.add5, i32* %arrayidx2 | |||
1836 | /// ... | |||
1837 | /// | |||
1838 | /// \return The pointer to the value of the previous store if the store can be | |||
1839 | /// hoisted into the predecessor block. 0 otherwise. | |||
1840 | static Value *isSafeToSpeculateStore(Instruction *I, BasicBlock *BrBB, | |||
1841 | BasicBlock *StoreBB, BasicBlock *EndBB) { | |||
1842 | StoreInst *StoreToHoist = dyn_cast<StoreInst>(I); | |||
1843 | if (!StoreToHoist) | |||
1844 | return nullptr; | |||
1845 | ||||
1846 | // Volatile or atomic. | |||
1847 | if (!StoreToHoist->isSimple()) | |||
1848 | return nullptr; | |||
1849 | ||||
1850 | Value *StorePtr = StoreToHoist->getPointerOperand(); | |||
1851 | ||||
1852 | // Look for a store to the same pointer in BrBB. | |||
1853 | unsigned MaxNumInstToLookAt = 9; | |||
1854 | for (Instruction &CurI : reverse(*BrBB)) { | |||
1855 | if (!MaxNumInstToLookAt) | |||
1856 | break; | |||
1857 | // Skip debug info. | |||
1858 | if (isa<DbgInfoIntrinsic>(CurI)) | |||
1859 | continue; | |||
1860 | --MaxNumInstToLookAt; | |||
1861 | ||||
1862 | // Could be calling an instruction that affects memory like free(). | |||
1863 | if (CurI.mayHaveSideEffects() && !isa<StoreInst>(CurI)) | |||
1864 | return nullptr; | |||
1865 | ||||
1866 | if (auto *SI = dyn_cast<StoreInst>(&CurI)) { | |||
1867 | // Found the previous store make sure it stores to the same location. | |||
1868 | if (SI->getPointerOperand() == StorePtr) | |||
1869 | // Found the previous store, return its value operand. | |||
1870 | return SI->getValueOperand(); | |||
1871 | return nullptr; // Unknown store. | |||
1872 | } | |||
1873 | } | |||
1874 | ||||
1875 | return nullptr; | |||
1876 | } | |||
1877 | ||||
1878 | /// \brief Speculate a conditional basic block flattening the CFG. | |||
1879 | /// | |||
1880 | /// Note that this is a very risky transform currently. Speculating | |||
1881 | /// instructions like this is most often not desirable. Instead, there is an MI | |||
1882 | /// pass which can do it with full awareness of the resource constraints. | |||
1883 | /// However, some cases are "obvious" and we should do directly. An example of | |||
1884 | /// this is speculating a single, reasonably cheap instruction. | |||
1885 | /// | |||
1886 | /// There is only one distinct advantage to flattening the CFG at the IR level: | |||
1887 | /// it makes very common but simplistic optimizations such as are common in | |||
1888 | /// instcombine and the DAG combiner more powerful by removing CFG edges and | |||
1889 | /// modeling their effects with easier to reason about SSA value graphs. | |||
1890 | /// | |||
1891 | /// | |||
1892 | /// An illustration of this transform is turning this IR: | |||
1893 | /// \code | |||
1894 | /// BB: | |||
1895 | /// %cmp = icmp ult %x, %y | |||
1896 | /// br i1 %cmp, label %EndBB, label %ThenBB | |||
1897 | /// ThenBB: | |||
1898 | /// %sub = sub %x, %y | |||
1899 | /// br label BB2 | |||
1900 | /// EndBB: | |||
1901 | /// %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ] | |||
1902 | /// ... | |||
1903 | /// \endcode | |||
1904 | /// | |||
1905 | /// Into this IR: | |||
1906 | /// \code | |||
1907 | /// BB: | |||
1908 | /// %cmp = icmp ult %x, %y | |||
1909 | /// %sub = sub %x, %y | |||
1910 | /// %cond = select i1 %cmp, 0, %sub | |||
1911 | /// ... | |||
1912 | /// \endcode | |||
1913 | /// | |||
1914 | /// \returns true if the conditional block is removed. | |||
1915 | static bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB, | |||
1916 | const TargetTransformInfo &TTI) { | |||
1917 | // Be conservative for now. FP select instruction can often be expensive. | |||
1918 | Value *BrCond = BI->getCondition(); | |||
1919 | if (isa<FCmpInst>(BrCond)) | |||
1920 | return false; | |||
1921 | ||||
1922 | BasicBlock *BB = BI->getParent(); | |||
1923 | BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0); | |||
1924 | ||||
1925 | // If ThenBB is actually on the false edge of the conditional branch, remember | |||
1926 | // to swap the select operands later. | |||
1927 | bool Invert = false; | |||
1928 | if (ThenBB != BI->getSuccessor(0)) { | |||
1929 | assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?")(static_cast <bool> (ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?") ? void (0) : __assert_fail ("ThenBB == BI->getSuccessor(1) && \"No edge from 'if' block?\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 1929, __extension__ __PRETTY_FUNCTION__)); | |||
1930 | Invert = true; | |||
1931 | } | |||
1932 | assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block")(static_cast <bool> (EndBB == BI->getSuccessor(!Invert ) && "No edge from to end block") ? void (0) : __assert_fail ("EndBB == BI->getSuccessor(!Invert) && \"No edge from to end block\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 1932, __extension__ __PRETTY_FUNCTION__)); | |||
1933 | ||||
1934 | // Keep a count of how many times instructions are used within CondBB when | |||
1935 | // they are candidates for sinking into CondBB. Specifically: | |||
1936 | // - They are defined in BB, and | |||
1937 | // - They have no side effects, and | |||
1938 | // - All of their uses are in CondBB. | |||
1939 | SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts; | |||
1940 | ||||
1941 | SmallVector<Instruction *, 4> SpeculatedDbgIntrinsics; | |||
1942 | ||||
1943 | unsigned SpeculationCost = 0; | |||
1944 | Value *SpeculatedStoreValue = nullptr; | |||
1945 | StoreInst *SpeculatedStore = nullptr; | |||
1946 | for (BasicBlock::iterator BBI = ThenBB->begin(), | |||
1947 | BBE = std::prev(ThenBB->end()); | |||
1948 | BBI != BBE; ++BBI) { | |||
1949 | Instruction *I = &*BBI; | |||
1950 | // Skip debug info. | |||
1951 | if (isa<DbgInfoIntrinsic>(I)) { | |||
1952 | SpeculatedDbgIntrinsics.push_back(I); | |||
1953 | continue; | |||
1954 | } | |||
1955 | ||||
1956 | // Only speculatively execute a single instruction (not counting the | |||
1957 | // terminator) for now. | |||
1958 | ++SpeculationCost; | |||
1959 | if (SpeculationCost > 1) | |||
1960 | return false; | |||
1961 | ||||
1962 | // Don't hoist the instruction if it's unsafe or expensive. | |||
1963 | if (!isSafeToSpeculativelyExecute(I) && | |||
1964 | !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore( | |||
1965 | I, BB, ThenBB, EndBB)))) | |||
1966 | return false; | |||
1967 | if (!SpeculatedStoreValue && | |||
1968 | ComputeSpeculationCost(I, TTI) > | |||
1969 | PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic) | |||
1970 | return false; | |||
1971 | ||||
1972 | // Store the store speculation candidate. | |||
1973 | if (SpeculatedStoreValue) | |||
1974 | SpeculatedStore = cast<StoreInst>(I); | |||
1975 | ||||
1976 | // Do not hoist the instruction if any of its operands are defined but not | |||
1977 | // used in BB. The transformation will prevent the operand from | |||
1978 | // being sunk into the use block. | |||
1979 | for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i) { | |||
1980 | Instruction *OpI = dyn_cast<Instruction>(*i); | |||
1981 | if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects()) | |||
1982 | continue; // Not a candidate for sinking. | |||
1983 | ||||
1984 | ++SinkCandidateUseCounts[OpI]; | |||
1985 | } | |||
1986 | } | |||
1987 | ||||
1988 | // Consider any sink candidates which are only used in CondBB as costs for | |||
1989 | // speculation. Note, while we iterate over a DenseMap here, we are summing | |||
1990 | // and so iteration order isn't significant. | |||
1991 | for (SmallDenseMap<Instruction *, unsigned, 4>::iterator | |||
1992 | I = SinkCandidateUseCounts.begin(), | |||
1993 | E = SinkCandidateUseCounts.end(); | |||
1994 | I != E; ++I) | |||
1995 | if (I->first->getNumUses() == I->second) { | |||
1996 | ++SpeculationCost; | |||
1997 | if (SpeculationCost > 1) | |||
1998 | return false; | |||
1999 | } | |||
2000 | ||||
2001 | // Check that the PHI nodes can be converted to selects. | |||
2002 | bool HaveRewritablePHIs = false; | |||
2003 | for (PHINode &PN : EndBB->phis()) { | |||
2004 | Value *OrigV = PN.getIncomingValueForBlock(BB); | |||
2005 | Value *ThenV = PN.getIncomingValueForBlock(ThenBB); | |||
2006 | ||||
2007 | // FIXME: Try to remove some of the duplication with HoistThenElseCodeToIf. | |||
2008 | // Skip PHIs which are trivial. | |||
2009 | if (ThenV == OrigV) | |||
2010 | continue; | |||
2011 | ||||
2012 | // Don't convert to selects if we could remove undefined behavior instead. | |||
2013 | if (passingValueIsAlwaysUndefined(OrigV, &PN) || | |||
2014 | passingValueIsAlwaysUndefined(ThenV, &PN)) | |||
2015 | return false; | |||
2016 | ||||
2017 | HaveRewritablePHIs = true; | |||
2018 | ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV); | |||
2019 | ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV); | |||
2020 | if (!OrigCE && !ThenCE) | |||
2021 | continue; // Known safe and cheap. | |||
2022 | ||||
2023 | if ((ThenCE && !isSafeToSpeculativelyExecute(ThenCE)) || | |||
2024 | (OrigCE && !isSafeToSpeculativelyExecute(OrigCE))) | |||
2025 | return false; | |||
2026 | unsigned OrigCost = OrigCE ? ComputeSpeculationCost(OrigCE, TTI) : 0; | |||
2027 | unsigned ThenCost = ThenCE ? ComputeSpeculationCost(ThenCE, TTI) : 0; | |||
2028 | unsigned MaxCost = | |||
2029 | 2 * PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic; | |||
2030 | if (OrigCost + ThenCost > MaxCost) | |||
2031 | return false; | |||
2032 | ||||
2033 | // Account for the cost of an unfolded ConstantExpr which could end up | |||
2034 | // getting expanded into Instructions. | |||
2035 | // FIXME: This doesn't account for how many operations are combined in the | |||
2036 | // constant expression. | |||
2037 | ++SpeculationCost; | |||
2038 | if (SpeculationCost > 1) | |||
2039 | return false; | |||
2040 | } | |||
2041 | ||||
2042 | // If there are no PHIs to process, bail early. This helps ensure idempotence | |||
2043 | // as well. | |||
2044 | if (!HaveRewritablePHIs && !(HoistCondStores && SpeculatedStoreValue)) | |||
2045 | return false; | |||
2046 | ||||
2047 | // If we get here, we can hoist the instruction and if-convert. | |||
2048 | DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";; } } while (false); | |||
2049 | ||||
2050 | // Insert a select of the value of the speculated store. | |||
2051 | if (SpeculatedStoreValue) { | |||
2052 | IRBuilder<NoFolder> Builder(BI); | |||
2053 | Value *TrueV = SpeculatedStore->getValueOperand(); | |||
2054 | Value *FalseV = SpeculatedStoreValue; | |||
2055 | if (Invert) | |||
2056 | std::swap(TrueV, FalseV); | |||
2057 | Value *S = Builder.CreateSelect( | |||
2058 | BrCond, TrueV, FalseV, "spec.store.select", BI); | |||
2059 | SpeculatedStore->setOperand(0, S); | |||
2060 | SpeculatedStore->applyMergedLocation(BI->getDebugLoc(), | |||
2061 | SpeculatedStore->getDebugLoc()); | |||
2062 | } | |||
2063 | ||||
2064 | // Metadata can be dependent on the condition we are hoisting above. | |||
2065 | // Conservatively strip all metadata on the instruction. | |||
2066 | for (auto &I : *ThenBB) | |||
2067 | I.dropUnknownNonDebugMetadata(); | |||
2068 | ||||
2069 | // Hoist the instructions. | |||
2070 | BB->getInstList().splice(BI->getIterator(), ThenBB->getInstList(), | |||
2071 | ThenBB->begin(), std::prev(ThenBB->end())); | |||
2072 | ||||
2073 | // Insert selects and rewrite the PHI operands. | |||
2074 | IRBuilder<NoFolder> Builder(BI); | |||
2075 | for (PHINode &PN : EndBB->phis()) { | |||
2076 | unsigned OrigI = PN.getBasicBlockIndex(BB); | |||
2077 | unsigned ThenI = PN.getBasicBlockIndex(ThenBB); | |||
2078 | Value *OrigV = PN.getIncomingValue(OrigI); | |||
2079 | Value *ThenV = PN.getIncomingValue(ThenI); | |||
2080 | ||||
2081 | // Skip PHIs which are trivial. | |||
2082 | if (OrigV == ThenV) | |||
2083 | continue; | |||
2084 | ||||
2085 | // Create a select whose true value is the speculatively executed value and | |||
2086 | // false value is the preexisting value. Swap them if the branch | |||
2087 | // destinations were inverted. | |||
2088 | Value *TrueV = ThenV, *FalseV = OrigV; | |||
2089 | if (Invert) | |||
2090 | std::swap(TrueV, FalseV); | |||
2091 | Value *V = Builder.CreateSelect( | |||
2092 | BrCond, TrueV, FalseV, "spec.select", BI); | |||
2093 | PN.setIncomingValue(OrigI, V); | |||
2094 | PN.setIncomingValue(ThenI, V); | |||
2095 | } | |||
2096 | ||||
2097 | // Remove speculated dbg intrinsics. | |||
2098 | // FIXME: Is it possible to do this in a more elegant way? Moving/merging the | |||
2099 | // dbg value for the different flows and inserting it after the select. | |||
2100 | for (Instruction *I : SpeculatedDbgIntrinsics) | |||
2101 | I->eraseFromParent(); | |||
2102 | ||||
2103 | ++NumSpeculations; | |||
2104 | return true; | |||
2105 | } | |||
2106 | ||||
2107 | /// Return true if we can thread a branch across this block. | |||
2108 | static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) { | |||
2109 | BranchInst *BI = cast<BranchInst>(BB->getTerminator()); | |||
2110 | unsigned Size = 0; | |||
2111 | ||||
2112 | for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) { | |||
2113 | if (isa<DbgInfoIntrinsic>(BBI)) | |||
2114 | continue; | |||
2115 | if (Size > 10) | |||
2116 | return false; // Don't clone large BB's. | |||
2117 | ++Size; | |||
2118 | ||||
2119 | // We can only support instructions that do not define values that are | |||
2120 | // live outside of the current basic block. | |||
2121 | for (User *U : BBI->users()) { | |||
2122 | Instruction *UI = cast<Instruction>(U); | |||
2123 | if (UI->getParent() != BB || isa<PHINode>(UI)) | |||
2124 | return false; | |||
2125 | } | |||
2126 | ||||
2127 | // Looks ok, continue checking. | |||
2128 | } | |||
2129 | ||||
2130 | return true; | |||
2131 | } | |||
2132 | ||||
2133 | /// If we have a conditional branch on a PHI node value that is defined in the | |||
2134 | /// same block as the branch and if any PHI entries are constants, thread edges | |||
2135 | /// corresponding to that entry to be branches to their ultimate destination. | |||
2136 | static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout &DL, | |||
2137 | AssumptionCache *AC) { | |||
2138 | BasicBlock *BB = BI->getParent(); | |||
2139 | PHINode *PN = dyn_cast<PHINode>(BI->getCondition()); | |||
2140 | // NOTE: we currently cannot transform this case if the PHI node is used | |||
2141 | // outside of the block. | |||
2142 | if (!PN || PN->getParent() != BB || !PN->hasOneUse()) | |||
2143 | return false; | |||
2144 | ||||
2145 | // Degenerate case of a single entry PHI. | |||
2146 | if (PN->getNumIncomingValues() == 1) { | |||
2147 | FoldSingleEntryPHINodes(PN->getParent()); | |||
2148 | return true; | |||
2149 | } | |||
2150 | ||||
2151 | // Now we know that this block has multiple preds and two succs. | |||
2152 | if (!BlockIsSimpleEnoughToThreadThrough(BB)) | |||
2153 | return false; | |||
2154 | ||||
2155 | // Can't fold blocks that contain noduplicate or convergent calls. | |||
2156 | if (any_of(*BB, [](const Instruction &I) { | |||
2157 | const CallInst *CI = dyn_cast<CallInst>(&I); | |||
2158 | return CI && (CI->cannotDuplicate() || CI->isConvergent()); | |||
2159 | })) | |||
2160 | return false; | |||
2161 | ||||
2162 | // Okay, this is a simple enough basic block. See if any phi values are | |||
2163 | // constants. | |||
2164 | for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { | |||
2165 | ConstantInt *CB = dyn_cast<ConstantInt>(PN->getIncomingValue(i)); | |||
2166 | if (!CB || !CB->getType()->isIntegerTy(1)) | |||
2167 | continue; | |||
2168 | ||||
2169 | // Okay, we now know that all edges from PredBB should be revectored to | |||
2170 | // branch to RealDest. | |||
2171 | BasicBlock *PredBB = PN->getIncomingBlock(i); | |||
2172 | BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue()); | |||
2173 | ||||
2174 | if (RealDest == BB) | |||
2175 | continue; // Skip self loops. | |||
2176 | // Skip if the predecessor's terminator is an indirect branch. | |||
2177 | if (isa<IndirectBrInst>(PredBB->getTerminator())) | |||
2178 | continue; | |||
2179 | ||||
2180 | // The dest block might have PHI nodes, other predecessors and other | |||
2181 | // difficult cases. Instead of being smart about this, just insert a new | |||
2182 | // block that jumps to the destination block, effectively splitting | |||
2183 | // the edge we are about to create. | |||
2184 | BasicBlock *EdgeBB = | |||
2185 | BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge", | |||
2186 | RealDest->getParent(), RealDest); | |||
2187 | BranchInst::Create(RealDest, EdgeBB); | |||
2188 | ||||
2189 | // Update PHI nodes. | |||
2190 | AddPredecessorToBlock(RealDest, EdgeBB, BB); | |||
2191 | ||||
2192 | // BB may have instructions that are being threaded over. Clone these | |||
2193 | // instructions into EdgeBB. We know that there will be no uses of the | |||
2194 | // cloned instructions outside of EdgeBB. | |||
2195 | BasicBlock::iterator InsertPt = EdgeBB->begin(); | |||
2196 | DenseMap<Value *, Value *> TranslateMap; // Track translated values. | |||
2197 | for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) { | |||
2198 | if (PHINode *PN = dyn_cast<PHINode>(BBI)) { | |||
2199 | TranslateMap[PN] = PN->getIncomingValueForBlock(PredBB); | |||
2200 | continue; | |||
2201 | } | |||
2202 | // Clone the instruction. | |||
2203 | Instruction *N = BBI->clone(); | |||
2204 | if (BBI->hasName()) | |||
2205 | N->setName(BBI->getName() + ".c"); | |||
2206 | ||||
2207 | // Update operands due to translation. | |||
2208 | for (User::op_iterator i = N->op_begin(), e = N->op_end(); i != e; ++i) { | |||
2209 | DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(*i); | |||
2210 | if (PI != TranslateMap.end()) | |||
2211 | *i = PI->second; | |||
2212 | } | |||
2213 | ||||
2214 | // Check for trivial simplification. | |||
2215 | if (Value *V = SimplifyInstruction(N, {DL, nullptr, nullptr, AC})) { | |||
2216 | if (!BBI->use_empty()) | |||
2217 | TranslateMap[&*BBI] = V; | |||
2218 | if (!N->mayHaveSideEffects()) { | |||
2219 | N->deleteValue(); // Instruction folded away, don't need actual inst | |||
2220 | N = nullptr; | |||
2221 | } | |||
2222 | } else { | |||
2223 | if (!BBI->use_empty()) | |||
2224 | TranslateMap[&*BBI] = N; | |||
2225 | } | |||
2226 | // Insert the new instruction into its new home. | |||
2227 | if (N) | |||
2228 | EdgeBB->getInstList().insert(InsertPt, N); | |||
2229 | ||||
2230 | // Register the new instruction with the assumption cache if necessary. | |||
2231 | if (auto *II = dyn_cast_or_null<IntrinsicInst>(N)) | |||
2232 | if (II->getIntrinsicID() == Intrinsic::assume) | |||
2233 | AC->registerAssumption(II); | |||
2234 | } | |||
2235 | ||||
2236 | // Loop over all of the edges from PredBB to BB, changing them to branch | |||
2237 | // to EdgeBB instead. | |||
2238 | TerminatorInst *PredBBTI = PredBB->getTerminator(); | |||
2239 | for (unsigned i = 0, e = PredBBTI->getNumSuccessors(); i != e; ++i) | |||
2240 | if (PredBBTI->getSuccessor(i) == BB) { | |||
2241 | BB->removePredecessor(PredBB); | |||
2242 | PredBBTI->setSuccessor(i, EdgeBB); | |||
2243 | } | |||
2244 | ||||
2245 | // Recurse, simplifying any other constants. | |||
2246 | return FoldCondBranchOnPHI(BI, DL, AC) | true; | |||
2247 | } | |||
2248 | ||||
2249 | return false; | |||
2250 | } | |||
2251 | ||||
2252 | /// Given a BB that starts with the specified two-entry PHI node, | |||
2253 | /// see if we can eliminate it. | |||
2254 | static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI, | |||
2255 | const DataLayout &DL) { | |||
2256 | // Ok, this is a two entry PHI node. Check to see if this is a simple "if | |||
2257 | // statement", which has a very simple dominance structure. Basically, we | |||
2258 | // are trying to find the condition that is being branched on, which | |||
2259 | // subsequently causes this merge to happen. We really want control | |||
2260 | // dependence information for this check, but simplifycfg can't keep it up | |||
2261 | // to date, and this catches most of the cases we care about anyway. | |||
2262 | BasicBlock *BB = PN->getParent(); | |||
2263 | BasicBlock *IfTrue, *IfFalse; | |||
2264 | Value *IfCond = GetIfCondition(BB, IfTrue, IfFalse); | |||
2265 | if (!IfCond || | |||
2266 | // Don't bother if the branch will be constant folded trivially. | |||
2267 | isa<ConstantInt>(IfCond)) | |||
2268 | return false; | |||
2269 | ||||
2270 | // Okay, we found that we can merge this two-entry phi node into a select. | |||
2271 | // Doing so would require us to fold *all* two entry phi nodes in this block. | |||
2272 | // At some point this becomes non-profitable (particularly if the target | |||
2273 | // doesn't support cmov's). Only do this transformation if there are two or | |||
2274 | // fewer PHI nodes in this block. | |||
2275 | unsigned NumPhis = 0; | |||
2276 | for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I) | |||
2277 | if (NumPhis > 2) | |||
2278 | return false; | |||
2279 | ||||
2280 | // Loop over the PHI's seeing if we can promote them all to select | |||
2281 | // instructions. While we are at it, keep track of the instructions | |||
2282 | // that need to be moved to the dominating block. | |||
2283 | SmallPtrSet<Instruction *, 4> AggressiveInsts; | |||
2284 | unsigned MaxCostVal0 = PHINodeFoldingThreshold, | |||
2285 | MaxCostVal1 = PHINodeFoldingThreshold; | |||
2286 | MaxCostVal0 *= TargetTransformInfo::TCC_Basic; | |||
2287 | MaxCostVal1 *= TargetTransformInfo::TCC_Basic; | |||
2288 | ||||
2289 | for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) { | |||
2290 | PHINode *PN = cast<PHINode>(II++); | |||
2291 | if (Value *V = SimplifyInstruction(PN, {DL, PN})) { | |||
2292 | PN->replaceAllUsesWith(V); | |||
2293 | PN->eraseFromParent(); | |||
2294 | continue; | |||
2295 | } | |||
2296 | ||||
2297 | if (!DominatesMergePoint(PN->getIncomingValue(0), BB, &AggressiveInsts, | |||
2298 | MaxCostVal0, TTI) || | |||
2299 | !DominatesMergePoint(PN->getIncomingValue(1), BB, &AggressiveInsts, | |||
2300 | MaxCostVal1, TTI)) | |||
2301 | return false; | |||
2302 | } | |||
2303 | ||||
2304 | // If we folded the first phi, PN dangles at this point. Refresh it. If | |||
2305 | // we ran out of PHIs then we simplified them all. | |||
2306 | PN = dyn_cast<PHINode>(BB->begin()); | |||
2307 | if (!PN) | |||
2308 | return true; | |||
2309 | ||||
2310 | // Don't fold i1 branches on PHIs which contain binary operators. These can | |||
2311 | // often be turned into switches and other things. | |||
2312 | if (PN->getType()->isIntegerTy(1) && | |||
2313 | (isa<BinaryOperator>(PN->getIncomingValue(0)) || | |||
2314 | isa<BinaryOperator>(PN->getIncomingValue(1)) || | |||
2315 | isa<BinaryOperator>(IfCond))) | |||
2316 | return false; | |||
2317 | ||||
2318 | // If all PHI nodes are promotable, check to make sure that all instructions | |||
2319 | // in the predecessor blocks can be promoted as well. If not, we won't be able | |||
2320 | // to get rid of the control flow, so it's not worth promoting to select | |||
2321 | // instructions. | |||
2322 | BasicBlock *DomBlock = nullptr; | |||
2323 | BasicBlock *IfBlock1 = PN->getIncomingBlock(0); | |||
2324 | BasicBlock *IfBlock2 = PN->getIncomingBlock(1); | |||
2325 | if (cast<BranchInst>(IfBlock1->getTerminator())->isConditional()) { | |||
2326 | IfBlock1 = nullptr; | |||
2327 | } else { | |||
2328 | DomBlock = *pred_begin(IfBlock1); | |||
2329 | for (BasicBlock::iterator I = IfBlock1->begin(); !isa<TerminatorInst>(I); | |||
2330 | ++I) | |||
2331 | if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I)) { | |||
2332 | // This is not an aggressive instruction that we can promote. | |||
2333 | // Because of this, we won't be able to get rid of the control flow, so | |||
2334 | // the xform is not worth it. | |||
2335 | return false; | |||
2336 | } | |||
2337 | } | |||
2338 | ||||
2339 | if (cast<BranchInst>(IfBlock2->getTerminator())->isConditional()) { | |||
2340 | IfBlock2 = nullptr; | |||
2341 | } else { | |||
2342 | DomBlock = *pred_begin(IfBlock2); | |||
2343 | for (BasicBlock::iterator I = IfBlock2->begin(); !isa<TerminatorInst>(I); | |||
2344 | ++I) | |||
2345 | if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I)) { | |||
2346 | // This is not an aggressive instruction that we can promote. | |||
2347 | // Because of this, we won't be able to get rid of the control flow, so | |||
2348 | // the xform is not worth it. | |||
2349 | return false; | |||
2350 | } | |||
2351 | } | |||
2352 | ||||
2353 | DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfCond << " T: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " << *IfCond << " T: " << IfTrue->getName() << " F: " << IfFalse->getName() << "\n"; } } while (false) | |||
2354 | << IfTrue->getName() << " F: " << IfFalse->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " << *IfCond << " T: " << IfTrue->getName() << " F: " << IfFalse->getName() << "\n"; } } while (false); | |||
2355 | ||||
2356 | // If we can still promote the PHI nodes after this gauntlet of tests, | |||
2357 | // do all of the PHI's now. | |||
2358 | Instruction *InsertPt = DomBlock->getTerminator(); | |||
| ||||
2359 | IRBuilder<NoFolder> Builder(InsertPt); | |||
2360 | ||||
2361 | // Move all 'aggressive' instructions, which are defined in the | |||
2362 | // conditional parts of the if's up to the dominating block. | |||
2363 | if (IfBlock1) { | |||
2364 | for (auto &I : *IfBlock1) | |||
2365 | I.dropUnknownNonDebugMetadata(); | |||
2366 | DomBlock->getInstList().splice(InsertPt->getIterator(), | |||
2367 | IfBlock1->getInstList(), IfBlock1->begin(), | |||
2368 | IfBlock1->getTerminator()->getIterator()); | |||
2369 | } | |||
2370 | if (IfBlock2) { | |||
2371 | for (auto &I : *IfBlock2) | |||
2372 | I.dropUnknownNonDebugMetadata(); | |||
2373 | DomBlock->getInstList().splice(InsertPt->getIterator(), | |||
2374 | IfBlock2->getInstList(), IfBlock2->begin(), | |||
2375 | IfBlock2->getTerminator()->getIterator()); | |||
2376 | } | |||
2377 | ||||
2378 | while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) { | |||
2379 | // Change the PHI node into a select instruction. | |||
2380 | Value *TrueVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfFalse); | |||
2381 | Value *FalseVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfTrue); | |||
2382 | ||||
2383 | Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", InsertPt); | |||
2384 | PN->replaceAllUsesWith(Sel); | |||
2385 | Sel->takeName(PN); | |||
2386 | PN->eraseFromParent(); | |||
2387 | } | |||
2388 | ||||
2389 | // At this point, IfBlock1 and IfBlock2 are both empty, so our if statement | |||
2390 | // has been flattened. Change DomBlock to jump directly to our new block to | |||
2391 | // avoid other simplifycfg's kicking in on the diamond. | |||
2392 | TerminatorInst *OldTI = DomBlock->getTerminator(); | |||
2393 | Builder.SetInsertPoint(OldTI); | |||
2394 | Builder.CreateBr(BB); | |||
2395 | OldTI->eraseFromParent(); | |||
2396 | return true; | |||
2397 | } | |||
2398 | ||||
2399 | /// If we found a conditional branch that goes to two returning blocks, | |||
2400 | /// try to merge them together into one return, | |||
2401 | /// introducing a select if the return values disagree. | |||
2402 | static bool SimplifyCondBranchToTwoReturns(BranchInst *BI, | |||
2403 | IRBuilder<> &Builder) { | |||
2404 | assert(BI->isConditional() && "Must be a conditional branch")(static_cast <bool> (BI->isConditional() && "Must be a conditional branch" ) ? void (0) : __assert_fail ("BI->isConditional() && \"Must be a conditional branch\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 2404, __extension__ __PRETTY_FUNCTION__)); | |||
2405 | BasicBlock *TrueSucc = BI->getSuccessor(0); | |||
2406 | BasicBlock *FalseSucc = BI->getSuccessor(1); | |||
2407 | ReturnInst *TrueRet = cast<ReturnInst>(TrueSucc->getTerminator()); | |||
2408 | ReturnInst *FalseRet = cast<ReturnInst>(FalseSucc->getTerminator()); | |||
2409 | ||||
2410 | // Check to ensure both blocks are empty (just a return) or optionally empty | |||
2411 | // with PHI nodes. If there are other instructions, merging would cause extra | |||
2412 | // computation on one path or the other. | |||
2413 | if (!TrueSucc->getFirstNonPHIOrDbg()->isTerminator()) | |||
2414 | return false; | |||
2415 | if (!FalseSucc->getFirstNonPHIOrDbg()->isTerminator()) | |||
2416 | return false; | |||
2417 | ||||
2418 | Builder.SetInsertPoint(BI); | |||
2419 | // Okay, we found a branch that is going to two return nodes. If | |||
2420 | // there is no return value for this function, just change the | |||
2421 | // branch into a return. | |||
2422 | if (FalseRet->getNumOperands() == 0) { | |||
2423 | TrueSucc->removePredecessor(BI->getParent()); | |||
2424 | FalseSucc->removePredecessor(BI->getParent()); | |||
2425 | Builder.CreateRetVoid(); | |||
2426 | EraseTerminatorInstAndDCECond(BI); | |||
2427 | return true; | |||
2428 | } | |||
2429 | ||||
2430 | // Otherwise, figure out what the true and false return values are | |||
2431 | // so we can insert a new select instruction. | |||
2432 | Value *TrueValue = TrueRet->getReturnValue(); | |||
2433 | Value *FalseValue = FalseRet->getReturnValue(); | |||
2434 | ||||
2435 | // Unwrap any PHI nodes in the return blocks. | |||
2436 | if (PHINode *TVPN = dyn_cast_or_null<PHINode>(TrueValue)) | |||
2437 | if (TVPN->getParent() == TrueSucc) | |||
2438 | TrueValue = TVPN->getIncomingValueForBlock(BI->getParent()); | |||
2439 | if (PHINode *FVPN = dyn_cast_or_null<PHINode>(FalseValue)) | |||
2440 | if (FVPN->getParent() == FalseSucc) | |||
2441 | FalseValue = FVPN->getIncomingValueForBlock(BI->getParent()); | |||
2442 | ||||
2443 | // In order for this transformation to be safe, we must be able to | |||
2444 | // unconditionally execute both operands to the return. This is | |||
2445 | // normally the case, but we could have a potentially-trapping | |||
2446 | // constant expression that prevents this transformation from being | |||
2447 | // safe. | |||
2448 | if (ConstantExpr *TCV = dyn_cast_or_null<ConstantExpr>(TrueValue)) | |||
2449 | if (TCV->canTrap()) | |||
2450 | return false; | |||
2451 | if (ConstantExpr *FCV = dyn_cast_or_null<ConstantExpr>(FalseValue)) | |||
2452 | if (FCV->canTrap()) | |||
2453 | return false; | |||
2454 | ||||
2455 | // Okay, we collected all the mapped values and checked them for sanity, and | |||
2456 | // defined to really do this transformation. First, update the CFG. | |||
2457 | TrueSucc->removePredecessor(BI->getParent()); | |||
2458 | FalseSucc->removePredecessor(BI->getParent()); | |||
2459 | ||||
2460 | // Insert select instructions where needed. | |||
2461 | Value *BrCond = BI->getCondition(); | |||
2462 | if (TrueValue) { | |||
2463 | // Insert a select if the results differ. | |||
2464 | if (TrueValue == FalseValue || isa<UndefValue>(FalseValue)) { | |||
2465 | } else if (isa<UndefValue>(TrueValue)) { | |||
2466 | TrueValue = FalseValue; | |||
2467 | } else { | |||
2468 | TrueValue = | |||
2469 | Builder.CreateSelect(BrCond, TrueValue, FalseValue, "retval", BI); | |||
2470 | } | |||
2471 | } | |||
2472 | ||||
2473 | Value *RI = | |||
2474 | !TrueValue ? Builder.CreateRetVoid() : Builder.CreateRet(TrueValue); | |||
2475 | ||||
2476 | (void)RI; | |||
2477 | ||||
2478 | DEBUG(dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:" << "\n " << *BI << "NewRet = " << * RI << "TRUEBLOCK: " << *TrueSucc << "FALSEBLOCK: " << *FalseSucc; } } while (false) | |||
2479 | << "\n " << *BI << "NewRet = " << *RIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:" << "\n " << *BI << "NewRet = " << * RI << "TRUEBLOCK: " << *TrueSucc << "FALSEBLOCK: " << *FalseSucc; } } while (false) | |||
2480 | << "TRUEBLOCK: " << *TrueSucc << "FALSEBLOCK: " << *FalseSucc)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:" << "\n " << *BI << "NewRet = " << * RI << "TRUEBLOCK: " << *TrueSucc << "FALSEBLOCK: " << *FalseSucc; } } while (false); | |||
2481 | ||||
2482 | EraseTerminatorInstAndDCECond(BI); | |||
2483 | ||||
2484 | return true; | |||
2485 | } | |||
2486 | ||||
2487 | /// Return true if the given instruction is available | |||
2488 | /// in its predecessor block. If yes, the instruction will be removed. | |||
2489 | static bool checkCSEInPredecessor(Instruction *Inst, BasicBlock *PB) { | |||
2490 | if (!isa<BinaryOperator>(Inst) && !isa<CmpInst>(Inst)) | |||
2491 | return false; | |||
2492 | for (Instruction &I : *PB) { | |||
2493 | Instruction *PBI = &I; | |||
2494 | // Check whether Inst and PBI generate the same value. | |||
2495 | if (Inst->isIdenticalTo(PBI)) { | |||
2496 | Inst->replaceAllUsesWith(PBI); | |||
2497 | Inst->eraseFromParent(); | |||
2498 | return true; | |||
2499 | } | |||
2500 | } | |||
2501 | return false; | |||
2502 | } | |||
2503 | ||||
2504 | /// Return true if either PBI or BI has branch weight available, and store | |||
2505 | /// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does | |||
2506 | /// not have branch weight, use 1:1 as its weight. | |||
2507 | static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI, | |||
2508 | uint64_t &PredTrueWeight, | |||
2509 | uint64_t &PredFalseWeight, | |||
2510 | uint64_t &SuccTrueWeight, | |||
2511 | uint64_t &SuccFalseWeight) { | |||
2512 | bool PredHasWeights = | |||
2513 | PBI->extractProfMetadata(PredTrueWeight, PredFalseWeight); | |||
2514 | bool SuccHasWeights = | |||
2515 | BI->extractProfMetadata(SuccTrueWeight, SuccFalseWeight); | |||
2516 | if (PredHasWeights || SuccHasWeights) { | |||
2517 | if (!PredHasWeights) | |||
2518 | PredTrueWeight = PredFalseWeight = 1; | |||
2519 | if (!SuccHasWeights) | |||
2520 | SuccTrueWeight = SuccFalseWeight = 1; | |||
2521 | return true; | |||
2522 | } else { | |||
2523 | return false; | |||
2524 | } | |||
2525 | } | |||
2526 | ||||
2527 | /// If this basic block is simple enough, and if a predecessor branches to us | |||
2528 | /// and one of our successors, fold the block into the predecessor and use | |||
2529 | /// logical operations to pick the right destination. | |||
2530 | bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) { | |||
2531 | BasicBlock *BB = BI->getParent(); | |||
2532 | ||||
2533 | Instruction *Cond = nullptr; | |||
2534 | if (BI->isConditional()) | |||
2535 | Cond = dyn_cast<Instruction>(BI->getCondition()); | |||
2536 | else { | |||
2537 | // For unconditional branch, check for a simple CFG pattern, where | |||
2538 | // BB has a single predecessor and BB's successor is also its predecessor's | |||
2539 | // successor. If such pattern exists, check for CSE between BB and its | |||
2540 | // predecessor. | |||
2541 | if (BasicBlock *PB = BB->getSinglePredecessor()) | |||
2542 | if (BranchInst *PBI = dyn_cast<BranchInst>(PB->getTerminator())) | |||
2543 | if (PBI->isConditional() && | |||
2544 | (BI->getSuccessor(0) == PBI->getSuccessor(0) || | |||
2545 | BI->getSuccessor(0) == PBI->getSuccessor(1))) { | |||
2546 | for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) { | |||
2547 | Instruction *Curr = &*I++; | |||
2548 | if (isa<CmpInst>(Curr)) { | |||
2549 | Cond = Curr; | |||
2550 | break; | |||
2551 | } | |||
2552 | // Quit if we can't remove this instruction. | |||
2553 | if (!checkCSEInPredecessor(Curr, PB)) | |||
2554 | return false; | |||
2555 | } | |||
2556 | } | |||
2557 | ||||
2558 | if (!Cond) | |||
2559 | return false; | |||
2560 | } | |||
2561 | ||||
2562 | if (!Cond || (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond)) || | |||
2563 | Cond->getParent() != BB || !Cond->hasOneUse()) | |||
2564 | return false; | |||
2565 | ||||
2566 | // Make sure the instruction after the condition is the cond branch. | |||
2567 | BasicBlock::iterator CondIt = ++Cond->getIterator(); | |||
2568 | ||||
2569 | // Ignore dbg intrinsics. | |||
2570 | while (isa<DbgInfoIntrinsic>(CondIt)) | |||
2571 | ++CondIt; | |||
2572 | ||||
2573 | if (&*CondIt != BI) | |||
2574 | return false; | |||
2575 | ||||
2576 | // Only allow this transformation if computing the condition doesn't involve | |||
2577 | // too many instructions and these involved instructions can be executed | |||
2578 | // unconditionally. We denote all involved instructions except the condition | |||
2579 | // as "bonus instructions", and only allow this transformation when the | |||
2580 | // number of the bonus instructions does not exceed a certain threshold. | |||
2581 | unsigned NumBonusInsts = 0; | |||
2582 | for (auto I = BB->begin(); Cond != &*I; ++I) { | |||
2583 | // Ignore dbg intrinsics. | |||
2584 | if (isa<DbgInfoIntrinsic>(I)) | |||
2585 | continue; | |||
2586 | if (!I->hasOneUse() || !isSafeToSpeculativelyExecute(&*I)) | |||
2587 | return false; | |||
2588 | // I has only one use and can be executed unconditionally. | |||
2589 | Instruction *User = dyn_cast<Instruction>(I->user_back()); | |||
2590 | if (User == nullptr || User->getParent() != BB) | |||
2591 | return false; | |||
2592 | // I is used in the same BB. Since BI uses Cond and doesn't have more slots | |||
2593 | // to use any other instruction, User must be an instruction between next(I) | |||
2594 | // and Cond. | |||
2595 | ++NumBonusInsts; | |||
2596 | // Early exits once we reach the limit. | |||
2597 | if (NumBonusInsts > BonusInstThreshold) | |||
2598 | return false; | |||
2599 | } | |||
2600 | ||||
2601 | // Cond is known to be a compare or binary operator. Check to make sure that | |||
2602 | // neither operand is a potentially-trapping constant expression. | |||
2603 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(0))) | |||
2604 | if (CE->canTrap()) | |||
2605 | return false; | |||
2606 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(1))) | |||
2607 | if (CE->canTrap()) | |||
2608 | return false; | |||
2609 | ||||
2610 | // Finally, don't infinitely unroll conditional loops. | |||
2611 | BasicBlock *TrueDest = BI->getSuccessor(0); | |||
2612 | BasicBlock *FalseDest = (BI->isConditional()) ? BI->getSuccessor(1) : nullptr; | |||
2613 | if (TrueDest == BB || FalseDest == BB) | |||
2614 | return false; | |||
2615 | ||||
2616 | for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) { | |||
2617 | BasicBlock *PredBlock = *PI; | |||
2618 | BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator()); | |||
2619 | ||||
2620 | // Check that we have two conditional branches. If there is a PHI node in | |||
2621 | // the common successor, verify that the same value flows in from both | |||
2622 | // blocks. | |||
2623 | SmallVector<PHINode *, 4> PHIs; | |||
2624 | if (!PBI || PBI->isUnconditional() || | |||
2625 | (BI->isConditional() && !SafeToMergeTerminators(BI, PBI)) || | |||
2626 | (!BI->isConditional() && | |||
2627 | !isProfitableToFoldUnconditional(BI, PBI, Cond, PHIs))) | |||
2628 | continue; | |||
2629 | ||||
2630 | // Determine if the two branches share a common destination. | |||
2631 | Instruction::BinaryOps Opc = Instruction::BinaryOpsEnd; | |||
2632 | bool InvertPredCond = false; | |||
2633 | ||||
2634 | if (BI->isConditional()) { | |||
2635 | if (PBI->getSuccessor(0) == TrueDest) { | |||
2636 | Opc = Instruction::Or; | |||
2637 | } else if (PBI->getSuccessor(1) == FalseDest) { | |||
2638 | Opc = Instruction::And; | |||
2639 | } else if (PBI->getSuccessor(0) == FalseDest) { | |||
2640 | Opc = Instruction::And; | |||
2641 | InvertPredCond = true; | |||
2642 | } else if (PBI->getSuccessor(1) == TrueDest) { | |||
2643 | Opc = Instruction::Or; | |||
2644 | InvertPredCond = true; | |||
2645 | } else { | |||
2646 | continue; | |||
2647 | } | |||
2648 | } else { | |||
2649 | if (PBI->getSuccessor(0) != TrueDest && PBI->getSuccessor(1) != TrueDest) | |||
2650 | continue; | |||
2651 | } | |||
2652 | ||||
2653 | DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB; } } while (false); | |||
2654 | IRBuilder<> Builder(PBI); | |||
2655 | ||||
2656 | // If we need to invert the condition in the pred block to match, do so now. | |||
2657 | if (InvertPredCond) { | |||
2658 | Value *NewCond = PBI->getCondition(); | |||
2659 | ||||
2660 | if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) { | |||
2661 | CmpInst *CI = cast<CmpInst>(NewCond); | |||
2662 | CI->setPredicate(CI->getInversePredicate()); | |||
2663 | } else { | |||
2664 | NewCond = | |||
2665 | Builder.CreateNot(NewCond, PBI->getCondition()->getName() + ".not"); | |||
2666 | } | |||
2667 | ||||
2668 | PBI->setCondition(NewCond); | |||
2669 | PBI->swapSuccessors(); | |||
2670 | } | |||
2671 | ||||
2672 | // If we have bonus instructions, clone them into the predecessor block. | |||
2673 | // Note that there may be multiple predecessor blocks, so we cannot move | |||
2674 | // bonus instructions to a predecessor block. | |||
2675 | ValueToValueMapTy VMap; // maps original values to cloned values | |||
2676 | // We already make sure Cond is the last instruction before BI. Therefore, | |||
2677 | // all instructions before Cond other than DbgInfoIntrinsic are bonus | |||
2678 | // instructions. | |||
2679 | for (auto BonusInst = BB->begin(); Cond != &*BonusInst; ++BonusInst) { | |||
2680 | if (isa<DbgInfoIntrinsic>(BonusInst)) | |||
2681 | continue; | |||
2682 | Instruction *NewBonusInst = BonusInst->clone(); | |||
2683 | RemapInstruction(NewBonusInst, VMap, | |||
2684 | RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); | |||
2685 | VMap[&*BonusInst] = NewBonusInst; | |||
2686 | ||||
2687 | // If we moved a load, we cannot any longer claim any knowledge about | |||
2688 | // its potential value. The previous information might have been valid | |||
2689 | // only given the branch precondition. | |||
2690 | // For an analogous reason, we must also drop all the metadata whose | |||
2691 | // semantics we don't understand. | |||
2692 | NewBonusInst->dropUnknownNonDebugMetadata(); | |||
2693 | ||||
2694 | PredBlock->getInstList().insert(PBI->getIterator(), NewBonusInst); | |||
2695 | NewBonusInst->takeName(&*BonusInst); | |||
2696 | BonusInst->setName(BonusInst->getName() + ".old"); | |||
2697 | } | |||
2698 | ||||
2699 | // Clone Cond into the predecessor basic block, and or/and the | |||
2700 | // two conditions together. | |||
2701 | Instruction *New = Cond->clone(); | |||
2702 | RemapInstruction(New, VMap, | |||
2703 | RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); | |||
2704 | PredBlock->getInstList().insert(PBI->getIterator(), New); | |||
2705 | New->takeName(Cond); | |||
2706 | Cond->setName(New->getName() + ".old"); | |||
2707 | ||||
2708 | if (BI->isConditional()) { | |||
2709 | Instruction *NewCond = cast<Instruction>( | |||
2710 | Builder.CreateBinOp(Opc, PBI->getCondition(), New, "or.cond")); | |||
2711 | PBI->setCondition(NewCond); | |||
2712 | ||||
2713 | uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight; | |||
2714 | bool HasWeights = | |||
2715 | extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight, | |||
2716 | SuccTrueWeight, SuccFalseWeight); | |||
2717 | SmallVector<uint64_t, 8> NewWeights; | |||
2718 | ||||
2719 | if (PBI->getSuccessor(0) == BB) { | |||
2720 | if (HasWeights) { | |||
2721 | // PBI: br i1 %x, BB, FalseDest | |||
2722 | // BI: br i1 %y, TrueDest, FalseDest | |||
2723 | // TrueWeight is TrueWeight for PBI * TrueWeight for BI. | |||
2724 | NewWeights.push_back(PredTrueWeight * SuccTrueWeight); | |||
2725 | // FalseWeight is FalseWeight for PBI * TotalWeight for BI + | |||
2726 | // TrueWeight for PBI * FalseWeight for BI. | |||
2727 | // We assume that total weights of a BranchInst can fit into 32 bits. | |||
2728 | // Therefore, we will not have overflow using 64-bit arithmetic. | |||
2729 | NewWeights.push_back(PredFalseWeight * | |||
2730 | (SuccFalseWeight + SuccTrueWeight) + | |||
2731 | PredTrueWeight * SuccFalseWeight); | |||
2732 | } | |||
2733 | AddPredecessorToBlock(TrueDest, PredBlock, BB); | |||
2734 | PBI->setSuccessor(0, TrueDest); | |||
2735 | } | |||
2736 | if (PBI->getSuccessor(1) == BB) { | |||
2737 | if (HasWeights) { | |||
2738 | // PBI: br i1 %x, TrueDest, BB | |||
2739 | // BI: br i1 %y, TrueDest, FalseDest | |||
2740 | // TrueWeight is TrueWeight for PBI * TotalWeight for BI + | |||
2741 | // FalseWeight for PBI * TrueWeight for BI. | |||
2742 | NewWeights.push_back(PredTrueWeight * | |||
2743 | (SuccFalseWeight + SuccTrueWeight) + | |||
2744 | PredFalseWeight * SuccTrueWeight); | |||
2745 | // FalseWeight is FalseWeight for PBI * FalseWeight for BI. | |||
2746 | NewWeights.push_back(PredFalseWeight * SuccFalseWeight); | |||
2747 | } | |||
2748 | AddPredecessorToBlock(FalseDest, PredBlock, BB); | |||
2749 | PBI->setSuccessor(1, FalseDest); | |||
2750 | } | |||
2751 | if (NewWeights.size() == 2) { | |||
2752 | // Halve the weights if any of them cannot fit in an uint32_t | |||
2753 | FitWeights(NewWeights); | |||
2754 | ||||
2755 | SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), | |||
2756 | NewWeights.end()); | |||
2757 | setBranchWeights(PBI, MDWeights[0], MDWeights[1]); | |||
2758 | } else | |||
2759 | PBI->setMetadata(LLVMContext::MD_prof, nullptr); | |||
2760 | } else { | |||
2761 | // Update PHI nodes in the common successors. | |||
2762 | for (unsigned i = 0, e = PHIs.size(); i != e; ++i) { | |||
2763 | ConstantInt *PBI_C = cast<ConstantInt>( | |||
2764 | PHIs[i]->getIncomingValueForBlock(PBI->getParent())); | |||
2765 | assert(PBI_C->getType()->isIntegerTy(1))(static_cast <bool> (PBI_C->getType()->isIntegerTy (1)) ? void (0) : __assert_fail ("PBI_C->getType()->isIntegerTy(1)" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 2765, __extension__ __PRETTY_FUNCTION__)); | |||
2766 | Instruction *MergedCond = nullptr; | |||
2767 | if (PBI->getSuccessor(0) == TrueDest) { | |||
2768 | // Create (PBI_Cond and PBI_C) or (!PBI_Cond and BI_Value) | |||
2769 | // PBI_C is true: PBI_Cond or (!PBI_Cond and BI_Value) | |||
2770 | // is false: !PBI_Cond and BI_Value | |||
2771 | Instruction *NotCond = cast<Instruction>( | |||
2772 | Builder.CreateNot(PBI->getCondition(), "not.cond")); | |||
2773 | MergedCond = cast<Instruction>( | |||
2774 | Builder.CreateBinOp(Instruction::And, NotCond, New, "and.cond")); | |||
2775 | if (PBI_C->isOne()) | |||
2776 | MergedCond = cast<Instruction>(Builder.CreateBinOp( | |||
2777 | Instruction::Or, PBI->getCondition(), MergedCond, "or.cond")); | |||
2778 | } else { | |||
2779 | // Create (PBI_Cond and BI_Value) or (!PBI_Cond and PBI_C) | |||
2780 | // PBI_C is true: (PBI_Cond and BI_Value) or (!PBI_Cond) | |||
2781 | // is false: PBI_Cond and BI_Value | |||
2782 | MergedCond = cast<Instruction>(Builder.CreateBinOp( | |||
2783 | Instruction::And, PBI->getCondition(), New, "and.cond")); | |||
2784 | if (PBI_C->isOne()) { | |||
2785 | Instruction *NotCond = cast<Instruction>( | |||
2786 | Builder.CreateNot(PBI->getCondition(), "not.cond")); | |||
2787 | MergedCond = cast<Instruction>(Builder.CreateBinOp( | |||
2788 | Instruction::Or, NotCond, MergedCond, "or.cond")); | |||
2789 | } | |||
2790 | } | |||
2791 | // Update PHI Node. | |||
2792 | PHIs[i]->setIncomingValue(PHIs[i]->getBasicBlockIndex(PBI->getParent()), | |||
2793 | MergedCond); | |||
2794 | } | |||
2795 | // Change PBI from Conditional to Unconditional. | |||
2796 | BranchInst *New_PBI = BranchInst::Create(TrueDest, PBI); | |||
2797 | EraseTerminatorInstAndDCECond(PBI); | |||
2798 | PBI = New_PBI; | |||
2799 | } | |||
2800 | ||||
2801 | // If BI was a loop latch, it may have had associated loop metadata. | |||
2802 | // We need to copy it to the new latch, that is, PBI. | |||
2803 | if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop)) | |||
2804 | PBI->setMetadata(LLVMContext::MD_loop, LoopMD); | |||
2805 | ||||
2806 | // TODO: If BB is reachable from all paths through PredBlock, then we | |||
2807 | // could replace PBI's branch probabilities with BI's. | |||
2808 | ||||
2809 | // Copy any debug value intrinsics into the end of PredBlock. | |||
2810 | for (Instruction &I : *BB) | |||
2811 | if (isa<DbgInfoIntrinsic>(I)) | |||
2812 | I.clone()->insertBefore(PBI); | |||
2813 | ||||
2814 | return true; | |||
2815 | } | |||
2816 | return false; | |||
2817 | } | |||
2818 | ||||
2819 | // If there is only one store in BB1 and BB2, return it, otherwise return | |||
2820 | // nullptr. | |||
2821 | static StoreInst *findUniqueStoreInBlocks(BasicBlock *BB1, BasicBlock *BB2) { | |||
2822 | StoreInst *S = nullptr; | |||
2823 | for (auto *BB : {BB1, BB2}) { | |||
2824 | if (!BB) | |||
2825 | continue; | |||
2826 | for (auto &I : *BB) | |||
2827 | if (auto *SI = dyn_cast<StoreInst>(&I)) { | |||
2828 | if (S) | |||
2829 | // Multiple stores seen. | |||
2830 | return nullptr; | |||
2831 | else | |||
2832 | S = SI; | |||
2833 | } | |||
2834 | } | |||
2835 | return S; | |||
2836 | } | |||
2837 | ||||
2838 | static Value *ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB, | |||
2839 | Value *AlternativeV = nullptr) { | |||
2840 | // PHI is going to be a PHI node that allows the value V that is defined in | |||
2841 | // BB to be referenced in BB's only successor. | |||
2842 | // | |||
2843 | // If AlternativeV is nullptr, the only value we care about in PHI is V. It | |||
2844 | // doesn't matter to us what the other operand is (it'll never get used). We | |||
2845 | // could just create a new PHI with an undef incoming value, but that could | |||
2846 | // increase register pressure if EarlyCSE/InstCombine can't fold it with some | |||
2847 | // other PHI. So here we directly look for some PHI in BB's successor with V | |||
2848 | // as an incoming operand. If we find one, we use it, else we create a new | |||
2849 | // one. | |||
2850 | // | |||
2851 | // If AlternativeV is not nullptr, we care about both incoming values in PHI. | |||
2852 | // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV] | |||
2853 | // where OtherBB is the single other predecessor of BB's only successor. | |||
2854 | PHINode *PHI = nullptr; | |||
2855 | BasicBlock *Succ = BB->getSingleSuccessor(); | |||
2856 | ||||
2857 | for (auto I = Succ->begin(); isa<PHINode>(I); ++I) | |||
2858 | if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) { | |||
2859 | PHI = cast<PHINode>(I); | |||
2860 | if (!AlternativeV) | |||
2861 | break; | |||
2862 | ||||
2863 | assert(std::distance(pred_begin(Succ), pred_end(Succ)) == 2)(static_cast <bool> (std::distance(pred_begin(Succ), pred_end (Succ)) == 2) ? void (0) : __assert_fail ("std::distance(pred_begin(Succ), pred_end(Succ)) == 2" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 2863, __extension__ __PRETTY_FUNCTION__)); | |||
2864 | auto PredI = pred_begin(Succ); | |||
2865 | BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI; | |||
2866 | if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV) | |||
2867 | break; | |||
2868 | PHI = nullptr; | |||
2869 | } | |||
2870 | if (PHI) | |||
2871 | return PHI; | |||
2872 | ||||
2873 | // If V is not an instruction defined in BB, just return it. | |||
2874 | if (!AlternativeV && | |||
2875 | (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB)) | |||
2876 | return V; | |||
2877 | ||||
2878 | PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge", &Succ->front()); | |||
2879 | PHI->addIncoming(V, BB); | |||
2880 | for (BasicBlock *PredBB : predecessors(Succ)) | |||
2881 | if (PredBB != BB) | |||
2882 | PHI->addIncoming( | |||
2883 | AlternativeV ? AlternativeV : UndefValue::get(V->getType()), PredBB); | |||
2884 | return PHI; | |||
2885 | } | |||
2886 | ||||
2887 | static bool mergeConditionalStoreToAddress(BasicBlock *PTB, BasicBlock *PFB, | |||
2888 | BasicBlock *QTB, BasicBlock *QFB, | |||
2889 | BasicBlock *PostBB, Value *Address, | |||
2890 | bool InvertPCond, bool InvertQCond, | |||
2891 | const DataLayout &DL) { | |||
2892 | auto IsaBitcastOfPointerType = [](const Instruction &I) { | |||
2893 | return Operator::getOpcode(&I) == Instruction::BitCast && | |||
2894 | I.getType()->isPointerTy(); | |||
2895 | }; | |||
2896 | ||||
2897 | // If we're not in aggressive mode, we only optimize if we have some | |||
2898 | // confidence that by optimizing we'll allow P and/or Q to be if-converted. | |||
2899 | auto IsWorthwhile = [&](BasicBlock *BB) { | |||
2900 | if (!BB) | |||
2901 | return true; | |||
2902 | // Heuristic: if the block can be if-converted/phi-folded and the | |||
2903 | // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to | |||
2904 | // thread this store. | |||
2905 | unsigned N = 0; | |||
2906 | for (auto &I : *BB) { | |||
2907 | // Cheap instructions viable for folding. | |||
2908 | if (isa<BinaryOperator>(I) || isa<GetElementPtrInst>(I) || | |||
2909 | isa<StoreInst>(I)) | |||
2910 | ++N; | |||
2911 | // Free instructions. | |||
2912 | else if (isa<TerminatorInst>(I) || isa<DbgInfoIntrinsic>(I) || | |||
2913 | IsaBitcastOfPointerType(I)) | |||
2914 | continue; | |||
2915 | else | |||
2916 | return false; | |||
2917 | } | |||
2918 | // The store we want to merge is counted in N, so add 1 to make sure | |||
2919 | // we're counting the instructions that would be left. | |||
2920 | return N <= (PHINodeFoldingThreshold + 1); | |||
2921 | }; | |||
2922 | ||||
2923 | if (!MergeCondStoresAggressively && | |||
2924 | (!IsWorthwhile(PTB) || !IsWorthwhile(PFB) || !IsWorthwhile(QTB) || | |||
2925 | !IsWorthwhile(QFB))) | |||
2926 | return false; | |||
2927 | ||||
2928 | // For every pointer, there must be exactly two stores, one coming from | |||
2929 | // PTB or PFB, and the other from QTB or QFB. We don't support more than one | |||
2930 | // store (to any address) in PTB,PFB or QTB,QFB. | |||
2931 | // FIXME: We could relax this restriction with a bit more work and performance | |||
2932 | // testing. | |||
2933 | StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB); | |||
2934 | StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB); | |||
2935 | if (!PStore || !QStore) | |||
2936 | return false; | |||
2937 | ||||
2938 | // Now check the stores are compatible. | |||
2939 | if (!QStore->isUnordered() || !PStore->isUnordered()) | |||
2940 | return false; | |||
2941 | ||||
2942 | // Check that sinking the store won't cause program behavior changes. Sinking | |||
2943 | // the store out of the Q blocks won't change any behavior as we're sinking | |||
2944 | // from a block to its unconditional successor. But we're moving a store from | |||
2945 | // the P blocks down through the middle block (QBI) and past both QFB and QTB. | |||
2946 | // So we need to check that there are no aliasing loads or stores in | |||
2947 | // QBI, QTB and QFB. We also need to check there are no conflicting memory | |||
2948 | // operations between PStore and the end of its parent block. | |||
2949 | // | |||
2950 | // The ideal way to do this is to query AliasAnalysis, but we don't | |||
2951 | // preserve AA currently so that is dangerous. Be super safe and just | |||
2952 | // check there are no other memory operations at all. | |||
2953 | for (auto &I : *QFB->getSinglePredecessor()) | |||
2954 | if (I.mayReadOrWriteMemory()) | |||
2955 | return false; | |||
2956 | for (auto &I : *QFB) | |||
2957 | if (&I != QStore && I.mayReadOrWriteMemory()) | |||
2958 | return false; | |||
2959 | if (QTB) | |||
2960 | for (auto &I : *QTB) | |||
2961 | if (&I != QStore && I.mayReadOrWriteMemory()) | |||
2962 | return false; | |||
2963 | for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end(); | |||
2964 | I != E; ++I) | |||
2965 | if (&*I != PStore && I->mayReadOrWriteMemory()) | |||
2966 | return false; | |||
2967 | ||||
2968 | // OK, we're going to sink the stores to PostBB. The store has to be | |||
2969 | // conditional though, so first create the predicate. | |||
2970 | Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator()) | |||
2971 | ->getCondition(); | |||
2972 | Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator()) | |||
2973 | ->getCondition(); | |||
2974 | ||||
2975 | Value *PPHI = ensureValueAvailableInSuccessor(PStore->getValueOperand(), | |||
2976 | PStore->getParent()); | |||
2977 | Value *QPHI = ensureValueAvailableInSuccessor(QStore->getValueOperand(), | |||
2978 | QStore->getParent(), PPHI); | |||
2979 | ||||
2980 | IRBuilder<> QB(&*PostBB->getFirstInsertionPt()); | |||
2981 | ||||
2982 | Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond); | |||
2983 | Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond); | |||
2984 | ||||
2985 | if (InvertPCond) | |||
2986 | PPred = QB.CreateNot(PPred); | |||
2987 | if (InvertQCond) | |||
2988 | QPred = QB.CreateNot(QPred); | |||
2989 | Value *CombinedPred = QB.CreateOr(PPred, QPred); | |||
2990 | ||||
2991 | auto *T = | |||
2992 | SplitBlockAndInsertIfThen(CombinedPred, &*QB.GetInsertPoint(), false); | |||
2993 | QB.SetInsertPoint(T); | |||
2994 | StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address)); | |||
2995 | AAMDNodes AAMD; | |||
2996 | PStore->getAAMetadata(AAMD, /*Merge=*/false); | |||
2997 | PStore->getAAMetadata(AAMD, /*Merge=*/true); | |||
2998 | SI->setAAMetadata(AAMD); | |||
2999 | unsigned PAlignment = PStore->getAlignment(); | |||
3000 | unsigned QAlignment = QStore->getAlignment(); | |||
3001 | unsigned TypeAlignment = | |||
3002 | DL.getABITypeAlignment(SI->getValueOperand()->getType()); | |||
3003 | unsigned MinAlignment; | |||
3004 | unsigned MaxAlignment; | |||
3005 | std::tie(MinAlignment, MaxAlignment) = std::minmax(PAlignment, QAlignment); | |||
3006 | // Choose the minimum alignment. If we could prove both stores execute, we | |||
3007 | // could use biggest one. In this case, though, we only know that one of the | |||
3008 | // stores executes. And we don't know it's safe to take the alignment from a | |||
3009 | // store that doesn't execute. | |||
3010 | if (MinAlignment != 0) { | |||
3011 | // Choose the minimum of all non-zero alignments. | |||
3012 | SI->setAlignment(MinAlignment); | |||
3013 | } else if (MaxAlignment != 0) { | |||
3014 | // Choose the minimal alignment between the non-zero alignment and the ABI | |||
3015 | // default alignment for the type of the stored value. | |||
3016 | SI->setAlignment(std::min(MaxAlignment, TypeAlignment)); | |||
3017 | } else { | |||
3018 | // If both alignments are zero, use ABI default alignment for the type of | |||
3019 | // the stored value. | |||
3020 | SI->setAlignment(TypeAlignment); | |||
3021 | } | |||
3022 | ||||
3023 | QStore->eraseFromParent(); | |||
3024 | PStore->eraseFromParent(); | |||
3025 | ||||
3026 | return true; | |||
3027 | } | |||
3028 | ||||
3029 | static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI, | |||
3030 | const DataLayout &DL) { | |||
3031 | // The intention here is to find diamonds or triangles (see below) where each | |||
3032 | // conditional block contains a store to the same address. Both of these | |||
3033 | // stores are conditional, so they can't be unconditionally sunk. But it may | |||
3034 | // be profitable to speculatively sink the stores into one merged store at the | |||
3035 | // end, and predicate the merged store on the union of the two conditions of | |||
3036 | // PBI and QBI. | |||
3037 | // | |||
3038 | // This can reduce the number of stores executed if both of the conditions are | |||
3039 | // true, and can allow the blocks to become small enough to be if-converted. | |||
3040 | // This optimization will also chain, so that ladders of test-and-set | |||
3041 | // sequences can be if-converted away. | |||
3042 | // | |||
3043 | // We only deal with simple diamonds or triangles: | |||
3044 | // | |||
3045 | // PBI or PBI or a combination of the two | |||
3046 | // / \ | \ | |||
3047 | // PTB PFB | PFB | |||
3048 | // \ / | / | |||
3049 | // QBI QBI | |||
3050 | // / \ | \ | |||
3051 | // QTB QFB | QFB | |||
3052 | // \ / | / | |||
3053 | // PostBB PostBB | |||
3054 | // | |||
3055 | // We model triangles as a type of diamond with a nullptr "true" block. | |||
3056 | // Triangles are canonicalized so that the fallthrough edge is represented by | |||
3057 | // a true condition, as in the diagram above. | |||
3058 | BasicBlock *PTB = PBI->getSuccessor(0); | |||
3059 | BasicBlock *PFB = PBI->getSuccessor(1); | |||
3060 | BasicBlock *QTB = QBI->getSuccessor(0); | |||
3061 | BasicBlock *QFB = QBI->getSuccessor(1); | |||
3062 | BasicBlock *PostBB = QFB->getSingleSuccessor(); | |||
3063 | ||||
3064 | // Make sure we have a good guess for PostBB. If QTB's only successor is | |||
3065 | // QFB, then QFB is a better PostBB. | |||
3066 | if (QTB->getSingleSuccessor() == QFB) | |||
3067 | PostBB = QFB; | |||
3068 | ||||
3069 | // If we couldn't find a good PostBB, stop. | |||
3070 | if (!PostBB) | |||
3071 | return false; | |||
3072 | ||||
3073 | bool InvertPCond = false, InvertQCond = false; | |||
3074 | // Canonicalize fallthroughs to the true branches. | |||
3075 | if (PFB == QBI->getParent()) { | |||
3076 | std::swap(PFB, PTB); | |||
3077 | InvertPCond = true; | |||
3078 | } | |||
3079 | if (QFB == PostBB) { | |||
3080 | std::swap(QFB, QTB); | |||
3081 | InvertQCond = true; | |||
3082 | } | |||
3083 | ||||
3084 | // From this point on we can assume PTB or QTB may be fallthroughs but PFB | |||
3085 | // and QFB may not. Model fallthroughs as a nullptr block. | |||
3086 | if (PTB == QBI->getParent()) | |||
3087 | PTB = nullptr; | |||
3088 | if (QTB == PostBB) | |||
3089 | QTB = nullptr; | |||
3090 | ||||
3091 | // Legality bailouts. We must have at least the non-fallthrough blocks and | |||
3092 | // the post-dominating block, and the non-fallthroughs must only have one | |||
3093 | // predecessor. | |||
3094 | auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) { | |||
3095 | return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S; | |||
3096 | }; | |||
3097 | if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) || | |||
3098 | !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB)) | |||
3099 | return false; | |||
3100 | if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) || | |||
3101 | (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB))) | |||
3102 | return false; | |||
3103 | if (!PostBB->hasNUses(2) || !QBI->getParent()->hasNUses(2)) | |||
3104 | return false; | |||
3105 | ||||
3106 | // OK, this is a sequence of two diamonds or triangles. | |||
3107 | // Check if there are stores in PTB or PFB that are repeated in QTB or QFB. | |||
3108 | SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses; | |||
3109 | for (auto *BB : {PTB, PFB}) { | |||
3110 | if (!BB) | |||
3111 | continue; | |||
3112 | for (auto &I : *BB) | |||
3113 | if (StoreInst *SI = dyn_cast<StoreInst>(&I)) | |||
3114 | PStoreAddresses.insert(SI->getPointerOperand()); | |||
3115 | } | |||
3116 | for (auto *BB : {QTB, QFB}) { | |||
3117 | if (!BB) | |||
3118 | continue; | |||
3119 | for (auto &I : *BB) | |||
3120 | if (StoreInst *SI = dyn_cast<StoreInst>(&I)) | |||
3121 | QStoreAddresses.insert(SI->getPointerOperand()); | |||
3122 | } | |||
3123 | ||||
3124 | set_intersect(PStoreAddresses, QStoreAddresses); | |||
3125 | // set_intersect mutates PStoreAddresses in place. Rename it here to make it | |||
3126 | // clear what it contains. | |||
3127 | auto &CommonAddresses = PStoreAddresses; | |||
3128 | ||||
3129 | bool Changed = false; | |||
3130 | for (auto *Address : CommonAddresses) | |||
3131 | Changed |= mergeConditionalStoreToAddress( | |||
3132 | PTB, PFB, QTB, QFB, PostBB, Address, InvertPCond, InvertQCond, DL); | |||
3133 | return Changed; | |||
3134 | } | |||
3135 | ||||
3136 | /// If we have a conditional branch as a predecessor of another block, | |||
3137 | /// this function tries to simplify it. We know | |||
3138 | /// that PBI and BI are both conditional branches, and BI is in one of the | |||
3139 | /// successor blocks of PBI - PBI branches to BI. | |||
3140 | static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, | |||
3141 | const DataLayout &DL) { | |||
3142 | assert(PBI->isConditional() && BI->isConditional())(static_cast <bool> (PBI->isConditional() && BI->isConditional()) ? void (0) : __assert_fail ("PBI->isConditional() && BI->isConditional()" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 3142, __extension__ __PRETTY_FUNCTION__)); | |||
3143 | BasicBlock *BB = BI->getParent(); | |||
3144 | ||||
3145 | // If this block ends with a branch instruction, and if there is a | |||
3146 | // predecessor that ends on a branch of the same condition, make | |||
3147 | // this conditional branch redundant. | |||
3148 | if (PBI->getCondition() == BI->getCondition() && | |||
3149 | PBI->getSuccessor(0) != PBI->getSuccessor(1)) { | |||
3150 | // Okay, the outcome of this conditional branch is statically | |||
3151 | // knowable. If this block had a single pred, handle specially. | |||
3152 | if (BB->getSinglePredecessor()) { | |||
3153 | // Turn this into a branch on constant. | |||
3154 | bool CondIsTrue = PBI->getSuccessor(0) == BB; | |||
3155 | BI->setCondition( | |||
3156 | ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue)); | |||
3157 | return true; // Nuke the branch on constant. | |||
3158 | } | |||
3159 | ||||
3160 | // Otherwise, if there are multiple predecessors, insert a PHI that merges | |||
3161 | // in the constant and simplify the block result. Subsequent passes of | |||
3162 | // simplifycfg will thread the block. | |||
3163 | if (BlockIsSimpleEnoughToThreadThrough(BB)) { | |||
3164 | pred_iterator PB = pred_begin(BB), PE = pred_end(BB); | |||
3165 | PHINode *NewPN = PHINode::Create( | |||
3166 | Type::getInt1Ty(BB->getContext()), std::distance(PB, PE), | |||
3167 | BI->getCondition()->getName() + ".pr", &BB->front()); | |||
3168 | // Okay, we're going to insert the PHI node. Since PBI is not the only | |||
3169 | // predecessor, compute the PHI'd conditional value for all of the preds. | |||
3170 | // Any predecessor where the condition is not computable we keep symbolic. | |||
3171 | for (pred_iterator PI = PB; PI != PE; ++PI) { | |||
3172 | BasicBlock *P = *PI; | |||
3173 | if ((PBI = dyn_cast<BranchInst>(P->getTerminator())) && PBI != BI && | |||
3174 | PBI->isConditional() && PBI->getCondition() == BI->getCondition() && | |||
3175 | PBI->getSuccessor(0) != PBI->getSuccessor(1)) { | |||
3176 | bool CondIsTrue = PBI->getSuccessor(0) == BB; | |||
3177 | NewPN->addIncoming( | |||
3178 | ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue), | |||
3179 | P); | |||
3180 | } else { | |||
3181 | NewPN->addIncoming(BI->getCondition(), P); | |||
3182 | } | |||
3183 | } | |||
3184 | ||||
3185 | BI->setCondition(NewPN); | |||
3186 | return true; | |||
3187 | } | |||
3188 | } | |||
3189 | ||||
3190 | if (auto *CE = dyn_cast<ConstantExpr>(BI->getCondition())) | |||
3191 | if (CE->canTrap()) | |||
3192 | return false; | |||
3193 | ||||
3194 | // If both branches are conditional and both contain stores to the same | |||
3195 | // address, remove the stores from the conditionals and create a conditional | |||
3196 | // merged store at the end. | |||
3197 | if (MergeCondStores && mergeConditionalStores(PBI, BI, DL)) | |||
3198 | return true; | |||
3199 | ||||
3200 | // If this is a conditional branch in an empty block, and if any | |||
3201 | // predecessors are a conditional branch to one of our destinations, | |||
3202 | // fold the conditions into logical ops and one cond br. | |||
3203 | BasicBlock::iterator BBI = BB->begin(); | |||
3204 | // Ignore dbg intrinsics. | |||
3205 | while (isa<DbgInfoIntrinsic>(BBI)) | |||
3206 | ++BBI; | |||
3207 | if (&*BBI != BI) | |||
3208 | return false; | |||
3209 | ||||
3210 | int PBIOp, BIOp; | |||
3211 | if (PBI->getSuccessor(0) == BI->getSuccessor(0)) { | |||
3212 | PBIOp = 0; | |||
3213 | BIOp = 0; | |||
3214 | } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) { | |||
3215 | PBIOp = 0; | |||
3216 | BIOp = 1; | |||
3217 | } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) { | |||
3218 | PBIOp = 1; | |||
3219 | BIOp = 0; | |||
3220 | } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) { | |||
3221 | PBIOp = 1; | |||
3222 | BIOp = 1; | |||
3223 | } else { | |||
3224 | return false; | |||
3225 | } | |||
3226 | ||||
3227 | // Check to make sure that the other destination of this branch | |||
3228 | // isn't BB itself. If so, this is an infinite loop that will | |||
3229 | // keep getting unwound. | |||
3230 | if (PBI->getSuccessor(PBIOp) == BB) | |||
3231 | return false; | |||
3232 | ||||
3233 | // Do not perform this transformation if it would require | |||
3234 | // insertion of a large number of select instructions. For targets | |||
3235 | // without predication/cmovs, this is a big pessimization. | |||
3236 | ||||
3237 | // Also do not perform this transformation if any phi node in the common | |||
3238 | // destination block can trap when reached by BB or PBB (PR17073). In that | |||
3239 | // case, it would be unsafe to hoist the operation into a select instruction. | |||
3240 | ||||
3241 | BasicBlock *CommonDest = PBI->getSuccessor(PBIOp); | |||
3242 | unsigned NumPhis = 0; | |||
3243 | for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II); | |||
3244 | ++II, ++NumPhis) { | |||
3245 | if (NumPhis > 2) // Disable this xform. | |||
3246 | return false; | |||
3247 | ||||
3248 | PHINode *PN = cast<PHINode>(II); | |||
3249 | Value *BIV = PN->getIncomingValueForBlock(BB); | |||
3250 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(BIV)) | |||
3251 | if (CE->canTrap()) | |||
3252 | return false; | |||
3253 | ||||
3254 | unsigned PBBIdx = PN->getBasicBlockIndex(PBI->getParent()); | |||
3255 | Value *PBIV = PN->getIncomingValue(PBBIdx); | |||
3256 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(PBIV)) | |||
3257 | if (CE->canTrap()) | |||
3258 | return false; | |||
3259 | } | |||
3260 | ||||
3261 | // Finally, if everything is ok, fold the branches to logical ops. | |||
3262 | BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1); | |||
3263 | ||||
3264 | DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "FOLDING BRs:" << *PBI ->getParent() << "AND: " << *BI->getParent( ); } } while (false) | |||
3265 | << "AND: " << *BI->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "FOLDING BRs:" << *PBI ->getParent() << "AND: " << *BI->getParent( ); } } while (false); | |||
3266 | ||||
3267 | // If OtherDest *is* BB, then BB is a basic block with a single conditional | |||
3268 | // branch in it, where one edge (OtherDest) goes back to itself but the other | |||
3269 | // exits. We don't *know* that the program avoids the infinite loop | |||
3270 | // (even though that seems likely). If we do this xform naively, we'll end up | |||
3271 | // recursively unpeeling the loop. Since we know that (after the xform is | |||
3272 | // done) that the block *is* infinite if reached, we just make it an obviously | |||
3273 | // infinite loop with no cond branch. | |||
3274 | if (OtherDest == BB) { | |||
3275 | // Insert it at the end of the function, because it's either code, | |||
3276 | // or it won't matter if it's hot. :) | |||
3277 | BasicBlock *InfLoopBlock = | |||
3278 | BasicBlock::Create(BB->getContext(), "infloop", BB->getParent()); | |||
3279 | BranchInst::Create(InfLoopBlock, InfLoopBlock); | |||
3280 | OtherDest = InfLoopBlock; | |||
3281 | } | |||
3282 | ||||
3283 | DEBUG(dbgs() << *PBI->getParent()->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << *PBI->getParent()->getParent (); } } while (false); | |||
3284 | ||||
3285 | // BI may have other predecessors. Because of this, we leave | |||
3286 | // it alone, but modify PBI. | |||
3287 | ||||
3288 | // Make sure we get to CommonDest on True&True directions. | |||
3289 | Value *PBICond = PBI->getCondition(); | |||
3290 | IRBuilder<NoFolder> Builder(PBI); | |||
3291 | if (PBIOp) | |||
3292 | PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not"); | |||
3293 | ||||
3294 | Value *BICond = BI->getCondition(); | |||
3295 | if (BIOp) | |||
3296 | BICond = Builder.CreateNot(BICond, BICond->getName() + ".not"); | |||
3297 | ||||
3298 | // Merge the conditions. | |||
3299 | Value *Cond = Builder.CreateOr(PBICond, BICond, "brmerge"); | |||
3300 | ||||
3301 | // Modify PBI to branch on the new condition to the new dests. | |||
3302 | PBI->setCondition(Cond); | |||
3303 | PBI->setSuccessor(0, CommonDest); | |||
3304 | PBI->setSuccessor(1, OtherDest); | |||
3305 | ||||
3306 | // Update branch weight for PBI. | |||
3307 | uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight; | |||
3308 | uint64_t PredCommon, PredOther, SuccCommon, SuccOther; | |||
3309 | bool HasWeights = | |||
3310 | extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight, | |||
3311 | SuccTrueWeight, SuccFalseWeight); | |||
3312 | if (HasWeights) { | |||
3313 | PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight; | |||
3314 | PredOther = PBIOp ? PredTrueWeight : PredFalseWeight; | |||
3315 | SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight; | |||
3316 | SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight; | |||
3317 | // The weight to CommonDest should be PredCommon * SuccTotal + | |||
3318 | // PredOther * SuccCommon. | |||
3319 | // The weight to OtherDest should be PredOther * SuccOther. | |||
3320 | uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) + | |||
3321 | PredOther * SuccCommon, | |||
3322 | PredOther * SuccOther}; | |||
3323 | // Halve the weights if any of them cannot fit in an uint32_t | |||
3324 | FitWeights(NewWeights); | |||
3325 | ||||
3326 | setBranchWeights(PBI, NewWeights[0], NewWeights[1]); | |||
3327 | } | |||
3328 | ||||
3329 | // OtherDest may have phi nodes. If so, add an entry from PBI's | |||
3330 | // block that are identical to the entries for BI's block. | |||
3331 | AddPredecessorToBlock(OtherDest, PBI->getParent(), BB); | |||
3332 | ||||
3333 | // We know that the CommonDest already had an edge from PBI to | |||
3334 | // it. If it has PHIs though, the PHIs may have different | |||
3335 | // entries for BB and PBI's BB. If so, insert a select to make | |||
3336 | // them agree. | |||
3337 | for (PHINode &PN : CommonDest->phis()) { | |||
3338 | Value *BIV = PN.getIncomingValueForBlock(BB); | |||
3339 | unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent()); | |||
3340 | Value *PBIV = PN.getIncomingValue(PBBIdx); | |||
3341 | if (BIV != PBIV) { | |||
3342 | // Insert a select in PBI to pick the right value. | |||
3343 | SelectInst *NV = cast<SelectInst>( | |||
3344 | Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux")); | |||
3345 | PN.setIncomingValue(PBBIdx, NV); | |||
3346 | // Although the select has the same condition as PBI, the original branch | |||
3347 | // weights for PBI do not apply to the new select because the select's | |||
3348 | // 'logical' edges are incoming edges of the phi that is eliminated, not | |||
3349 | // the outgoing edges of PBI. | |||
3350 | if (HasWeights) { | |||
3351 | uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight; | |||
3352 | uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight; | |||
3353 | uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight; | |||
3354 | uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight; | |||
3355 | // The weight to PredCommonDest should be PredCommon * SuccTotal. | |||
3356 | // The weight to PredOtherDest should be PredOther * SuccCommon. | |||
3357 | uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther), | |||
3358 | PredOther * SuccCommon}; | |||
3359 | ||||
3360 | FitWeights(NewWeights); | |||
3361 | ||||
3362 | setBranchWeights(NV, NewWeights[0], NewWeights[1]); | |||
3363 | } | |||
3364 | } | |||
3365 | } | |||
3366 | ||||
3367 | DEBUG(dbgs() << "INTO: " << *PBI->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "INTO: " << *PBI-> getParent(); } } while (false); | |||
3368 | DEBUG(dbgs() << *PBI->getParent()->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << *PBI->getParent()->getParent (); } } while (false); | |||
3369 | ||||
3370 | // This basic block is probably dead. We know it has at least | |||
3371 | // one fewer predecessor. | |||
3372 | return true; | |||
3373 | } | |||
3374 | ||||
3375 | // Simplifies a terminator by replacing it with a branch to TrueBB if Cond is | |||
3376 | // true or to FalseBB if Cond is false. | |||
3377 | // Takes care of updating the successors and removing the old terminator. | |||
3378 | // Also makes sure not to introduce new successors by assuming that edges to | |||
3379 | // non-successor TrueBBs and FalseBBs aren't reachable. | |||
3380 | static bool SimplifyTerminatorOnSelect(TerminatorInst *OldTerm, Value *Cond, | |||
3381 | BasicBlock *TrueBB, BasicBlock *FalseBB, | |||
3382 | uint32_t TrueWeight, | |||
3383 | uint32_t FalseWeight) { | |||
3384 | // Remove any superfluous successor edges from the CFG. | |||
3385 | // First, figure out which successors to preserve. | |||
3386 | // If TrueBB and FalseBB are equal, only try to preserve one copy of that | |||
3387 | // successor. | |||
3388 | BasicBlock *KeepEdge1 = TrueBB; | |||
3389 | BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr; | |||
3390 | ||||
3391 | // Then remove the rest. | |||
3392 | for (BasicBlock *Succ : OldTerm->successors()) { | |||
3393 | // Make sure only to keep exactly one copy of each edge. | |||
3394 | if (Succ == KeepEdge1) | |||
3395 | KeepEdge1 = nullptr; | |||
3396 | else if (Succ == KeepEdge2) | |||
3397 | KeepEdge2 = nullptr; | |||
3398 | else | |||
3399 | Succ->removePredecessor(OldTerm->getParent(), | |||
3400 | /*DontDeleteUselessPHIs=*/true); | |||
3401 | } | |||
3402 | ||||
3403 | IRBuilder<> Builder(OldTerm); | |||
3404 | Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc()); | |||
3405 | ||||
3406 | // Insert an appropriate new terminator. | |||
3407 | if (!KeepEdge1 && !KeepEdge2) { | |||
3408 | if (TrueBB == FalseBB) | |||
3409 | // We were only looking for one successor, and it was present. | |||
3410 | // Create an unconditional branch to it. | |||
3411 | Builder.CreateBr(TrueBB); | |||
3412 | else { | |||
3413 | // We found both of the successors we were looking for. | |||
3414 | // Create a conditional branch sharing the condition of the select. | |||
3415 | BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB); | |||
3416 | if (TrueWeight != FalseWeight) | |||
3417 | setBranchWeights(NewBI, TrueWeight, FalseWeight); | |||
3418 | } | |||
3419 | } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) { | |||
3420 | // Neither of the selected blocks were successors, so this | |||
3421 | // terminator must be unreachable. | |||
3422 | new UnreachableInst(OldTerm->getContext(), OldTerm); | |||
3423 | } else { | |||
3424 | // One of the selected values was a successor, but the other wasn't. | |||
3425 | // Insert an unconditional branch to the one that was found; | |||
3426 | // the edge to the one that wasn't must be unreachable. | |||
3427 | if (!KeepEdge1) | |||
3428 | // Only TrueBB was found. | |||
3429 | Builder.CreateBr(TrueBB); | |||
3430 | else | |||
3431 | // Only FalseBB was found. | |||
3432 | Builder.CreateBr(FalseBB); | |||
3433 | } | |||
3434 | ||||
3435 | EraseTerminatorInstAndDCECond(OldTerm); | |||
3436 | return true; | |||
3437 | } | |||
3438 | ||||
3439 | // Replaces | |||
3440 | // (switch (select cond, X, Y)) on constant X, Y | |||
3441 | // with a branch - conditional if X and Y lead to distinct BBs, | |||
3442 | // unconditional otherwise. | |||
3443 | static bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select) { | |||
3444 | // Check for constant integer values in the select. | |||
3445 | ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue()); | |||
3446 | ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue()); | |||
3447 | if (!TrueVal || !FalseVal) | |||
3448 | return false; | |||
3449 | ||||
3450 | // Find the relevant condition and destinations. | |||
3451 | Value *Condition = Select->getCondition(); | |||
3452 | BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor(); | |||
3453 | BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor(); | |||
3454 | ||||
3455 | // Get weight for TrueBB and FalseBB. | |||
3456 | uint32_t TrueWeight = 0, FalseWeight = 0; | |||
3457 | SmallVector<uint64_t, 8> Weights; | |||
3458 | bool HasWeights = HasBranchWeights(SI); | |||
3459 | if (HasWeights) { | |||
3460 | GetBranchWeights(SI, Weights); | |||
3461 | if (Weights.size() == 1 + SI->getNumCases()) { | |||
3462 | TrueWeight = | |||
3463 | (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()]; | |||
3464 | FalseWeight = | |||
3465 | (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()]; | |||
3466 | } | |||
3467 | } | |||
3468 | ||||
3469 | // Perform the actual simplification. | |||
3470 | return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight, | |||
3471 | FalseWeight); | |||
3472 | } | |||
3473 | ||||
3474 | // Replaces | |||
3475 | // (indirectbr (select cond, blockaddress(@fn, BlockA), | |||
3476 | // blockaddress(@fn, BlockB))) | |||
3477 | // with | |||
3478 | // (br cond, BlockA, BlockB). | |||
3479 | static bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI) { | |||
3480 | // Check that both operands of the select are block addresses. | |||
3481 | BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue()); | |||
3482 | BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue()); | |||
3483 | if (!TBA || !FBA) | |||
3484 | return false; | |||
3485 | ||||
3486 | // Extract the actual blocks. | |||
3487 | BasicBlock *TrueBB = TBA->getBasicBlock(); | |||
3488 | BasicBlock *FalseBB = FBA->getBasicBlock(); | |||
3489 | ||||
3490 | // Perform the actual simplification. | |||
3491 | return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0, | |||
3492 | 0); | |||
3493 | } | |||
3494 | ||||
3495 | /// This is called when we find an icmp instruction | |||
3496 | /// (a seteq/setne with a constant) as the only instruction in a | |||
3497 | /// block that ends with an uncond branch. We are looking for a very specific | |||
3498 | /// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In | |||
3499 | /// this case, we merge the first two "or's of icmp" into a switch, but then the | |||
3500 | /// default value goes to an uncond block with a seteq in it, we get something | |||
3501 | /// like: | |||
3502 | /// | |||
3503 | /// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ] | |||
3504 | /// DEFAULT: | |||
3505 | /// %tmp = icmp eq i8 %A, 92 | |||
3506 | /// br label %end | |||
3507 | /// end: | |||
3508 | /// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ] | |||
3509 | /// | |||
3510 | /// We prefer to split the edge to 'end' so that there is a true/false entry to | |||
3511 | /// the PHI, merging the third icmp into the switch. | |||
3512 | static bool tryToSimplifyUncondBranchWithICmpInIt( | |||
3513 | ICmpInst *ICI, IRBuilder<> &Builder, const DataLayout &DL, | |||
3514 | const TargetTransformInfo &TTI, const SimplifyCFGOptions &Options) { | |||
3515 | BasicBlock *BB = ICI->getParent(); | |||
3516 | ||||
3517 | // If the block has any PHIs in it or the icmp has multiple uses, it is too | |||
3518 | // complex. | |||
3519 | if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse()) | |||
| ||||
3520 | return false; | |||
3521 | ||||
3522 | Value *V = ICI->getOperand(0); | |||
3523 | ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1)); | |||
3524 | ||||
3525 | // The pattern we're looking for is where our only predecessor is a switch on | |||
3526 | // 'V' and this block is the default case for the switch. In this case we can | |||
3527 | // fold the compared value into the switch to simplify things. | |||
3528 | BasicBlock *Pred = BB->getSinglePredecessor(); | |||
3529 | if (!Pred || !isa<SwitchInst>(Pred->getTerminator())) | |||
3530 | return false; | |||
3531 | ||||
3532 | SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator()); | |||
3533 | if (SI->getCondition() != V) | |||
3534 | return false; | |||
3535 | ||||
3536 | // If BB is reachable on a non-default case, then we simply know the value of | |||
3537 | // V in this block. Substitute it and constant fold the icmp instruction | |||
3538 | // away. | |||
3539 | if (SI->getDefaultDest() != BB) { | |||
3540 | ConstantInt *VVal = SI->findCaseDest(BB); | |||
3541 | assert(VVal && "Should have a unique destination value")(static_cast <bool> (VVal && "Should have a unique destination value" ) ? void (0) : __assert_fail ("VVal && \"Should have a unique destination value\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 3541, __extension__ __PRETTY_FUNCTION__)); | |||
3542 | ICI->setOperand(0, VVal); | |||
3543 | ||||
3544 | if (Value *V = SimplifyInstruction(ICI, {DL, ICI})) { | |||
3545 | ICI->replaceAllUsesWith(V); | |||
3546 | ICI->eraseFromParent(); | |||
3547 | } | |||
3548 | // BB is now empty, so it is likely to simplify away. | |||
3549 | return simplifyCFG(BB, TTI, Options) | true; | |||
3550 | } | |||
3551 | ||||
3552 | // Ok, the block is reachable from the default dest. If the constant we're | |||
3553 | // comparing exists in one of the other edges, then we can constant fold ICI | |||
3554 | // and zap it. | |||
3555 | if (SI->findCaseValue(Cst) != SI->case_default()) { | |||
3556 | Value *V; | |||
3557 | if (ICI->getPredicate() == ICmpInst::ICMP_EQ) | |||
3558 | V = ConstantInt::getFalse(BB->getContext()); | |||
3559 | else | |||
3560 | V = ConstantInt::getTrue(BB->getContext()); | |||
3561 | ||||
3562 | ICI->replaceAllUsesWith(V); | |||
3563 | ICI->eraseFromParent(); | |||
3564 | // BB is now empty, so it is likely to simplify away. | |||
3565 | return simplifyCFG(BB, TTI, Options) | true; | |||
3566 | } | |||
3567 | ||||
3568 | // The use of the icmp has to be in the 'end' block, by the only PHI node in | |||
3569 | // the block. | |||
3570 | BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0); | |||
3571 | PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back()); | |||
3572 | if (PHIUse == nullptr || PHIUse != &SuccBlock->front() || | |||
3573 | isa<PHINode>(++BasicBlock::iterator(PHIUse))) | |||
3574 | return false; | |||
3575 | ||||
3576 | // If the icmp is a SETEQ, then the default dest gets false, the new edge gets | |||
3577 | // true in the PHI. | |||
3578 | Constant *DefaultCst = ConstantInt::getTrue(BB->getContext()); | |||
3579 | Constant *NewCst = ConstantInt::getFalse(BB->getContext()); | |||
3580 | ||||
3581 | if (ICI->getPredicate() == ICmpInst::ICMP_EQ) | |||
3582 | std::swap(DefaultCst, NewCst); | |||
3583 | ||||
3584 | // Replace ICI (which is used by the PHI for the default value) with true or | |||
3585 | // false depending on if it is EQ or NE. | |||
3586 | ICI->replaceAllUsesWith(DefaultCst); | |||
3587 | ICI->eraseFromParent(); | |||
3588 | ||||
3589 | // Okay, the switch goes to this block on a default value. Add an edge from | |||
3590 | // the switch to the merge point on the compared value. | |||
3591 | BasicBlock *NewBB = | |||
3592 | BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB); | |||
3593 | SmallVector<uint64_t, 8> Weights; | |||
3594 | bool HasWeights = HasBranchWeights(SI); | |||
3595 | if (HasWeights) { | |||
3596 | GetBranchWeights(SI, Weights); | |||
3597 | if (Weights.size() == 1 + SI->getNumCases()) { | |||
3598 | // Split weight for default case to case for "Cst". | |||
3599 | Weights[0] = (Weights[0] + 1) >> 1; | |||
3600 | Weights.push_back(Weights[0]); | |||
3601 | ||||
3602 | SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end()); | |||
3603 | setBranchWeights(SI, MDWeights); | |||
3604 | } | |||
3605 | } | |||
3606 | SI->addCase(Cst, NewBB); | |||
3607 | ||||
3608 | // NewBB branches to the phi block, add the uncond branch and the phi entry. | |||
3609 | Builder.SetInsertPoint(NewBB); | |||
3610 | Builder.SetCurrentDebugLocation(SI->getDebugLoc()); | |||
3611 | Builder.CreateBr(SuccBlock); | |||
3612 | PHIUse->addIncoming(NewCst, NewBB); | |||
3613 | return true; | |||
3614 | } | |||
3615 | ||||
3616 | /// The specified branch is a conditional branch. | |||
3617 | /// Check to see if it is branching on an or/and chain of icmp instructions, and | |||
3618 | /// fold it into a switch instruction if so. | |||
3619 | static bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder, | |||
3620 | const DataLayout &DL) { | |||
3621 | Instruction *Cond = dyn_cast<Instruction>(BI->getCondition()); | |||
3622 | if (!Cond) | |||
3623 | return false; | |||
3624 | ||||
3625 | // Change br (X == 0 | X == 1), T, F into a switch instruction. | |||
3626 | // If this is a bunch of seteq's or'd together, or if it's a bunch of | |||
3627 | // 'setne's and'ed together, collect them. | |||
3628 | ||||
3629 | // Try to gather values from a chain of and/or to be turned into a switch | |||
3630 | ConstantComparesGatherer ConstantCompare(Cond, DL); | |||
3631 | // Unpack the result | |||
3632 | SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals; | |||
3633 | Value *CompVal = ConstantCompare.CompValue; | |||
3634 | unsigned UsedICmps = ConstantCompare.UsedICmps; | |||
3635 | Value *ExtraCase = ConstantCompare.Extra; | |||
3636 | ||||
3637 | // If we didn't have a multiply compared value, fail. | |||
3638 | if (!CompVal) | |||
3639 | return false; | |||
3640 | ||||
3641 | // Avoid turning single icmps into a switch. | |||
3642 | if (UsedICmps <= 1) | |||
3643 | return false; | |||
3644 | ||||
3645 | bool TrueWhenEqual = (Cond->getOpcode() == Instruction::Or); | |||
3646 | ||||
3647 | // There might be duplicate constants in the list, which the switch | |||
3648 | // instruction can't handle, remove them now. | |||
3649 | array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate); | |||
3650 | Values.erase(std::unique(Values.begin(), Values.end()), Values.end()); | |||
3651 | ||||
3652 | // If Extra was used, we require at least two switch values to do the | |||
3653 | // transformation. A switch with one value is just a conditional branch. | |||
3654 | if (ExtraCase && Values.size() < 2) | |||
3655 | return false; | |||
3656 | ||||
3657 | // TODO: Preserve branch weight metadata, similarly to how | |||
3658 | // FoldValueComparisonIntoPredecessors preserves it. | |||
3659 | ||||
3660 | // Figure out which block is which destination. | |||
3661 | BasicBlock *DefaultBB = BI->getSuccessor(1); | |||
3662 | BasicBlock *EdgeBB = BI->getSuccessor(0); | |||
3663 | if (!TrueWhenEqual) | |||
3664 | std::swap(DefaultBB, EdgeBB); | |||
3665 | ||||
3666 | BasicBlock *BB = BI->getParent(); | |||
3667 | ||||
3668 | DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Converting 'icmp' chain with " << Values.size() << " cases into SWITCH. BB is:\n" << *BB; } } while (false) | |||
3669 | << " cases into SWITCH. BB is:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Converting 'icmp' chain with " << Values.size() << " cases into SWITCH. BB is:\n" << *BB; } } while (false) | |||
3670 | << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Converting 'icmp' chain with " << Values.size() << " cases into SWITCH. BB is:\n" << *BB; } } while (false); | |||
3671 | ||||
3672 | // If there are any extra values that couldn't be folded into the switch | |||
3673 | // then we evaluate them with an explicit branch first. Split the block | |||
3674 | // right before the condbr to handle it. | |||
3675 | if (ExtraCase) { | |||
3676 | BasicBlock *NewBB = | |||
3677 | BB->splitBasicBlock(BI->getIterator(), "switch.early.test"); | |||
3678 | // Remove the uncond branch added to the old block. | |||
3679 | TerminatorInst *OldTI = BB->getTerminator(); | |||
3680 | Builder.SetInsertPoint(OldTI); | |||
3681 | ||||
3682 | if (TrueWhenEqual) | |||
3683 | Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB); | |||
3684 | else | |||
3685 | Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB); | |||
3686 | ||||
3687 | OldTI->eraseFromParent(); | |||
3688 | ||||
3689 | // If there are PHI nodes in EdgeBB, then we need to add a new entry to them | |||
3690 | // for the edge we just added. | |||
3691 | AddPredecessorToBlock(EdgeBB, BB, NewBB); | |||
3692 | ||||
3693 | DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCasedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCase << "\nEXTRABB = " << *BB; } } while (false) | |||
3694 | << "\nEXTRABB = " << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCase << "\nEXTRABB = " << *BB; } } while (false); | |||
3695 | BB = NewBB; | |||
3696 | } | |||
3697 | ||||
3698 | Builder.SetInsertPoint(BI); | |||
3699 | // Convert pointer to int before we switch. | |||
3700 | if (CompVal->getType()->isPointerTy()) { | |||
3701 | CompVal = Builder.CreatePtrToInt( | |||
3702 | CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr"); | |||
3703 | } | |||
3704 | ||||
3705 | // Create the new switch instruction now. | |||
3706 | SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size()); | |||
3707 | ||||
3708 | // Add all of the 'cases' to the switch instruction. | |||
3709 | for (unsigned i = 0, e = Values.size(); i != e; ++i) | |||
3710 | New->addCase(Values[i], EdgeBB); | |||
3711 | ||||
3712 | // We added edges from PI to the EdgeBB. As such, if there were any | |||
3713 | // PHI nodes in EdgeBB, they need entries to be added corresponding to | |||
3714 | // the number of edges added. | |||
3715 | for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) { | |||
3716 | PHINode *PN = cast<PHINode>(BBI); | |||
3717 | Value *InVal = PN->getIncomingValueForBlock(BB); | |||
3718 | for (unsigned i = 0, e = Values.size() - 1; i != e; ++i) | |||
3719 | PN->addIncoming(InVal, BB); | |||
3720 | } | |||
3721 | ||||
3722 | // Erase the old branch instruction. | |||
3723 | EraseTerminatorInstAndDCECond(BI); | |||
3724 | ||||
3725 | DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n'; } } while (false); | |||
3726 | return true; | |||
3727 | } | |||
3728 | ||||
3729 | bool SimplifyCFGOpt::SimplifyResume(ResumeInst *RI, IRBuilder<> &Builder) { | |||
3730 | if (isa<PHINode>(RI->getValue())) | |||
3731 | return SimplifyCommonResume(RI); | |||
3732 | else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) && | |||
3733 | RI->getValue() == RI->getParent()->getFirstNonPHI()) | |||
3734 | // The resume must unwind the exception that caused control to branch here. | |||
3735 | return SimplifySingleResume(RI); | |||
3736 | ||||
3737 | return false; | |||
3738 | } | |||
3739 | ||||
3740 | // Simplify resume that is shared by several landing pads (phi of landing pad). | |||
3741 | bool SimplifyCFGOpt::SimplifyCommonResume(ResumeInst *RI) { | |||
3742 | BasicBlock *BB = RI->getParent(); | |||
3743 | ||||
3744 | // Check that there are no other instructions except for debug intrinsics | |||
3745 | // between the phi of landing pads (RI->getValue()) and resume instruction. | |||
3746 | BasicBlock::iterator I = cast<Instruction>(RI->getValue())->getIterator(), | |||
3747 | E = RI->getIterator(); | |||
3748 | while (++I != E) | |||
3749 | if (!isa<DbgInfoIntrinsic>(I)) | |||
3750 | return false; | |||
3751 | ||||
3752 | SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks; | |||
3753 | auto *PhiLPInst = cast<PHINode>(RI->getValue()); | |||
3754 | ||||
3755 | // Check incoming blocks to see if any of them are trivial. | |||
3756 | for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End; | |||
3757 | Idx++) { | |||
3758 | auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx); | |||
3759 | auto *IncomingValue = PhiLPInst->getIncomingValue(Idx); | |||
3760 | ||||
3761 | // If the block has other successors, we can not delete it because | |||
3762 | // it has other dependents. | |||
3763 | if (IncomingBB->getUniqueSuccessor() != BB) | |||
3764 | continue; | |||
3765 | ||||
3766 | auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI()); | |||
3767 | // Not the landing pad that caused the control to branch here. | |||
3768 | if (IncomingValue != LandingPad) | |||
3769 | continue; | |||
3770 | ||||
3771 | bool isTrivial = true; | |||
3772 | ||||
3773 | I = IncomingBB->getFirstNonPHI()->getIterator(); | |||
3774 | E = IncomingBB->getTerminator()->getIterator(); | |||
3775 | while (++I != E) | |||
3776 | if (!isa<DbgInfoIntrinsic>(I)) { | |||
3777 | isTrivial = false; | |||
3778 | break; | |||
3779 | } | |||
3780 | ||||
3781 | if (isTrivial) | |||
3782 | TrivialUnwindBlocks.insert(IncomingBB); | |||
3783 | } | |||
3784 | ||||
3785 | // If no trivial unwind blocks, don't do any simplifications. | |||
3786 | if (TrivialUnwindBlocks.empty()) | |||
3787 | return false; | |||
3788 | ||||
3789 | // Turn all invokes that unwind here into calls. | |||
3790 | for (auto *TrivialBB : TrivialUnwindBlocks) { | |||
3791 | // Blocks that will be simplified should be removed from the phi node. | |||
3792 | // Note there could be multiple edges to the resume block, and we need | |||
3793 | // to remove them all. | |||
3794 | while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1) | |||
3795 | BB->removePredecessor(TrivialBB, true); | |||
3796 | ||||
3797 | for (pred_iterator PI = pred_begin(TrivialBB), PE = pred_end(TrivialBB); | |||
3798 | PI != PE;) { | |||
3799 | BasicBlock *Pred = *PI++; | |||
3800 | removeUnwindEdge(Pred); | |||
3801 | } | |||
3802 | ||||
3803 | // In each SimplifyCFG run, only the current processed block can be erased. | |||
3804 | // Otherwise, it will break the iteration of SimplifyCFG pass. So instead | |||
3805 | // of erasing TrivialBB, we only remove the branch to the common resume | |||
3806 | // block so that we can later erase the resume block since it has no | |||
3807 | // predecessors. | |||
3808 | TrivialBB->getTerminator()->eraseFromParent(); | |||
3809 | new UnreachableInst(RI->getContext(), TrivialBB); | |||
3810 | } | |||
3811 | ||||
3812 | // Delete the resume block if all its predecessors have been removed. | |||
3813 | if (pred_empty(BB)) | |||
3814 | BB->eraseFromParent(); | |||
3815 | ||||
3816 | return !TrivialUnwindBlocks.empty(); | |||
3817 | } | |||
3818 | ||||
3819 | // Simplify resume that is only used by a single (non-phi) landing pad. | |||
3820 | bool SimplifyCFGOpt::SimplifySingleResume(ResumeInst *RI) { | |||
3821 | BasicBlock *BB = RI->getParent(); | |||
3822 | LandingPadInst *LPInst = dyn_cast<LandingPadInst>(BB->getFirstNonPHI()); | |||
3823 | assert(RI->getValue() == LPInst &&(static_cast <bool> (RI->getValue() == LPInst && "Resume must unwind the exception that caused control to here" ) ? void (0) : __assert_fail ("RI->getValue() == LPInst && \"Resume must unwind the exception that caused control to here\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 3824, __extension__ __PRETTY_FUNCTION__)) | |||
3824 | "Resume must unwind the exception that caused control to here")(static_cast <bool> (RI->getValue() == LPInst && "Resume must unwind the exception that caused control to here" ) ? void (0) : __assert_fail ("RI->getValue() == LPInst && \"Resume must unwind the exception that caused control to here\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 3824, __extension__ __PRETTY_FUNCTION__)); | |||
3825 | ||||
3826 | // Check that there are no other instructions except for debug intrinsics. | |||
3827 | BasicBlock::iterator I = LPInst->getIterator(), E = RI->getIterator(); | |||
3828 | while (++I != E) | |||
3829 | if (!isa<DbgInfoIntrinsic>(I)) | |||
3830 | return false; | |||
3831 | ||||
3832 | // Turn all invokes that unwind here into calls and delete the basic block. | |||
3833 | for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE;) { | |||
3834 | BasicBlock *Pred = *PI++; | |||
3835 | removeUnwindEdge(Pred); | |||
3836 | } | |||
3837 | ||||
3838 | // The landingpad is now unreachable. Zap it. | |||
3839 | BB->eraseFromParent(); | |||
3840 | if (LoopHeaders) | |||
3841 | LoopHeaders->erase(BB); | |||
3842 | return true; | |||
3843 | } | |||
3844 | ||||
3845 | static bool removeEmptyCleanup(CleanupReturnInst *RI) { | |||
3846 | // If this is a trivial cleanup pad that executes no instructions, it can be | |||
3847 | // eliminated. If the cleanup pad continues to the caller, any predecessor | |||
3848 | // that is an EH pad will be updated to continue to the caller and any | |||
3849 | // predecessor that terminates with an invoke instruction will have its invoke | |||
3850 | // instruction converted to a call instruction. If the cleanup pad being | |||
3851 | // simplified does not continue to the caller, each predecessor will be | |||
3852 | // updated to continue to the unwind destination of the cleanup pad being | |||
3853 | // simplified. | |||
3854 | BasicBlock *BB = RI->getParent(); | |||
3855 | CleanupPadInst *CPInst = RI->getCleanupPad(); | |||
3856 | if (CPInst->getParent() != BB) | |||
3857 | // This isn't an empty cleanup. | |||
3858 | return false; | |||
3859 | ||||
3860 | // We cannot kill the pad if it has multiple uses. This typically arises | |||
3861 | // from unreachable basic blocks. | |||
3862 | if (!CPInst->hasOneUse()) | |||
3863 | return false; | |||
3864 | ||||
3865 | // Check that there are no other instructions except for benign intrinsics. | |||
3866 | BasicBlock::iterator I = CPInst->getIterator(), E = RI->getIterator(); | |||
3867 | while (++I != E) { | |||
3868 | auto *II = dyn_cast<IntrinsicInst>(I); | |||
3869 | if (!II) | |||
3870 | return false; | |||
3871 | ||||
3872 | Intrinsic::ID IntrinsicID = II->getIntrinsicID(); | |||
3873 | switch (IntrinsicID) { | |||
3874 | case Intrinsic::dbg_declare: | |||
3875 | case Intrinsic::dbg_value: | |||
3876 | case Intrinsic::lifetime_end: | |||
3877 | break; | |||
3878 | default: | |||
3879 | return false; | |||
3880 | } | |||
3881 | } | |||
3882 | ||||
3883 | // If the cleanup return we are simplifying unwinds to the caller, this will | |||
3884 | // set UnwindDest to nullptr. | |||
3885 | BasicBlock *UnwindDest = RI->getUnwindDest(); | |||
3886 | Instruction *DestEHPad = UnwindDest ? UnwindDest->getFirstNonPHI() : nullptr; | |||
3887 | ||||
3888 | // We're about to remove BB from the control flow. Before we do, sink any | |||
3889 | // PHINodes into the unwind destination. Doing this before changing the | |||
3890 | // control flow avoids some potentially slow checks, since we can currently | |||
3891 | // be certain that UnwindDest and BB have no common predecessors (since they | |||
3892 | // are both EH pads). | |||
3893 | if (UnwindDest) { | |||
3894 | // First, go through the PHI nodes in UnwindDest and update any nodes that | |||
3895 | // reference the block we are removing | |||
3896 | for (BasicBlock::iterator I = UnwindDest->begin(), | |||
3897 | IE = DestEHPad->getIterator(); | |||
3898 | I != IE; ++I) { | |||
3899 | PHINode *DestPN = cast<PHINode>(I); | |||
3900 | ||||
3901 | int Idx = DestPN->getBasicBlockIndex(BB); | |||
3902 | // Since BB unwinds to UnwindDest, it has to be in the PHI node. | |||
3903 | assert(Idx != -1)(static_cast <bool> (Idx != -1) ? void (0) : __assert_fail ("Idx != -1", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 3903, __extension__ __PRETTY_FUNCTION__)); | |||
3904 | // This PHI node has an incoming value that corresponds to a control | |||
3905 | // path through the cleanup pad we are removing. If the incoming | |||
3906 | // value is in the cleanup pad, it must be a PHINode (because we | |||
3907 | // verified above that the block is otherwise empty). Otherwise, the | |||
3908 | // value is either a constant or a value that dominates the cleanup | |||
3909 | // pad being removed. | |||
3910 | // | |||
3911 | // Because BB and UnwindDest are both EH pads, all of their | |||
3912 | // predecessors must unwind to these blocks, and since no instruction | |||
3913 | // can have multiple unwind destinations, there will be no overlap in | |||
3914 | // incoming blocks between SrcPN and DestPN. | |||
3915 | Value *SrcVal = DestPN->getIncomingValue(Idx); | |||
3916 | PHINode *SrcPN = dyn_cast<PHINode>(SrcVal); | |||
3917 | ||||
3918 | // Remove the entry for the block we are deleting. | |||
3919 | DestPN->removeIncomingValue(Idx, false); | |||
3920 | ||||
3921 | if (SrcPN && SrcPN->getParent() == BB) { | |||
3922 | // If the incoming value was a PHI node in the cleanup pad we are | |||
3923 | // removing, we need to merge that PHI node's incoming values into | |||
3924 | // DestPN. | |||
3925 | for (unsigned SrcIdx = 0, SrcE = SrcPN->getNumIncomingValues(); | |||
3926 | SrcIdx != SrcE; ++SrcIdx) { | |||
3927 | DestPN->addIncoming(SrcPN->getIncomingValue(SrcIdx), | |||
3928 | SrcPN->getIncomingBlock(SrcIdx)); | |||
3929 | } | |||
3930 | } else { | |||
3931 | // Otherwise, the incoming value came from above BB and | |||
3932 | // so we can just reuse it. We must associate all of BB's | |||
3933 | // predecessors with this value. | |||
3934 | for (auto *pred : predecessors(BB)) { | |||
3935 | DestPN->addIncoming(SrcVal, pred); | |||
3936 | } | |||
3937 | } | |||
3938 | } | |||
3939 | ||||
3940 | // Sink any remaining PHI nodes directly into UnwindDest. | |||
3941 | Instruction *InsertPt = DestEHPad; | |||
3942 | for (BasicBlock::iterator I = BB->begin(), | |||
3943 | IE = BB->getFirstNonPHI()->getIterator(); | |||
3944 | I != IE;) { | |||
3945 | // The iterator must be incremented here because the instructions are | |||
3946 | // being moved to another block. | |||
3947 | PHINode *PN = cast<PHINode>(I++); | |||
3948 | if (PN->use_empty()) | |||
3949 | // If the PHI node has no uses, just leave it. It will be erased | |||
3950 | // when we erase BB below. | |||
3951 | continue; | |||
3952 | ||||
3953 | // Otherwise, sink this PHI node into UnwindDest. | |||
3954 | // Any predecessors to UnwindDest which are not already represented | |||
3955 | // must be back edges which inherit the value from the path through | |||
3956 | // BB. In this case, the PHI value must reference itself. | |||
3957 | for (auto *pred : predecessors(UnwindDest)) | |||
3958 | if (pred != BB) | |||
3959 | PN->addIncoming(PN, pred); | |||
3960 | PN->moveBefore(InsertPt); | |||
3961 | } | |||
3962 | } | |||
3963 | ||||
3964 | for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE;) { | |||
3965 | // The iterator must be updated here because we are removing this pred. | |||
3966 | BasicBlock *PredBB = *PI++; | |||
3967 | if (UnwindDest == nullptr) { | |||
3968 | removeUnwindEdge(PredBB); | |||
3969 | } else { | |||
3970 | TerminatorInst *TI = PredBB->getTerminator(); | |||
3971 | TI->replaceUsesOfWith(BB, UnwindDest); | |||
3972 | } | |||
3973 | } | |||
3974 | ||||
3975 | // The cleanup pad is now unreachable. Zap it. | |||
3976 | BB->eraseFromParent(); | |||
3977 | return true; | |||
3978 | } | |||
3979 | ||||
3980 | // Try to merge two cleanuppads together. | |||
3981 | static bool mergeCleanupPad(CleanupReturnInst *RI) { | |||
3982 | // Skip any cleanuprets which unwind to caller, there is nothing to merge | |||
3983 | // with. | |||
3984 | BasicBlock *UnwindDest = RI->getUnwindDest(); | |||
3985 | if (!UnwindDest) | |||
3986 | return false; | |||
3987 | ||||
3988 | // This cleanupret isn't the only predecessor of this cleanuppad, it wouldn't | |||
3989 | // be safe to merge without code duplication. | |||
3990 | if (UnwindDest->getSinglePredecessor() != RI->getParent()) | |||
3991 | return false; | |||
3992 | ||||
3993 | // Verify that our cleanuppad's unwind destination is another cleanuppad. | |||
3994 | auto *SuccessorCleanupPad = dyn_cast<CleanupPadInst>(&UnwindDest->front()); | |||
3995 | if (!SuccessorCleanupPad) | |||
3996 | return false; | |||
3997 | ||||
3998 | CleanupPadInst *PredecessorCleanupPad = RI->getCleanupPad(); | |||
3999 | // Replace any uses of the successor cleanupad with the predecessor pad | |||
4000 | // The only cleanuppad uses should be this cleanupret, it's cleanupret and | |||
4001 | // funclet bundle operands. | |||
4002 | SuccessorCleanupPad->replaceAllUsesWith(PredecessorCleanupPad); | |||
4003 | // Remove the old cleanuppad. | |||
4004 | SuccessorCleanupPad->eraseFromParent(); | |||
4005 | // Now, we simply replace the cleanupret with a branch to the unwind | |||
4006 | // destination. | |||
4007 | BranchInst::Create(UnwindDest, RI->getParent()); | |||
4008 | RI->eraseFromParent(); | |||
4009 | ||||
4010 | return true; | |||
4011 | } | |||
4012 | ||||
4013 | bool SimplifyCFGOpt::SimplifyCleanupReturn(CleanupReturnInst *RI) { | |||
4014 | // It is possible to transiantly have an undef cleanuppad operand because we | |||
4015 | // have deleted some, but not all, dead blocks. | |||
4016 | // Eventually, this block will be deleted. | |||
4017 | if (isa<UndefValue>(RI->getOperand(0))) | |||
4018 | return false; | |||
4019 | ||||
4020 | if (mergeCleanupPad(RI)) | |||
4021 | return true; | |||
4022 | ||||
4023 | if (removeEmptyCleanup(RI)) | |||
4024 | return true; | |||
4025 | ||||
4026 | return false; | |||
4027 | } | |||
4028 | ||||
4029 | bool SimplifyCFGOpt::SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) { | |||
4030 | BasicBlock *BB = RI->getParent(); | |||
4031 | if (!BB->getFirstNonPHIOrDbg()->isTerminator()) | |||
4032 | return false; | |||
4033 | ||||
4034 | // Find predecessors that end with branches. | |||
4035 | SmallVector<BasicBlock *, 8> UncondBranchPreds; | |||
4036 | SmallVector<BranchInst *, 8> CondBranchPreds; | |||
4037 | for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) { | |||
4038 | BasicBlock *P = *PI; | |||
4039 | TerminatorInst *PTI = P->getTerminator(); | |||
4040 | if (BranchInst *BI = dyn_cast<BranchInst>(PTI)) { | |||
4041 | if (BI->isUnconditional()) | |||
4042 | UncondBranchPreds.push_back(P); | |||
4043 | else | |||
4044 | CondBranchPreds.push_back(BI); | |||
4045 | } | |||
4046 | } | |||
4047 | ||||
4048 | // If we found some, do the transformation! | |||
4049 | if (!UncondBranchPreds.empty() && DupRet) { | |||
4050 | while (!UncondBranchPreds.empty()) { | |||
4051 | BasicBlock *Pred = UncondBranchPreds.pop_back_val(); | |||
4052 | DEBUG(dbgs() << "FOLDING: " << *BBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "FOLDING: " << *BB << "INTO UNCOND BRANCH PRED: " << *Pred; } } while (false ) | |||
4053 | << "INTO UNCOND BRANCH PRED: " << *Pred)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "FOLDING: " << *BB << "INTO UNCOND BRANCH PRED: " << *Pred; } } while (false ); | |||
4054 | (void)FoldReturnIntoUncondBranch(RI, BB, Pred); | |||
4055 | } | |||
4056 | ||||
4057 | // If we eliminated all predecessors of the block, delete the block now. | |||
4058 | if (pred_empty(BB)) { | |||
4059 | // We know there are no successors, so just nuke the block. | |||
4060 | BB->eraseFromParent(); | |||
4061 | if (LoopHeaders) | |||
4062 | LoopHeaders->erase(BB); | |||
4063 | } | |||
4064 | ||||
4065 | return true; | |||
4066 | } | |||
4067 | ||||
4068 | // Check out all of the conditional branches going to this return | |||
4069 | // instruction. If any of them just select between returns, change the | |||
4070 | // branch itself into a select/return pair. | |||
4071 | while (!CondBranchPreds.empty()) { | |||
4072 | BranchInst *BI = CondBranchPreds.pop_back_val(); | |||
4073 | ||||
4074 | // Check to see if the non-BB successor is also a return block. | |||
4075 | if (isa<ReturnInst>(BI->getSuccessor(0)->getTerminator()) && | |||
4076 | isa<ReturnInst>(BI->getSuccessor(1)->getTerminator()) && | |||
4077 | SimplifyCondBranchToTwoReturns(BI, Builder)) | |||
4078 | return true; | |||
4079 | } | |||
4080 | return false; | |||
4081 | } | |||
4082 | ||||
4083 | bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) { | |||
4084 | BasicBlock *BB = UI->getParent(); | |||
4085 | ||||
4086 | bool Changed = false; | |||
4087 | ||||
4088 | // If there are any instructions immediately before the unreachable that can | |||
4089 | // be removed, do so. | |||
4090 | while (UI->getIterator() != BB->begin()) { | |||
4091 | BasicBlock::iterator BBI = UI->getIterator(); | |||
4092 | --BBI; | |||
4093 | // Do not delete instructions that can have side effects which might cause | |||
4094 | // the unreachable to not be reachable; specifically, calls and volatile | |||
4095 | // operations may have this effect. | |||
4096 | if (isa<CallInst>(BBI) && !isa<DbgInfoIntrinsic>(BBI)) | |||
4097 | break; | |||
4098 | ||||
4099 | if (BBI->mayHaveSideEffects()) { | |||
4100 | if (auto *SI = dyn_cast<StoreInst>(BBI)) { | |||
4101 | if (SI->isVolatile()) | |||
4102 | break; | |||
4103 | } else if (auto *LI = dyn_cast<LoadInst>(BBI)) { | |||
4104 | if (LI->isVolatile()) | |||
4105 | break; | |||
4106 | } else if (auto *RMWI = dyn_cast<AtomicRMWInst>(BBI)) { | |||
4107 | if (RMWI->isVolatile()) | |||
4108 | break; | |||
4109 | } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(BBI)) { | |||
4110 | if (CXI->isVolatile()) | |||
4111 | break; | |||
4112 | } else if (isa<CatchPadInst>(BBI)) { | |||
4113 | // A catchpad may invoke exception object constructors and such, which | |||
4114 | // in some languages can be arbitrary code, so be conservative by | |||
4115 | // default. | |||
4116 | // For CoreCLR, it just involves a type test, so can be removed. | |||
4117 | if (classifyEHPersonality(BB->getParent()->getPersonalityFn()) != | |||
4118 | EHPersonality::CoreCLR) | |||
4119 | break; | |||
4120 | } else if (!isa<FenceInst>(BBI) && !isa<VAArgInst>(BBI) && | |||
4121 | !isa<LandingPadInst>(BBI)) { | |||
4122 | break; | |||
4123 | } | |||
4124 | // Note that deleting LandingPad's here is in fact okay, although it | |||
4125 | // involves a bit of subtle reasoning. If this inst is a LandingPad, | |||
4126 | // all the predecessors of this block will be the unwind edges of Invokes, | |||
4127 | // and we can therefore guarantee this block will be erased. | |||
4128 | } | |||
4129 | ||||
4130 | // Delete this instruction (any uses are guaranteed to be dead) | |||
4131 | if (!BBI->use_empty()) | |||
4132 | BBI->replaceAllUsesWith(UndefValue::get(BBI->getType())); | |||
4133 | BBI->eraseFromParent(); | |||
4134 | Changed = true; | |||
4135 | } | |||
4136 | ||||
4137 | // If the unreachable instruction is the first in the block, take a gander | |||
4138 | // at all of the predecessors of this instruction, and simplify them. | |||
4139 | if (&BB->front() != UI) | |||
4140 | return Changed; | |||
4141 | ||||
4142 | SmallVector<BasicBlock *, 8> Preds(pred_begin(BB), pred_end(BB)); | |||
4143 | for (unsigned i = 0, e = Preds.size(); i != e; ++i) { | |||
4144 | TerminatorInst *TI = Preds[i]->getTerminator(); | |||
4145 | IRBuilder<> Builder(TI); | |||
4146 | if (auto *BI = dyn_cast<BranchInst>(TI)) { | |||
4147 | if (BI->isUnconditional()) { | |||
4148 | if (BI->getSuccessor(0) == BB) { | |||
4149 | new UnreachableInst(TI->getContext(), TI); | |||
4150 | TI->eraseFromParent(); | |||
4151 | Changed = true; | |||
4152 | } | |||
4153 | } else { | |||
4154 | if (BI->getSuccessor(0) == BB) { | |||
4155 | Builder.CreateBr(BI->getSuccessor(1)); | |||
4156 | EraseTerminatorInstAndDCECond(BI); | |||
4157 | } else if (BI->getSuccessor(1) == BB) { | |||
4158 | Builder.CreateBr(BI->getSuccessor(0)); | |||
4159 | EraseTerminatorInstAndDCECond(BI); | |||
4160 | Changed = true; | |||
4161 | } | |||
4162 | } | |||
4163 | } else if (auto *SI = dyn_cast<SwitchInst>(TI)) { | |||
4164 | for (auto i = SI->case_begin(), e = SI->case_end(); i != e;) { | |||
4165 | if (i->getCaseSuccessor() != BB) { | |||
4166 | ++i; | |||
4167 | continue; | |||
4168 | } | |||
4169 | BB->removePredecessor(SI->getParent()); | |||
4170 | i = SI->removeCase(i); | |||
4171 | e = SI->case_end(); | |||
4172 | Changed = true; | |||
4173 | } | |||
4174 | } else if (auto *II = dyn_cast<InvokeInst>(TI)) { | |||
4175 | if (II->getUnwindDest() == BB) { | |||
4176 | removeUnwindEdge(TI->getParent()); | |||
4177 | Changed = true; | |||
4178 | } | |||
4179 | } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) { | |||
4180 | if (CSI->getUnwindDest() == BB) { | |||
4181 | removeUnwindEdge(TI->getParent()); | |||
4182 | Changed = true; | |||
4183 | continue; | |||
4184 | } | |||
4185 | ||||
4186 | for (CatchSwitchInst::handler_iterator I = CSI->handler_begin(), | |||
4187 | E = CSI->handler_end(); | |||
4188 | I != E; ++I) { | |||
4189 | if (*I == BB) { | |||
4190 | CSI->removeHandler(I); | |||
4191 | --I; | |||
4192 | --E; | |||
4193 | Changed = true; | |||
4194 | } | |||
4195 | } | |||
4196 | if (CSI->getNumHandlers() == 0) { | |||
4197 | BasicBlock *CatchSwitchBB = CSI->getParent(); | |||
4198 | if (CSI->hasUnwindDest()) { | |||
4199 | // Redirect preds to the unwind dest | |||
4200 | CatchSwitchBB->replaceAllUsesWith(CSI->getUnwindDest()); | |||
4201 | } else { | |||
4202 | // Rewrite all preds to unwind to caller (or from invoke to call). | |||
4203 | SmallVector<BasicBlock *, 8> EHPreds(predecessors(CatchSwitchBB)); | |||
4204 | for (BasicBlock *EHPred : EHPreds) | |||
4205 | removeUnwindEdge(EHPred); | |||
4206 | } | |||
4207 | // The catchswitch is no longer reachable. | |||
4208 | new UnreachableInst(CSI->getContext(), CSI); | |||
4209 | CSI->eraseFromParent(); | |||
4210 | Changed = true; | |||
4211 | } | |||
4212 | } else if (isa<CleanupReturnInst>(TI)) { | |||
4213 | new UnreachableInst(TI->getContext(), TI); | |||
4214 | TI->eraseFromParent(); | |||
4215 | Changed = true; | |||
4216 | } | |||
4217 | } | |||
4218 | ||||
4219 | // If this block is now dead, remove it. | |||
4220 | if (pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) { | |||
4221 | // We know there are no successors, so just nuke the block. | |||
4222 | BB->eraseFromParent(); | |||
4223 | if (LoopHeaders) | |||
4224 | LoopHeaders->erase(BB); | |||
4225 | return true; | |||
4226 | } | |||
4227 | ||||
4228 | return Changed; | |||
4229 | } | |||
4230 | ||||
4231 | static bool CasesAreContiguous(SmallVectorImpl<ConstantInt *> &Cases) { | |||
4232 | assert(Cases.size() >= 1)(static_cast <bool> (Cases.size() >= 1) ? void (0) : __assert_fail ("Cases.size() >= 1", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4232, __extension__ __PRETTY_FUNCTION__)); | |||
4233 | ||||
4234 | array_pod_sort(Cases.begin(), Cases.end(), ConstantIntSortPredicate); | |||
4235 | for (size_t I = 1, E = Cases.size(); I != E; ++I) { | |||
4236 | if (Cases[I - 1]->getValue() != Cases[I]->getValue() + 1) | |||
4237 | return false; | |||
4238 | } | |||
4239 | return true; | |||
4240 | } | |||
4241 | ||||
4242 | /// Turn a switch with two reachable destinations into an integer range | |||
4243 | /// comparison and branch. | |||
4244 | static bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder) { | |||
4245 | assert(SI->getNumCases() > 1 && "Degenerate switch?")(static_cast <bool> (SI->getNumCases() > 1 && "Degenerate switch?") ? void (0) : __assert_fail ("SI->getNumCases() > 1 && \"Degenerate switch?\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4245, __extension__ __PRETTY_FUNCTION__)); | |||
4246 | ||||
4247 | bool HasDefault = | |||
4248 | !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg()); | |||
4249 | ||||
4250 | // Partition the cases into two sets with different destinations. | |||
4251 | BasicBlock *DestA = HasDefault ? SI->getDefaultDest() : nullptr; | |||
4252 | BasicBlock *DestB = nullptr; | |||
4253 | SmallVector<ConstantInt *, 16> CasesA; | |||
4254 | SmallVector<ConstantInt *, 16> CasesB; | |||
4255 | ||||
4256 | for (auto Case : SI->cases()) { | |||
4257 | BasicBlock *Dest = Case.getCaseSuccessor(); | |||
4258 | if (!DestA) | |||
4259 | DestA = Dest; | |||
4260 | if (Dest == DestA) { | |||
4261 | CasesA.push_back(Case.getCaseValue()); | |||
4262 | continue; | |||
4263 | } | |||
4264 | if (!DestB) | |||
4265 | DestB = Dest; | |||
4266 | if (Dest == DestB) { | |||
4267 | CasesB.push_back(Case.getCaseValue()); | |||
4268 | continue; | |||
4269 | } | |||
4270 | return false; // More than two destinations. | |||
4271 | } | |||
4272 | ||||
4273 | assert(DestA && DestB &&(static_cast <bool> (DestA && DestB && "Single-destination switch should have been folded." ) ? void (0) : __assert_fail ("DestA && DestB && \"Single-destination switch should have been folded.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4274, __extension__ __PRETTY_FUNCTION__)) | |||
4274 | "Single-destination switch should have been folded.")(static_cast <bool> (DestA && DestB && "Single-destination switch should have been folded." ) ? void (0) : __assert_fail ("DestA && DestB && \"Single-destination switch should have been folded.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4274, __extension__ __PRETTY_FUNCTION__)); | |||
4275 | assert(DestA != DestB)(static_cast <bool> (DestA != DestB) ? void (0) : __assert_fail ("DestA != DestB", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4275, __extension__ __PRETTY_FUNCTION__)); | |||
4276 | assert(DestB != SI->getDefaultDest())(static_cast <bool> (DestB != SI->getDefaultDest()) ? void (0) : __assert_fail ("DestB != SI->getDefaultDest()" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4276, __extension__ __PRETTY_FUNCTION__)); | |||
4277 | assert(!CasesB.empty() && "There must be non-default cases.")(static_cast <bool> (!CasesB.empty() && "There must be non-default cases." ) ? void (0) : __assert_fail ("!CasesB.empty() && \"There must be non-default cases.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4277, __extension__ __PRETTY_FUNCTION__)); | |||
4278 | assert(!CasesA.empty() || HasDefault)(static_cast <bool> (!CasesA.empty() || HasDefault) ? void (0) : __assert_fail ("!CasesA.empty() || HasDefault", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4278, __extension__ __PRETTY_FUNCTION__)); | |||
4279 | ||||
4280 | // Figure out if one of the sets of cases form a contiguous range. | |||
4281 | SmallVectorImpl<ConstantInt *> *ContiguousCases = nullptr; | |||
4282 | BasicBlock *ContiguousDest = nullptr; | |||
4283 | BasicBlock *OtherDest = nullptr; | |||
4284 | if (!CasesA.empty() && CasesAreContiguous(CasesA)) { | |||
4285 | ContiguousCases = &CasesA; | |||
4286 | ContiguousDest = DestA; | |||
4287 | OtherDest = DestB; | |||
4288 | } else if (CasesAreContiguous(CasesB)) { | |||
4289 | ContiguousCases = &CasesB; | |||
4290 | ContiguousDest = DestB; | |||
4291 | OtherDest = DestA; | |||
4292 | } else | |||
4293 | return false; | |||
4294 | ||||
4295 | // Start building the compare and branch. | |||
4296 | ||||
4297 | Constant *Offset = ConstantExpr::getNeg(ContiguousCases->back()); | |||
4298 | Constant *NumCases = | |||
4299 | ConstantInt::get(Offset->getType(), ContiguousCases->size()); | |||
4300 | ||||
4301 | Value *Sub = SI->getCondition(); | |||
4302 | if (!Offset->isNullValue()) | |||
4303 | Sub = Builder.CreateAdd(Sub, Offset, Sub->getName() + ".off"); | |||
4304 | ||||
4305 | Value *Cmp; | |||
4306 | // If NumCases overflowed, then all possible values jump to the successor. | |||
4307 | if (NumCases->isNullValue() && !ContiguousCases->empty()) | |||
4308 | Cmp = ConstantInt::getTrue(SI->getContext()); | |||
4309 | else | |||
4310 | Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch"); | |||
4311 | BranchInst *NewBI = Builder.CreateCondBr(Cmp, ContiguousDest, OtherDest); | |||
4312 | ||||
4313 | // Update weight for the newly-created conditional branch. | |||
4314 | if (HasBranchWeights(SI)) { | |||
4315 | SmallVector<uint64_t, 8> Weights; | |||
4316 | GetBranchWeights(SI, Weights); | |||
4317 | if (Weights.size() == 1 + SI->getNumCases()) { | |||
4318 | uint64_t TrueWeight = 0; | |||
4319 | uint64_t FalseWeight = 0; | |||
4320 | for (size_t I = 0, E = Weights.size(); I != E; ++I) { | |||
4321 | if (SI->getSuccessor(I) == ContiguousDest) | |||
4322 | TrueWeight += Weights[I]; | |||
4323 | else | |||
4324 | FalseWeight += Weights[I]; | |||
4325 | } | |||
4326 | while (TrueWeight > UINT32_MAX(4294967295U) || FalseWeight > UINT32_MAX(4294967295U)) { | |||
4327 | TrueWeight /= 2; | |||
4328 | FalseWeight /= 2; | |||
4329 | } | |||
4330 | setBranchWeights(NewBI, TrueWeight, FalseWeight); | |||
4331 | } | |||
4332 | } | |||
4333 | ||||
4334 | // Prune obsolete incoming values off the successors' PHI nodes. | |||
4335 | for (auto BBI = ContiguousDest->begin(); isa<PHINode>(BBI); ++BBI) { | |||
4336 | unsigned PreviousEdges = ContiguousCases->size(); | |||
4337 | if (ContiguousDest == SI->getDefaultDest()) | |||
4338 | ++PreviousEdges; | |||
4339 | for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I) | |||
4340 | cast<PHINode>(BBI)->removeIncomingValue(SI->getParent()); | |||
4341 | } | |||
4342 | for (auto BBI = OtherDest->begin(); isa<PHINode>(BBI); ++BBI) { | |||
4343 | unsigned PreviousEdges = SI->getNumCases() - ContiguousCases->size(); | |||
4344 | if (OtherDest == SI->getDefaultDest()) | |||
4345 | ++PreviousEdges; | |||
4346 | for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I) | |||
4347 | cast<PHINode>(BBI)->removeIncomingValue(SI->getParent()); | |||
4348 | } | |||
4349 | ||||
4350 | // Drop the switch. | |||
4351 | SI->eraseFromParent(); | |||
4352 | ||||
4353 | return true; | |||
4354 | } | |||
4355 | ||||
4356 | /// Compute masked bits for the condition of a switch | |||
4357 | /// and use it to remove dead cases. | |||
4358 | static bool eliminateDeadSwitchCases(SwitchInst *SI, AssumptionCache *AC, | |||
4359 | const DataLayout &DL) { | |||
4360 | Value *Cond = SI->getCondition(); | |||
4361 | unsigned Bits = Cond->getType()->getIntegerBitWidth(); | |||
4362 | KnownBits Known = computeKnownBits(Cond, DL, 0, AC, SI); | |||
4363 | ||||
4364 | // We can also eliminate cases by determining that their values are outside of | |||
4365 | // the limited range of the condition based on how many significant (non-sign) | |||
4366 | // bits are in the condition value. | |||
4367 | unsigned ExtraSignBits = ComputeNumSignBits(Cond, DL, 0, AC, SI) - 1; | |||
4368 | unsigned MaxSignificantBitsInCond = Bits - ExtraSignBits; | |||
4369 | ||||
4370 | // Gather dead cases. | |||
4371 | SmallVector<ConstantInt *, 8> DeadCases; | |||
4372 | for (auto &Case : SI->cases()) { | |||
4373 | const APInt &CaseVal = Case.getCaseValue()->getValue(); | |||
4374 | if (Known.Zero.intersects(CaseVal) || !Known.One.isSubsetOf(CaseVal) || | |||
4375 | (CaseVal.getMinSignedBits() > MaxSignificantBitsInCond)) { | |||
4376 | DeadCases.push_back(Case.getCaseValue()); | |||
4377 | DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseVal << " is dead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SimplifyCFG: switch case " << CaseVal << " is dead.\n"; } } while (false); | |||
4378 | } | |||
4379 | } | |||
4380 | ||||
4381 | // If we can prove that the cases must cover all possible values, the | |||
4382 | // default destination becomes dead and we can remove it. If we know some | |||
4383 | // of the bits in the value, we can use that to more precisely compute the | |||
4384 | // number of possible unique case values. | |||
4385 | bool HasDefault = | |||
4386 | !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg()); | |||
4387 | const unsigned NumUnknownBits = | |||
4388 | Bits - (Known.Zero | Known.One).countPopulation(); | |||
4389 | assert(NumUnknownBits <= Bits)(static_cast <bool> (NumUnknownBits <= Bits) ? void ( 0) : __assert_fail ("NumUnknownBits <= Bits", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4389, __extension__ __PRETTY_FUNCTION__)); | |||
4390 | if (HasDefault && DeadCases.empty() && | |||
4391 | NumUnknownBits < 64 /* avoid overflow */ && | |||
4392 | SI->getNumCases() == (1ULL << NumUnknownBits)) { | |||
4393 | DEBUG(dbgs() << "SimplifyCFG: switch default is dead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "SimplifyCFG: switch default is dead.\n" ; } } while (false); | |||
4394 | BasicBlock *NewDefault = | |||
4395 | SplitBlockPredecessors(SI->getDefaultDest(), SI->getParent(), ""); | |||
4396 | SI->setDefaultDest(&*NewDefault); | |||
4397 | SplitBlock(&*NewDefault, &NewDefault->front()); | |||
4398 | auto *OldTI = NewDefault->getTerminator(); | |||
4399 | new UnreachableInst(SI->getContext(), OldTI); | |||
4400 | EraseTerminatorInstAndDCECond(OldTI); | |||
4401 | return true; | |||
4402 | } | |||
4403 | ||||
4404 | SmallVector<uint64_t, 8> Weights; | |||
4405 | bool HasWeight = HasBranchWeights(SI); | |||
4406 | if (HasWeight) { | |||
4407 | GetBranchWeights(SI, Weights); | |||
4408 | HasWeight = (Weights.size() == 1 + SI->getNumCases()); | |||
4409 | } | |||
4410 | ||||
4411 | // Remove dead cases from the switch. | |||
4412 | for (ConstantInt *DeadCase : DeadCases) { | |||
4413 | SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase); | |||
4414 | assert(CaseI != SI->case_default() &&(static_cast <bool> (CaseI != SI->case_default() && "Case was not found. Probably mistake in DeadCases forming." ) ? void (0) : __assert_fail ("CaseI != SI->case_default() && \"Case was not found. Probably mistake in DeadCases forming.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4415, __extension__ __PRETTY_FUNCTION__)) | |||
4415 | "Case was not found. Probably mistake in DeadCases forming.")(static_cast <bool> (CaseI != SI->case_default() && "Case was not found. Probably mistake in DeadCases forming." ) ? void (0) : __assert_fail ("CaseI != SI->case_default() && \"Case was not found. Probably mistake in DeadCases forming.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4415, __extension__ __PRETTY_FUNCTION__)); | |||
4416 | if (HasWeight) { | |||
4417 | std::swap(Weights[CaseI->getCaseIndex() + 1], Weights.back()); | |||
4418 | Weights.pop_back(); | |||
4419 | } | |||
4420 | ||||
4421 | // Prune unused values from PHI nodes. | |||
4422 | CaseI->getCaseSuccessor()->removePredecessor(SI->getParent()); | |||
4423 | SI->removeCase(CaseI); | |||
4424 | } | |||
4425 | if (HasWeight && Weights.size() >= 2) { | |||
4426 | SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end()); | |||
4427 | setBranchWeights(SI, MDWeights); | |||
4428 | } | |||
4429 | ||||
4430 | return !DeadCases.empty(); | |||
4431 | } | |||
4432 | ||||
4433 | /// If BB would be eligible for simplification by | |||
4434 | /// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated | |||
4435 | /// by an unconditional branch), look at the phi node for BB in the successor | |||
4436 | /// block and see if the incoming value is equal to CaseValue. If so, return | |||
4437 | /// the phi node, and set PhiIndex to BB's index in the phi node. | |||
4438 | static PHINode *FindPHIForConditionForwarding(ConstantInt *CaseValue, | |||
4439 | BasicBlock *BB, int *PhiIndex) { | |||
4440 | if (BB->getFirstNonPHIOrDbg() != BB->getTerminator()) | |||
4441 | return nullptr; // BB must be empty to be a candidate for simplification. | |||
4442 | if (!BB->getSinglePredecessor()) | |||
4443 | return nullptr; // BB must be dominated by the switch. | |||
4444 | ||||
4445 | BranchInst *Branch = dyn_cast<BranchInst>(BB->getTerminator()); | |||
4446 | if (!Branch || !Branch->isUnconditional()) | |||
4447 | return nullptr; // Terminator must be unconditional branch. | |||
4448 | ||||
4449 | BasicBlock *Succ = Branch->getSuccessor(0); | |||
4450 | ||||
4451 | for (PHINode &PHI : Succ->phis()) { | |||
4452 | int Idx = PHI.getBasicBlockIndex(BB); | |||
4453 | assert(Idx >= 0 && "PHI has no entry for predecessor?")(static_cast <bool> (Idx >= 0 && "PHI has no entry for predecessor?" ) ? void (0) : __assert_fail ("Idx >= 0 && \"PHI has no entry for predecessor?\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4453, __extension__ __PRETTY_FUNCTION__)); | |||
4454 | ||||
4455 | Value *InValue = PHI.getIncomingValue(Idx); | |||
4456 | if (InValue != CaseValue) | |||
4457 | continue; | |||
4458 | ||||
4459 | *PhiIndex = Idx; | |||
4460 | return &PHI; | |||
4461 | } | |||
4462 | ||||
4463 | return nullptr; | |||
4464 | } | |||
4465 | ||||
4466 | /// Try to forward the condition of a switch instruction to a phi node | |||
4467 | /// dominated by the switch, if that would mean that some of the destination | |||
4468 | /// blocks of the switch can be folded away. Return true if a change is made. | |||
4469 | static bool ForwardSwitchConditionToPHI(SwitchInst *SI) { | |||
4470 | using ForwardingNodesMap = DenseMap<PHINode *, SmallVector<int, 4>>; | |||
4471 | ||||
4472 | ForwardingNodesMap ForwardingNodes; | |||
4473 | BasicBlock *SwitchBlock = SI->getParent(); | |||
4474 | bool Changed = false; | |||
4475 | for (auto &Case : SI->cases()) { | |||
4476 | ConstantInt *CaseValue = Case.getCaseValue(); | |||
4477 | BasicBlock *CaseDest = Case.getCaseSuccessor(); | |||
4478 | ||||
4479 | // Replace phi operands in successor blocks that are using the constant case | |||
4480 | // value rather than the switch condition variable: | |||
4481 | // switchbb: | |||
4482 | // switch i32 %x, label %default [ | |||
4483 | // i32 17, label %succ | |||
4484 | // ... | |||
4485 | // succ: | |||
4486 | // %r = phi i32 ... [ 17, %switchbb ] ... | |||
4487 | // --> | |||
4488 | // %r = phi i32 ... [ %x, %switchbb ] ... | |||
4489 | ||||
4490 | for (PHINode &Phi : CaseDest->phis()) { | |||
4491 | // This only works if there is exactly 1 incoming edge from the switch to | |||
4492 | // a phi. If there is >1, that means multiple cases of the switch map to 1 | |||
4493 | // value in the phi, and that phi value is not the switch condition. Thus, | |||
4494 | // this transform would not make sense (the phi would be invalid because | |||
4495 | // a phi can't have different incoming values from the same block). | |||
4496 | int SwitchBBIdx = Phi.getBasicBlockIndex(SwitchBlock); | |||
4497 | if (Phi.getIncomingValue(SwitchBBIdx) == CaseValue && | |||
4498 | count(Phi.blocks(), SwitchBlock) == 1) { | |||
4499 | Phi.setIncomingValue(SwitchBBIdx, SI->getCondition()); | |||
4500 | Changed = true; | |||
4501 | } | |||
4502 | } | |||
4503 | ||||
4504 | // Collect phi nodes that are indirectly using this switch's case constants. | |||
4505 | int PhiIdx; | |||
4506 | if (auto *Phi = FindPHIForConditionForwarding(CaseValue, CaseDest, &PhiIdx)) | |||
4507 | ForwardingNodes[Phi].push_back(PhiIdx); | |||
4508 | } | |||
4509 | ||||
4510 | for (auto &ForwardingNode : ForwardingNodes) { | |||
4511 | PHINode *Phi = ForwardingNode.first; | |||
4512 | SmallVectorImpl<int> &Indexes = ForwardingNode.second; | |||
4513 | if (Indexes.size() < 2) | |||
4514 | continue; | |||
4515 | ||||
4516 | for (int Index : Indexes) | |||
4517 | Phi->setIncomingValue(Index, SI->getCondition()); | |||
4518 | Changed = true; | |||
4519 | } | |||
4520 | ||||
4521 | return Changed; | |||
4522 | } | |||
4523 | ||||
4524 | /// Return true if the backend will be able to handle | |||
4525 | /// initializing an array of constants like C. | |||
4526 | static bool ValidLookupTableConstant(Constant *C, const TargetTransformInfo &TTI) { | |||
4527 | if (C->isThreadDependent()) | |||
4528 | return false; | |||
4529 | if (C->isDLLImportDependent()) | |||
4530 | return false; | |||
4531 | ||||
4532 | if (!isa<ConstantFP>(C) && !isa<ConstantInt>(C) && | |||
4533 | !isa<ConstantPointerNull>(C) && !isa<GlobalValue>(C) && | |||
4534 | !isa<UndefValue>(C) && !isa<ConstantExpr>(C)) | |||
4535 | return false; | |||
4536 | ||||
4537 | if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { | |||
4538 | if (!CE->isGEPWithNoNotionalOverIndexing()) | |||
4539 | return false; | |||
4540 | if (!ValidLookupTableConstant(CE->getOperand(0), TTI)) | |||
4541 | return false; | |||
4542 | } | |||
4543 | ||||
4544 | if (!TTI.shouldBuildLookupTablesForConstant(C)) | |||
4545 | return false; | |||
4546 | ||||
4547 | return true; | |||
4548 | } | |||
4549 | ||||
4550 | /// If V is a Constant, return it. Otherwise, try to look up | |||
4551 | /// its constant value in ConstantPool, returning 0 if it's not there. | |||
4552 | static Constant * | |||
4553 | LookupConstant(Value *V, | |||
4554 | const SmallDenseMap<Value *, Constant *> &ConstantPool) { | |||
4555 | if (Constant *C = dyn_cast<Constant>(V)) | |||
4556 | return C; | |||
4557 | return ConstantPool.lookup(V); | |||
4558 | } | |||
4559 | ||||
4560 | /// Try to fold instruction I into a constant. This works for | |||
4561 | /// simple instructions such as binary operations where both operands are | |||
4562 | /// constant or can be replaced by constants from the ConstantPool. Returns the | |||
4563 | /// resulting constant on success, 0 otherwise. | |||
4564 | static Constant * | |||
4565 | ConstantFold(Instruction *I, const DataLayout &DL, | |||
4566 | const SmallDenseMap<Value *, Constant *> &ConstantPool) { | |||
4567 | if (SelectInst *Select = dyn_cast<SelectInst>(I)) { | |||
4568 | Constant *A = LookupConstant(Select->getCondition(), ConstantPool); | |||
4569 | if (!A) | |||
4570 | return nullptr; | |||
4571 | if (A->isAllOnesValue()) | |||
4572 | return LookupConstant(Select->getTrueValue(), ConstantPool); | |||
4573 | if (A->isNullValue()) | |||
4574 | return LookupConstant(Select->getFalseValue(), ConstantPool); | |||
4575 | return nullptr; | |||
4576 | } | |||
4577 | ||||
4578 | SmallVector<Constant *, 4> COps; | |||
4579 | for (unsigned N = 0, E = I->getNumOperands(); N != E; ++N) { | |||
4580 | if (Constant *A = LookupConstant(I->getOperand(N), ConstantPool)) | |||
4581 | COps.push_back(A); | |||
4582 | else | |||
4583 | return nullptr; | |||
4584 | } | |||
4585 | ||||
4586 | if (CmpInst *Cmp = dyn_cast<CmpInst>(I)) { | |||
4587 | return ConstantFoldCompareInstOperands(Cmp->getPredicate(), COps[0], | |||
4588 | COps[1], DL); | |||
4589 | } | |||
4590 | ||||
4591 | return ConstantFoldInstOperands(I, COps, DL); | |||
4592 | } | |||
4593 | ||||
4594 | /// Try to determine the resulting constant values in phi nodes | |||
4595 | /// at the common destination basic block, *CommonDest, for one of the case | |||
4596 | /// destionations CaseDest corresponding to value CaseVal (0 for the default | |||
4597 | /// case), of a switch instruction SI. | |||
4598 | static bool | |||
4599 | GetCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest, | |||
4600 | BasicBlock **CommonDest, | |||
4601 | SmallVectorImpl<std::pair<PHINode *, Constant *>> &Res, | |||
4602 | const DataLayout &DL, const TargetTransformInfo &TTI) { | |||
4603 | // The block from which we enter the common destination. | |||
4604 | BasicBlock *Pred = SI->getParent(); | |||
4605 | ||||
4606 | // If CaseDest is empty except for some side-effect free instructions through | |||
4607 | // which we can constant-propagate the CaseVal, continue to its successor. | |||
4608 | SmallDenseMap<Value *, Constant *> ConstantPool; | |||
4609 | ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal)); | |||
4610 | for (BasicBlock::iterator I = CaseDest->begin(), E = CaseDest->end(); I != E; | |||
4611 | ++I) { | |||
4612 | if (TerminatorInst *T = dyn_cast<TerminatorInst>(I)) { | |||
4613 | // If the terminator is a simple branch, continue to the next block. | |||
4614 | if (T->getNumSuccessors() != 1 || T->isExceptional()) | |||
4615 | return false; | |||
4616 | Pred = CaseDest; | |||
4617 | CaseDest = T->getSuccessor(0); | |||
4618 | } else if (isa<DbgInfoIntrinsic>(I)) { | |||
4619 | // Skip debug intrinsic. | |||
4620 | continue; | |||
4621 | } else if (Constant *C = ConstantFold(&*I, DL, ConstantPool)) { | |||
4622 | // Instruction is side-effect free and constant. | |||
4623 | ||||
4624 | // If the instruction has uses outside this block or a phi node slot for | |||
4625 | // the block, it is not safe to bypass the instruction since it would then | |||
4626 | // no longer dominate all its uses. | |||
4627 | for (auto &Use : I->uses()) { | |||
4628 | User *User = Use.getUser(); | |||
4629 | if (Instruction *I = dyn_cast<Instruction>(User)) | |||
4630 | if (I->getParent() == CaseDest) | |||
4631 | continue; | |||
4632 | if (PHINode *Phi = dyn_cast<PHINode>(User)) | |||
4633 | if (Phi->getIncomingBlock(Use) == CaseDest) | |||
4634 | continue; | |||
4635 | return false; | |||
4636 | } | |||
4637 | ||||
4638 | ConstantPool.insert(std::make_pair(&*I, C)); | |||
4639 | } else { | |||
4640 | break; | |||
4641 | } | |||
4642 | } | |||
4643 | ||||
4644 | // If we did not have a CommonDest before, use the current one. | |||
4645 | if (!*CommonDest) | |||
4646 | *CommonDest = CaseDest; | |||
4647 | // If the destination isn't the common one, abort. | |||
4648 | if (CaseDest != *CommonDest) | |||
4649 | return false; | |||
4650 | ||||
4651 | // Get the values for this case from phi nodes in the destination block. | |||
4652 | for (PHINode &PHI : (*CommonDest)->phis()) { | |||
4653 | int Idx = PHI.getBasicBlockIndex(Pred); | |||
4654 | if (Idx == -1) | |||
4655 | continue; | |||
4656 | ||||
4657 | Constant *ConstVal = | |||
4658 | LookupConstant(PHI.getIncomingValue(Idx), ConstantPool); | |||
4659 | if (!ConstVal) | |||
4660 | return false; | |||
4661 | ||||
4662 | // Be conservative about which kinds of constants we support. | |||
4663 | if (!ValidLookupTableConstant(ConstVal, TTI)) | |||
4664 | return false; | |||
4665 | ||||
4666 | Res.push_back(std::make_pair(&PHI, ConstVal)); | |||
4667 | } | |||
4668 | ||||
4669 | return Res.size() > 0; | |||
4670 | } | |||
4671 | ||||
4672 | // Helper function used to add CaseVal to the list of cases that generate | |||
4673 | // Result. Returns the updated number of cases that generate this result. | |||
4674 | static uintptr_t MapCaseToResult(ConstantInt *CaseVal, | |||
4675 | SwitchCaseResultVectorTy &UniqueResults, | |||
4676 | Constant *Result) { | |||
4677 | for (auto &I : UniqueResults) { | |||
4678 | if (I.first == Result) { | |||
4679 | I.second.push_back(CaseVal); | |||
4680 | return I.second.size(); | |||
4681 | } | |||
4682 | } | |||
4683 | UniqueResults.push_back( | |||
4684 | std::make_pair(Result, SmallVector<ConstantInt *, 4>(1, CaseVal))); | |||
4685 | return 1; | |||
4686 | } | |||
4687 | ||||
4688 | // Helper function that initializes a map containing | |||
4689 | // results for the PHI node of the common destination block for a switch | |||
4690 | // instruction. Returns false if multiple PHI nodes have been found or if | |||
4691 | // there is not a common destination block for the switch. | |||
4692 | static bool | |||
4693 | InitializeUniqueCases(SwitchInst *SI, PHINode *&PHI, BasicBlock *&CommonDest, | |||
4694 | SwitchCaseResultVectorTy &UniqueResults, | |||
4695 | Constant *&DefaultResult, const DataLayout &DL, | |||
4696 | const TargetTransformInfo &TTI, | |||
4697 | uintptr_t MaxUniqueResults, uintptr_t MaxCasesPerResult) { | |||
4698 | for (auto &I : SI->cases()) { | |||
4699 | ConstantInt *CaseVal = I.getCaseValue(); | |||
4700 | ||||
4701 | // Resulting value at phi nodes for this case value. | |||
4702 | SwitchCaseResultsTy Results; | |||
4703 | if (!GetCaseResults(SI, CaseVal, I.getCaseSuccessor(), &CommonDest, Results, | |||
4704 | DL, TTI)) | |||
4705 | return false; | |||
4706 | ||||
4707 | // Only one value per case is permitted. | |||
4708 | if (Results.size() > 1) | |||
4709 | return false; | |||
4710 | ||||
4711 | // Add the case->result mapping to UniqueResults. | |||
4712 | const uintptr_t NumCasesForResult = | |||
4713 | MapCaseToResult(CaseVal, UniqueResults, Results.begin()->second); | |||
4714 | ||||
4715 | // Early out if there are too many cases for this result. | |||
4716 | if (NumCasesForResult > MaxCasesPerResult) | |||
4717 | return false; | |||
4718 | ||||
4719 | // Early out if there are too many unique results. | |||
4720 | if (UniqueResults.size() > MaxUniqueResults) | |||
4721 | return false; | |||
4722 | ||||
4723 | // Check the PHI consistency. | |||
4724 | if (!PHI) | |||
4725 | PHI = Results[0].first; | |||
4726 | else if (PHI != Results[0].first) | |||
4727 | return false; | |||
4728 | } | |||
4729 | // Find the default result value. | |||
4730 | SmallVector<std::pair<PHINode *, Constant *>, 1> DefaultResults; | |||
4731 | BasicBlock *DefaultDest = SI->getDefaultDest(); | |||
4732 | GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, DefaultResults, | |||
4733 | DL, TTI); | |||
4734 | // If the default value is not found abort unless the default destination | |||
4735 | // is unreachable. | |||
4736 | DefaultResult = | |||
4737 | DefaultResults.size() == 1 ? DefaultResults.begin()->second : nullptr; | |||
4738 | if ((!DefaultResult && | |||
4739 | !isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg()))) | |||
4740 | return false; | |||
4741 | ||||
4742 | return true; | |||
4743 | } | |||
4744 | ||||
4745 | // Helper function that checks if it is possible to transform a switch with only | |||
4746 | // two cases (or two cases + default) that produces a result into a select. | |||
4747 | // Example: | |||
4748 | // switch (a) { | |||
4749 | // case 10: %0 = icmp eq i32 %a, 10 | |||
4750 | // return 10; %1 = select i1 %0, i32 10, i32 4 | |||
4751 | // case 20: ----> %2 = icmp eq i32 %a, 20 | |||
4752 | // return 2; %3 = select i1 %2, i32 2, i32 %1 | |||
4753 | // default: | |||
4754 | // return 4; | |||
4755 | // } | |||
4756 | static Value *ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector, | |||
4757 | Constant *DefaultResult, Value *Condition, | |||
4758 | IRBuilder<> &Builder) { | |||
4759 | assert(ResultVector.size() == 2 &&(static_cast <bool> (ResultVector.size() == 2 && "We should have exactly two unique results at this point") ? void (0) : __assert_fail ("ResultVector.size() == 2 && \"We should have exactly two unique results at this point\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4760, __extension__ __PRETTY_FUNCTION__)) | |||
4760 | "We should have exactly two unique results at this point")(static_cast <bool> (ResultVector.size() == 2 && "We should have exactly two unique results at this point") ? void (0) : __assert_fail ("ResultVector.size() == 2 && \"We should have exactly two unique results at this point\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4760, __extension__ __PRETTY_FUNCTION__)); | |||
4761 | // If we are selecting between only two cases transform into a simple | |||
4762 | // select or a two-way select if default is possible. | |||
4763 | if (ResultVector[0].second.size() == 1 && | |||
4764 | ResultVector[1].second.size() == 1) { | |||
4765 | ConstantInt *const FirstCase = ResultVector[0].second[0]; | |||
4766 | ConstantInt *const SecondCase = ResultVector[1].second[0]; | |||
4767 | ||||
4768 | bool DefaultCanTrigger = DefaultResult; | |||
4769 | Value *SelectValue = ResultVector[1].first; | |||
4770 | if (DefaultCanTrigger) { | |||
4771 | Value *const ValueCompare = | |||
4772 | Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp"); | |||
4773 | SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first, | |||
4774 | DefaultResult, "switch.select"); | |||
4775 | } | |||
4776 | Value *const ValueCompare = | |||
4777 | Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp"); | |||
4778 | return Builder.CreateSelect(ValueCompare, ResultVector[0].first, | |||
4779 | SelectValue, "switch.select"); | |||
4780 | } | |||
4781 | ||||
4782 | return nullptr; | |||
4783 | } | |||
4784 | ||||
4785 | // Helper function to cleanup a switch instruction that has been converted into | |||
4786 | // a select, fixing up PHI nodes and basic blocks. | |||
4787 | static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI, | |||
4788 | Value *SelectValue, | |||
4789 | IRBuilder<> &Builder) { | |||
4790 | BasicBlock *SelectBB = SI->getParent(); | |||
4791 | while (PHI->getBasicBlockIndex(SelectBB) >= 0) | |||
4792 | PHI->removeIncomingValue(SelectBB); | |||
4793 | PHI->addIncoming(SelectValue, SelectBB); | |||
4794 | ||||
4795 | Builder.CreateBr(PHI->getParent()); | |||
4796 | ||||
4797 | // Remove the switch. | |||
4798 | for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) { | |||
4799 | BasicBlock *Succ = SI->getSuccessor(i); | |||
4800 | ||||
4801 | if (Succ == PHI->getParent()) | |||
4802 | continue; | |||
4803 | Succ->removePredecessor(SelectBB); | |||
4804 | } | |||
4805 | SI->eraseFromParent(); | |||
4806 | } | |||
4807 | ||||
4808 | /// If the switch is only used to initialize one or more | |||
4809 | /// phi nodes in a common successor block with only two different | |||
4810 | /// constant values, replace the switch with select. | |||
4811 | static bool switchToSelect(SwitchInst *SI, IRBuilder<> &Builder, | |||
4812 | const DataLayout &DL, | |||
4813 | const TargetTransformInfo &TTI) { | |||
4814 | Value *const Cond = SI->getCondition(); | |||
4815 | PHINode *PHI = nullptr; | |||
4816 | BasicBlock *CommonDest = nullptr; | |||
4817 | Constant *DefaultResult; | |||
4818 | SwitchCaseResultVectorTy UniqueResults; | |||
4819 | // Collect all the cases that will deliver the same value from the switch. | |||
4820 | if (!InitializeUniqueCases(SI, PHI, CommonDest, UniqueResults, DefaultResult, | |||
4821 | DL, TTI, 2, 1)) | |||
4822 | return false; | |||
4823 | // Selects choose between maximum two values. | |||
4824 | if (UniqueResults.size() != 2) | |||
4825 | return false; | |||
4826 | assert(PHI != nullptr && "PHI for value select not found")(static_cast <bool> (PHI != nullptr && "PHI for value select not found" ) ? void (0) : __assert_fail ("PHI != nullptr && \"PHI for value select not found\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4826, __extension__ __PRETTY_FUNCTION__)); | |||
4827 | ||||
4828 | Builder.SetInsertPoint(SI); | |||
4829 | Value *SelectValue = | |||
4830 | ConvertTwoCaseSwitch(UniqueResults, DefaultResult, Cond, Builder); | |||
4831 | if (SelectValue) { | |||
4832 | RemoveSwitchAfterSelectConversion(SI, PHI, SelectValue, Builder); | |||
4833 | return true; | |||
4834 | } | |||
4835 | // The switch couldn't be converted into a select. | |||
4836 | return false; | |||
4837 | } | |||
4838 | ||||
4839 | namespace { | |||
4840 | ||||
4841 | /// This class represents a lookup table that can be used to replace a switch. | |||
4842 | class SwitchLookupTable { | |||
4843 | public: | |||
4844 | /// Create a lookup table to use as a switch replacement with the contents | |||
4845 | /// of Values, using DefaultValue to fill any holes in the table. | |||
4846 | SwitchLookupTable( | |||
4847 | Module &M, uint64_t TableSize, ConstantInt *Offset, | |||
4848 | const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values, | |||
4849 | Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName); | |||
4850 | ||||
4851 | /// Build instructions with Builder to retrieve the value at | |||
4852 | /// the position given by Index in the lookup table. | |||
4853 | Value *BuildLookup(Value *Index, IRBuilder<> &Builder); | |||
4854 | ||||
4855 | /// Return true if a table with TableSize elements of | |||
4856 | /// type ElementType would fit in a target-legal register. | |||
4857 | static bool WouldFitInRegister(const DataLayout &DL, uint64_t TableSize, | |||
4858 | Type *ElementType); | |||
4859 | ||||
4860 | private: | |||
4861 | // Depending on the contents of the table, it can be represented in | |||
4862 | // different ways. | |||
4863 | enum { | |||
4864 | // For tables where each element contains the same value, we just have to | |||
4865 | // store that single value and return it for each lookup. | |||
4866 | SingleValueKind, | |||
4867 | ||||
4868 | // For tables where there is a linear relationship between table index | |||
4869 | // and values. We calculate the result with a simple multiplication | |||
4870 | // and addition instead of a table lookup. | |||
4871 | LinearMapKind, | |||
4872 | ||||
4873 | // For small tables with integer elements, we can pack them into a bitmap | |||
4874 | // that fits into a target-legal register. Values are retrieved by | |||
4875 | // shift and mask operations. | |||
4876 | BitMapKind, | |||
4877 | ||||
4878 | // The table is stored as an array of values. Values are retrieved by load | |||
4879 | // instructions from the table. | |||
4880 | ArrayKind | |||
4881 | } Kind; | |||
4882 | ||||
4883 | // For SingleValueKind, this is the single value. | |||
4884 | Constant *SingleValue = nullptr; | |||
4885 | ||||
4886 | // For BitMapKind, this is the bitmap. | |||
4887 | ConstantInt *BitMap = nullptr; | |||
4888 | IntegerType *BitMapElementTy = nullptr; | |||
4889 | ||||
4890 | // For LinearMapKind, these are the constants used to derive the value. | |||
4891 | ConstantInt *LinearOffset = nullptr; | |||
4892 | ConstantInt *LinearMultiplier = nullptr; | |||
4893 | ||||
4894 | // For ArrayKind, this is the array. | |||
4895 | GlobalVariable *Array = nullptr; | |||
4896 | }; | |||
4897 | ||||
4898 | } // end anonymous namespace | |||
4899 | ||||
4900 | SwitchLookupTable::SwitchLookupTable( | |||
4901 | Module &M, uint64_t TableSize, ConstantInt *Offset, | |||
4902 | const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values, | |||
4903 | Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName) { | |||
4904 | assert(Values.size() && "Can't build lookup table without values!")(static_cast <bool> (Values.size() && "Can't build lookup table without values!" ) ? void (0) : __assert_fail ("Values.size() && \"Can't build lookup table without values!\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4904, __extension__ __PRETTY_FUNCTION__)); | |||
4905 | assert(TableSize >= Values.size() && "Can't fit values in table!")(static_cast <bool> (TableSize >= Values.size() && "Can't fit values in table!") ? void (0) : __assert_fail ("TableSize >= Values.size() && \"Can't fit values in table!\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4905, __extension__ __PRETTY_FUNCTION__)); | |||
4906 | ||||
4907 | // If all values in the table are equal, this is that value. | |||
4908 | SingleValue = Values.begin()->second; | |||
4909 | ||||
4910 | Type *ValueType = Values.begin()->second->getType(); | |||
4911 | ||||
4912 | // Build up the table contents. | |||
4913 | SmallVector<Constant *, 64> TableContents(TableSize); | |||
4914 | for (size_t I = 0, E = Values.size(); I != E; ++I) { | |||
4915 | ConstantInt *CaseVal = Values[I].first; | |||
4916 | Constant *CaseRes = Values[I].second; | |||
4917 | assert(CaseRes->getType() == ValueType)(static_cast <bool> (CaseRes->getType() == ValueType ) ? void (0) : __assert_fail ("CaseRes->getType() == ValueType" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4917, __extension__ __PRETTY_FUNCTION__)); | |||
4918 | ||||
4919 | uint64_t Idx = (CaseVal->getValue() - Offset->getValue()).getLimitedValue(); | |||
4920 | TableContents[Idx] = CaseRes; | |||
4921 | ||||
4922 | if (CaseRes != SingleValue) | |||
4923 | SingleValue = nullptr; | |||
4924 | } | |||
4925 | ||||
4926 | // Fill in any holes in the table with the default result. | |||
4927 | if (Values.size() < TableSize) { | |||
4928 | assert(DefaultValue &&(static_cast <bool> (DefaultValue && "Need a default value to fill the lookup table holes." ) ? void (0) : __assert_fail ("DefaultValue && \"Need a default value to fill the lookup table holes.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4929, __extension__ __PRETTY_FUNCTION__)) | |||
4929 | "Need a default value to fill the lookup table holes.")(static_cast <bool> (DefaultValue && "Need a default value to fill the lookup table holes." ) ? void (0) : __assert_fail ("DefaultValue && \"Need a default value to fill the lookup table holes.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4929, __extension__ __PRETTY_FUNCTION__)); | |||
4930 | assert(DefaultValue->getType() == ValueType)(static_cast <bool> (DefaultValue->getType() == ValueType ) ? void (0) : __assert_fail ("DefaultValue->getType() == ValueType" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4930, __extension__ __PRETTY_FUNCTION__)); | |||
4931 | for (uint64_t I = 0; I < TableSize; ++I) { | |||
4932 | if (!TableContents[I]) | |||
4933 | TableContents[I] = DefaultValue; | |||
4934 | } | |||
4935 | ||||
4936 | if (DefaultValue != SingleValue) | |||
4937 | SingleValue = nullptr; | |||
4938 | } | |||
4939 | ||||
4940 | // If each element in the table contains the same value, we only need to store | |||
4941 | // that single value. | |||
4942 | if (SingleValue) { | |||
4943 | Kind = SingleValueKind; | |||
4944 | return; | |||
4945 | } | |||
4946 | ||||
4947 | // Check if we can derive the value with a linear transformation from the | |||
4948 | // table index. | |||
4949 | if (isa<IntegerType>(ValueType)) { | |||
4950 | bool LinearMappingPossible = true; | |||
4951 | APInt PrevVal; | |||
4952 | APInt DistToPrev; | |||
4953 | assert(TableSize >= 2 && "Should be a SingleValue table.")(static_cast <bool> (TableSize >= 2 && "Should be a SingleValue table." ) ? void (0) : __assert_fail ("TableSize >= 2 && \"Should be a SingleValue table.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 4953, __extension__ __PRETTY_FUNCTION__)); | |||
4954 | // Check if there is the same distance between two consecutive values. | |||
4955 | for (uint64_t I = 0; I < TableSize; ++I) { | |||
4956 | ConstantInt *ConstVal = dyn_cast<ConstantInt>(TableContents[I]); | |||
4957 | if (!ConstVal) { | |||
4958 | // This is an undef. We could deal with it, but undefs in lookup tables | |||
4959 | // are very seldom. It's probably not worth the additional complexity. | |||
4960 | LinearMappingPossible = false; | |||
4961 | break; | |||
4962 | } | |||
4963 | const APInt &Val = ConstVal->getValue(); | |||
4964 | if (I != 0) { | |||
4965 | APInt Dist = Val - PrevVal; | |||
4966 | if (I == 1) { | |||
4967 | DistToPrev = Dist; | |||
4968 | } else if (Dist != DistToPrev) { | |||
4969 | LinearMappingPossible = false; | |||
4970 | break; | |||
4971 | } | |||
4972 | } | |||
4973 | PrevVal = Val; | |||
4974 | } | |||
4975 | if (LinearMappingPossible) { | |||
4976 | LinearOffset = cast<ConstantInt>(TableContents[0]); | |||
4977 | LinearMultiplier = ConstantInt::get(M.getContext(), DistToPrev); | |||
4978 | Kind = LinearMapKind; | |||
4979 | ++NumLinearMaps; | |||
4980 | return; | |||
4981 | } | |||
4982 | } | |||
4983 | ||||
4984 | // If the type is integer and the table fits in a register, build a bitmap. | |||
4985 | if (WouldFitInRegister(DL, TableSize, ValueType)) { | |||
4986 | IntegerType *IT = cast<IntegerType>(ValueType); | |||
4987 | APInt TableInt(TableSize * IT->getBitWidth(), 0); | |||
4988 | for (uint64_t I = TableSize; I > 0; --I) { | |||
4989 | TableInt <<= IT->getBitWidth(); | |||
4990 | // Insert values into the bitmap. Undef values are set to zero. | |||
4991 | if (!isa<UndefValue>(TableContents[I - 1])) { | |||
4992 | ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]); | |||
4993 | TableInt |= Val->getValue().zext(TableInt.getBitWidth()); | |||
4994 | } | |||
4995 | } | |||
4996 | BitMap = ConstantInt::get(M.getContext(), TableInt); | |||
4997 | BitMapElementTy = IT; | |||
4998 | Kind = BitMapKind; | |||
4999 | ++NumBitMaps; | |||
5000 | return; | |||
5001 | } | |||
5002 | ||||
5003 | // Store the table in an array. | |||
5004 | ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize); | |||
5005 | Constant *Initializer = ConstantArray::get(ArrayTy, TableContents); | |||
5006 | ||||
5007 | Array = new GlobalVariable(M, ArrayTy, /*constant=*/true, | |||
5008 | GlobalVariable::PrivateLinkage, Initializer, | |||
5009 | "switch.table." + FuncName); | |||
5010 | Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); | |||
5011 | Kind = ArrayKind; | |||
5012 | } | |||
5013 | ||||
5014 | Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) { | |||
5015 | switch (Kind) { | |||
5016 | case SingleValueKind: | |||
5017 | return SingleValue; | |||
5018 | case LinearMapKind: { | |||
5019 | // Derive the result value from the input value. | |||
5020 | Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(), | |||
5021 | false, "switch.idx.cast"); | |||
5022 | if (!LinearMultiplier->isOne()) | |||
5023 | Result = Builder.CreateMul(Result, LinearMultiplier, "switch.idx.mult"); | |||
5024 | if (!LinearOffset->isZero()) | |||
5025 | Result = Builder.CreateAdd(Result, LinearOffset, "switch.offset"); | |||
5026 | return Result; | |||
5027 | } | |||
5028 | case BitMapKind: { | |||
5029 | // Type of the bitmap (e.g. i59). | |||
5030 | IntegerType *MapTy = BitMap->getType(); | |||
5031 | ||||
5032 | // Cast Index to the same type as the bitmap. | |||
5033 | // Note: The Index is <= the number of elements in the table, so | |||
5034 | // truncating it to the width of the bitmask is safe. | |||
5035 | Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast"); | |||
5036 | ||||
5037 | // Multiply the shift amount by the element width. | |||
5038 | ShiftAmt = Builder.CreateMul( | |||
5039 | ShiftAmt, ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()), | |||
5040 | "switch.shiftamt"); | |||
5041 | ||||
5042 | // Shift down. | |||
5043 | Value *DownShifted = | |||
5044 | Builder.CreateLShr(BitMap, ShiftAmt, "switch.downshift"); | |||
5045 | // Mask off. | |||
5046 | return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked"); | |||
5047 | } | |||
5048 | case ArrayKind: { | |||
5049 | // Make sure the table index will not overflow when treated as signed. | |||
5050 | IntegerType *IT = cast<IntegerType>(Index->getType()); | |||
5051 | uint64_t TableSize = | |||
5052 | Array->getInitializer()->getType()->getArrayNumElements(); | |||
5053 | if (TableSize > (1ULL << (IT->getBitWidth() - 1))) | |||
5054 | Index = Builder.CreateZExt( | |||
5055 | Index, IntegerType::get(IT->getContext(), IT->getBitWidth() + 1), | |||
5056 | "switch.tableidx.zext"); | |||
5057 | ||||
5058 | Value *GEPIndices[] = {Builder.getInt32(0), Index}; | |||
5059 | Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array, | |||
5060 | GEPIndices, "switch.gep"); | |||
5061 | return Builder.CreateLoad(GEP, "switch.load"); | |||
5062 | } | |||
5063 | } | |||
5064 | llvm_unreachable("Unknown lookup table kind!")::llvm::llvm_unreachable_internal("Unknown lookup table kind!" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5064); | |||
5065 | } | |||
5066 | ||||
5067 | bool SwitchLookupTable::WouldFitInRegister(const DataLayout &DL, | |||
5068 | uint64_t TableSize, | |||
5069 | Type *ElementType) { | |||
5070 | auto *IT = dyn_cast<IntegerType>(ElementType); | |||
5071 | if (!IT) | |||
5072 | return false; | |||
5073 | // FIXME: If the type is wider than it needs to be, e.g. i8 but all values | |||
5074 | // are <= 15, we could try to narrow the type. | |||
5075 | ||||
5076 | // Avoid overflow, fitsInLegalInteger uses unsigned int for the width. | |||
5077 | if (TableSize >= UINT_MAX(2147483647 *2U +1U) / IT->getBitWidth()) | |||
5078 | return false; | |||
5079 | return DL.fitsInLegalInteger(TableSize * IT->getBitWidth()); | |||
5080 | } | |||
5081 | ||||
5082 | /// Determine whether a lookup table should be built for this switch, based on | |||
5083 | /// the number of cases, size of the table, and the types of the results. | |||
5084 | static bool | |||
5085 | ShouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize, | |||
5086 | const TargetTransformInfo &TTI, const DataLayout &DL, | |||
5087 | const SmallDenseMap<PHINode *, Type *> &ResultTypes) { | |||
5088 | if (SI->getNumCases() > TableSize || TableSize >= UINT64_MAX(18446744073709551615UL) / 10) | |||
5089 | return false; // TableSize overflowed, or mul below might overflow. | |||
5090 | ||||
5091 | bool AllTablesFitInRegister = true; | |||
5092 | bool HasIllegalType = false; | |||
5093 | for (const auto &I : ResultTypes) { | |||
5094 | Type *Ty = I.second; | |||
5095 | ||||
5096 | // Saturate this flag to true. | |||
5097 | HasIllegalType = HasIllegalType || !TTI.isTypeLegal(Ty); | |||
5098 | ||||
5099 | // Saturate this flag to false. | |||
5100 | AllTablesFitInRegister = | |||
5101 | AllTablesFitInRegister && | |||
5102 | SwitchLookupTable::WouldFitInRegister(DL, TableSize, Ty); | |||
5103 | ||||
5104 | // If both flags saturate, we're done. NOTE: This *only* works with | |||
5105 | // saturating flags, and all flags have to saturate first due to the | |||
5106 | // non-deterministic behavior of iterating over a dense map. | |||
5107 | if (HasIllegalType && !AllTablesFitInRegister) | |||
5108 | break; | |||
5109 | } | |||
5110 | ||||
5111 | // If each table would fit in a register, we should build it anyway. | |||
5112 | if (AllTablesFitInRegister) | |||
5113 | return true; | |||
5114 | ||||
5115 | // Don't build a table that doesn't fit in-register if it has illegal types. | |||
5116 | if (HasIllegalType) | |||
5117 | return false; | |||
5118 | ||||
5119 | // The table density should be at least 40%. This is the same criterion as for | |||
5120 | // jump tables, see SelectionDAGBuilder::handleJTSwitchCase. | |||
5121 | // FIXME: Find the best cut-off. | |||
5122 | return SI->getNumCases() * 10 >= TableSize * 4; | |||
5123 | } | |||
5124 | ||||
5125 | /// Try to reuse the switch table index compare. Following pattern: | |||
5126 | /// \code | |||
5127 | /// if (idx < tablesize) | |||
5128 | /// r = table[idx]; // table does not contain default_value | |||
5129 | /// else | |||
5130 | /// r = default_value; | |||
5131 | /// if (r != default_value) | |||
5132 | /// ... | |||
5133 | /// \endcode | |||
5134 | /// Is optimized to: | |||
5135 | /// \code | |||
5136 | /// cond = idx < tablesize; | |||
5137 | /// if (cond) | |||
5138 | /// r = table[idx]; | |||
5139 | /// else | |||
5140 | /// r = default_value; | |||
5141 | /// if (cond) | |||
5142 | /// ... | |||
5143 | /// \endcode | |||
5144 | /// Jump threading will then eliminate the second if(cond). | |||
5145 | static void reuseTableCompare( | |||
5146 | User *PhiUser, BasicBlock *PhiBlock, BranchInst *RangeCheckBranch, | |||
5147 | Constant *DefaultValue, | |||
5148 | const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values) { | |||
5149 | ICmpInst *CmpInst = dyn_cast<ICmpInst>(PhiUser); | |||
5150 | if (!CmpInst) | |||
5151 | return; | |||
5152 | ||||
5153 | // We require that the compare is in the same block as the phi so that jump | |||
5154 | // threading can do its work afterwards. | |||
5155 | if (CmpInst->getParent() != PhiBlock) | |||
5156 | return; | |||
5157 | ||||
5158 | Constant *CmpOp1 = dyn_cast<Constant>(CmpInst->getOperand(1)); | |||
5159 | if (!CmpOp1) | |||
5160 | return; | |||
5161 | ||||
5162 | Value *RangeCmp = RangeCheckBranch->getCondition(); | |||
5163 | Constant *TrueConst = ConstantInt::getTrue(RangeCmp->getType()); | |||
5164 | Constant *FalseConst = ConstantInt::getFalse(RangeCmp->getType()); | |||
5165 | ||||
5166 | // Check if the compare with the default value is constant true or false. | |||
5167 | Constant *DefaultConst = ConstantExpr::getICmp(CmpInst->getPredicate(), | |||
5168 | DefaultValue, CmpOp1, true); | |||
5169 | if (DefaultConst != TrueConst && DefaultConst != FalseConst) | |||
5170 | return; | |||
5171 | ||||
5172 | // Check if the compare with the case values is distinct from the default | |||
5173 | // compare result. | |||
5174 | for (auto ValuePair : Values) { | |||
5175 | Constant *CaseConst = ConstantExpr::getICmp(CmpInst->getPredicate(), | |||
5176 | ValuePair.second, CmpOp1, true); | |||
5177 | if (!CaseConst || CaseConst == DefaultConst || isa<UndefValue>(CaseConst)) | |||
5178 | return; | |||
5179 | assert((CaseConst == TrueConst || CaseConst == FalseConst) &&(static_cast <bool> ((CaseConst == TrueConst || CaseConst == FalseConst) && "Expect true or false as compare result." ) ? void (0) : __assert_fail ("(CaseConst == TrueConst || CaseConst == FalseConst) && \"Expect true or false as compare result.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5180, __extension__ __PRETTY_FUNCTION__)) | |||
5180 | "Expect true or false as compare result.")(static_cast <bool> ((CaseConst == TrueConst || CaseConst == FalseConst) && "Expect true or false as compare result." ) ? void (0) : __assert_fail ("(CaseConst == TrueConst || CaseConst == FalseConst) && \"Expect true or false as compare result.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5180, __extension__ __PRETTY_FUNCTION__)); | |||
5181 | } | |||
5182 | ||||
5183 | // Check if the branch instruction dominates the phi node. It's a simple | |||
5184 | // dominance check, but sufficient for our needs. | |||
5185 | // Although this check is invariant in the calling loops, it's better to do it | |||
5186 | // at this late stage. Practically we do it at most once for a switch. | |||
5187 | BasicBlock *BranchBlock = RangeCheckBranch->getParent(); | |||
5188 | for (auto PI = pred_begin(PhiBlock), E = pred_end(PhiBlock); PI != E; ++PI) { | |||
5189 | BasicBlock *Pred = *PI; | |||
5190 | if (Pred != BranchBlock && Pred->getUniquePredecessor() != BranchBlock) | |||
5191 | return; | |||
5192 | } | |||
5193 | ||||
5194 | if (DefaultConst == FalseConst) { | |||
5195 | // The compare yields the same result. We can replace it. | |||
5196 | CmpInst->replaceAllUsesWith(RangeCmp); | |||
5197 | ++NumTableCmpReuses; | |||
5198 | } else { | |||
5199 | // The compare yields the same result, just inverted. We can replace it. | |||
5200 | Value *InvertedTableCmp = BinaryOperator::CreateXor( | |||
5201 | RangeCmp, ConstantInt::get(RangeCmp->getType(), 1), "inverted.cmp", | |||
5202 | RangeCheckBranch); | |||
5203 | CmpInst->replaceAllUsesWith(InvertedTableCmp); | |||
5204 | ++NumTableCmpReuses; | |||
5205 | } | |||
5206 | } | |||
5207 | ||||
5208 | /// If the switch is only used to initialize one or more phi nodes in a common | |||
5209 | /// successor block with different constant values, replace the switch with | |||
5210 | /// lookup tables. | |||
5211 | static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder, | |||
5212 | const DataLayout &DL, | |||
5213 | const TargetTransformInfo &TTI) { | |||
5214 | assert(SI->getNumCases() > 1 && "Degenerate switch?")(static_cast <bool> (SI->getNumCases() > 1 && "Degenerate switch?") ? void (0) : __assert_fail ("SI->getNumCases() > 1 && \"Degenerate switch?\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5214, __extension__ __PRETTY_FUNCTION__)); | |||
5215 | ||||
5216 | Function *Fn = SI->getParent()->getParent(); | |||
5217 | // Only build lookup table when we have a target that supports it or the | |||
5218 | // attribute is not set. | |||
5219 | if (!TTI.shouldBuildLookupTables() || | |||
5220 | (Fn->getFnAttribute("no-jump-tables").getValueAsString() == "true")) | |||
5221 | return false; | |||
5222 | ||||
5223 | // FIXME: If the switch is too sparse for a lookup table, perhaps we could | |||
5224 | // split off a dense part and build a lookup table for that. | |||
5225 | ||||
5226 | // FIXME: This creates arrays of GEPs to constant strings, which means each | |||
5227 | // GEP needs a runtime relocation in PIC code. We should just build one big | |||
5228 | // string and lookup indices into that. | |||
5229 | ||||
5230 | // Ignore switches with less than three cases. Lookup tables will not make | |||
5231 | // them faster, so we don't analyze them. | |||
5232 | if (SI->getNumCases() < 3) | |||
5233 | return false; | |||
5234 | ||||
5235 | // Figure out the corresponding result for each case value and phi node in the | |||
5236 | // common destination, as well as the min and max case values. | |||
5237 | assert(SI->case_begin() != SI->case_end())(static_cast <bool> (SI->case_begin() != SI->case_end ()) ? void (0) : __assert_fail ("SI->case_begin() != SI->case_end()" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5237, __extension__ __PRETTY_FUNCTION__)); | |||
5238 | SwitchInst::CaseIt CI = SI->case_begin(); | |||
5239 | ConstantInt *MinCaseVal = CI->getCaseValue(); | |||
5240 | ConstantInt *MaxCaseVal = CI->getCaseValue(); | |||
5241 | ||||
5242 | BasicBlock *CommonDest = nullptr; | |||
5243 | ||||
5244 | using ResultListTy = SmallVector<std::pair<ConstantInt *, Constant *>, 4>; | |||
5245 | SmallDenseMap<PHINode *, ResultListTy> ResultLists; | |||
5246 | ||||
5247 | SmallDenseMap<PHINode *, Constant *> DefaultResults; | |||
5248 | SmallDenseMap<PHINode *, Type *> ResultTypes; | |||
5249 | SmallVector<PHINode *, 4> PHIs; | |||
5250 | ||||
5251 | for (SwitchInst::CaseIt E = SI->case_end(); CI != E; ++CI) { | |||
5252 | ConstantInt *CaseVal = CI->getCaseValue(); | |||
5253 | if (CaseVal->getValue().slt(MinCaseVal->getValue())) | |||
5254 | MinCaseVal = CaseVal; | |||
5255 | if (CaseVal->getValue().sgt(MaxCaseVal->getValue())) | |||
5256 | MaxCaseVal = CaseVal; | |||
5257 | ||||
5258 | // Resulting value at phi nodes for this case value. | |||
5259 | using ResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>; | |||
5260 | ResultsTy Results; | |||
5261 | if (!GetCaseResults(SI, CaseVal, CI->getCaseSuccessor(), &CommonDest, | |||
5262 | Results, DL, TTI)) | |||
5263 | return false; | |||
5264 | ||||
5265 | // Append the result from this case to the list for each phi. | |||
5266 | for (const auto &I : Results) { | |||
5267 | PHINode *PHI = I.first; | |||
5268 | Constant *Value = I.second; | |||
5269 | if (!ResultLists.count(PHI)) | |||
5270 | PHIs.push_back(PHI); | |||
5271 | ResultLists[PHI].push_back(std::make_pair(CaseVal, Value)); | |||
5272 | } | |||
5273 | } | |||
5274 | ||||
5275 | // Keep track of the result types. | |||
5276 | for (PHINode *PHI : PHIs) { | |||
5277 | ResultTypes[PHI] = ResultLists[PHI][0].second->getType(); | |||
5278 | } | |||
5279 | ||||
5280 | uint64_t NumResults = ResultLists[PHIs[0]].size(); | |||
5281 | APInt RangeSpread = MaxCaseVal->getValue() - MinCaseVal->getValue(); | |||
5282 | uint64_t TableSize = RangeSpread.getLimitedValue() + 1; | |||
5283 | bool TableHasHoles = (NumResults < TableSize); | |||
5284 | ||||
5285 | // If the table has holes, we need a constant result for the default case | |||
5286 | // or a bitmask that fits in a register. | |||
5287 | SmallVector<std::pair<PHINode *, Constant *>, 4> DefaultResultsList; | |||
5288 | bool HasDefaultResults = | |||
5289 | GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, | |||
5290 | DefaultResultsList, DL, TTI); | |||
5291 | ||||
5292 | bool NeedMask = (TableHasHoles && !HasDefaultResults); | |||
5293 | if (NeedMask) { | |||
5294 | // As an extra penalty for the validity test we require more cases. | |||
5295 | if (SI->getNumCases() < 4) // FIXME: Find best threshold value (benchmark). | |||
5296 | return false; | |||
5297 | if (!DL.fitsInLegalInteger(TableSize)) | |||
5298 | return false; | |||
5299 | } | |||
5300 | ||||
5301 | for (const auto &I : DefaultResultsList) { | |||
5302 | PHINode *PHI = I.first; | |||
5303 | Constant *Result = I.second; | |||
5304 | DefaultResults[PHI] = Result; | |||
5305 | } | |||
5306 | ||||
5307 | if (!ShouldBuildLookupTable(SI, TableSize, TTI, DL, ResultTypes)) | |||
5308 | return false; | |||
5309 | ||||
5310 | // Create the BB that does the lookups. | |||
5311 | Module &Mod = *CommonDest->getParent()->getParent(); | |||
5312 | BasicBlock *LookupBB = BasicBlock::Create( | |||
5313 | Mod.getContext(), "switch.lookup", CommonDest->getParent(), CommonDest); | |||
5314 | ||||
5315 | // Compute the table index value. | |||
5316 | Builder.SetInsertPoint(SI); | |||
5317 | Value *TableIndex; | |||
5318 | if (MinCaseVal->isNullValue()) | |||
5319 | TableIndex = SI->getCondition(); | |||
5320 | else | |||
5321 | TableIndex = Builder.CreateSub(SI->getCondition(), MinCaseVal, | |||
5322 | "switch.tableidx"); | |||
5323 | ||||
5324 | // Compute the maximum table size representable by the integer type we are | |||
5325 | // switching upon. | |||
5326 | unsigned CaseSize = MinCaseVal->getType()->getPrimitiveSizeInBits(); | |||
5327 | uint64_t MaxTableSize = CaseSize > 63 ? UINT64_MAX(18446744073709551615UL) : 1ULL << CaseSize; | |||
5328 | assert(MaxTableSize >= TableSize &&(static_cast <bool> (MaxTableSize >= TableSize && "It is impossible for a switch to have more entries than the max " "representable value of its input integer type's size.") ? void (0) : __assert_fail ("MaxTableSize >= TableSize && \"It is impossible for a switch to have more entries than the max \" \"representable value of its input integer type's size.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5330, __extension__ __PRETTY_FUNCTION__)) | |||
5329 | "It is impossible for a switch to have more entries than the max "(static_cast <bool> (MaxTableSize >= TableSize && "It is impossible for a switch to have more entries than the max " "representable value of its input integer type's size.") ? void (0) : __assert_fail ("MaxTableSize >= TableSize && \"It is impossible for a switch to have more entries than the max \" \"representable value of its input integer type's size.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5330, __extension__ __PRETTY_FUNCTION__)) | |||
5330 | "representable value of its input integer type's size.")(static_cast <bool> (MaxTableSize >= TableSize && "It is impossible for a switch to have more entries than the max " "representable value of its input integer type's size.") ? void (0) : __assert_fail ("MaxTableSize >= TableSize && \"It is impossible for a switch to have more entries than the max \" \"representable value of its input integer type's size.\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5330, __extension__ __PRETTY_FUNCTION__)); | |||
5331 | ||||
5332 | // If the default destination is unreachable, or if the lookup table covers | |||
5333 | // all values of the conditional variable, branch directly to the lookup table | |||
5334 | // BB. Otherwise, check that the condition is within the case range. | |||
5335 | const bool DefaultIsReachable = | |||
5336 | !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg()); | |||
5337 | const bool GeneratingCoveredLookupTable = (MaxTableSize == TableSize); | |||
5338 | BranchInst *RangeCheckBranch = nullptr; | |||
5339 | ||||
5340 | if (!DefaultIsReachable || GeneratingCoveredLookupTable) { | |||
5341 | Builder.CreateBr(LookupBB); | |||
5342 | // Note: We call removeProdecessor later since we need to be able to get the | |||
5343 | // PHI value for the default case in case we're using a bit mask. | |||
5344 | } else { | |||
5345 | Value *Cmp = Builder.CreateICmpULT( | |||
5346 | TableIndex, ConstantInt::get(MinCaseVal->getType(), TableSize)); | |||
5347 | RangeCheckBranch = | |||
5348 | Builder.CreateCondBr(Cmp, LookupBB, SI->getDefaultDest()); | |||
5349 | } | |||
5350 | ||||
5351 | // Populate the BB that does the lookups. | |||
5352 | Builder.SetInsertPoint(LookupBB); | |||
5353 | ||||
5354 | if (NeedMask) { | |||
5355 | // Before doing the lookup, we do the hole check. The LookupBB is therefore | |||
5356 | // re-purposed to do the hole check, and we create a new LookupBB. | |||
5357 | BasicBlock *MaskBB = LookupBB; | |||
5358 | MaskBB->setName("switch.hole_check"); | |||
5359 | LookupBB = BasicBlock::Create(Mod.getContext(), "switch.lookup", | |||
5360 | CommonDest->getParent(), CommonDest); | |||
5361 | ||||
5362 | // Make the mask's bitwidth at least 8-bit and a power-of-2 to avoid | |||
5363 | // unnecessary illegal types. | |||
5364 | uint64_t TableSizePowOf2 = NextPowerOf2(std::max(7ULL, TableSize - 1ULL)); | |||
5365 | APInt MaskInt(TableSizePowOf2, 0); | |||
5366 | APInt One(TableSizePowOf2, 1); | |||
5367 | // Build bitmask; fill in a 1 bit for every case. | |||
5368 | const ResultListTy &ResultList = ResultLists[PHIs[0]]; | |||
5369 | for (size_t I = 0, E = ResultList.size(); I != E; ++I) { | |||
5370 | uint64_t Idx = (ResultList[I].first->getValue() - MinCaseVal->getValue()) | |||
5371 | .getLimitedValue(); | |||
5372 | MaskInt |= One << Idx; | |||
5373 | } | |||
5374 | ConstantInt *TableMask = ConstantInt::get(Mod.getContext(), MaskInt); | |||
5375 | ||||
5376 | // Get the TableIndex'th bit of the bitmask. | |||
5377 | // If this bit is 0 (meaning hole) jump to the default destination, | |||
5378 | // else continue with table lookup. | |||
5379 | IntegerType *MapTy = TableMask->getType(); | |||
5380 | Value *MaskIndex = | |||
5381 | Builder.CreateZExtOrTrunc(TableIndex, MapTy, "switch.maskindex"); | |||
5382 | Value *Shifted = Builder.CreateLShr(TableMask, MaskIndex, "switch.shifted"); | |||
5383 | Value *LoBit = Builder.CreateTrunc( | |||
5384 | Shifted, Type::getInt1Ty(Mod.getContext()), "switch.lobit"); | |||
5385 | Builder.CreateCondBr(LoBit, LookupBB, SI->getDefaultDest()); | |||
5386 | ||||
5387 | Builder.SetInsertPoint(LookupBB); | |||
5388 | AddPredecessorToBlock(SI->getDefaultDest(), MaskBB, SI->getParent()); | |||
5389 | } | |||
5390 | ||||
5391 | if (!DefaultIsReachable || GeneratingCoveredLookupTable) { | |||
5392 | // We cached PHINodes in PHIs. To avoid accessing deleted PHINodes later, | |||
5393 | // do not delete PHINodes here. | |||
5394 | SI->getDefaultDest()->removePredecessor(SI->getParent(), | |||
5395 | /*DontDeleteUselessPHIs=*/true); | |||
5396 | } | |||
5397 | ||||
5398 | bool ReturnedEarly = false; | |||
5399 | for (size_t I = 0, E = PHIs.size(); I != E; ++I) { | |||
5400 | PHINode *PHI = PHIs[I]; | |||
5401 | const ResultListTy &ResultList = ResultLists[PHI]; | |||
5402 | ||||
5403 | // If using a bitmask, use any value to fill the lookup table holes. | |||
5404 | Constant *DV = NeedMask ? ResultLists[PHI][0].second : DefaultResults[PHI]; | |||
5405 | StringRef FuncName = Fn->getName(); | |||
5406 | SwitchLookupTable Table(Mod, TableSize, MinCaseVal, ResultList, DV, DL, | |||
5407 | FuncName); | |||
5408 | ||||
5409 | Value *Result = Table.BuildLookup(TableIndex, Builder); | |||
5410 | ||||
5411 | // If the result is used to return immediately from the function, we want to | |||
5412 | // do that right here. | |||
5413 | if (PHI->hasOneUse() && isa<ReturnInst>(*PHI->user_begin()) && | |||
5414 | PHI->user_back() == CommonDest->getFirstNonPHIOrDbg()) { | |||
5415 | Builder.CreateRet(Result); | |||
5416 | ReturnedEarly = true; | |||
5417 | break; | |||
5418 | } | |||
5419 | ||||
5420 | // Do a small peephole optimization: re-use the switch table compare if | |||
5421 | // possible. | |||
5422 | if (!TableHasHoles && HasDefaultResults && RangeCheckBranch) { | |||
5423 | BasicBlock *PhiBlock = PHI->getParent(); | |||
5424 | // Search for compare instructions which use the phi. | |||
5425 | for (auto *User : PHI->users()) { | |||
5426 | reuseTableCompare(User, PhiBlock, RangeCheckBranch, DV, ResultList); | |||
5427 | } | |||
5428 | } | |||
5429 | ||||
5430 | PHI->addIncoming(Result, LookupBB); | |||
5431 | } | |||
5432 | ||||
5433 | if (!ReturnedEarly) | |||
5434 | Builder.CreateBr(CommonDest); | |||
5435 | ||||
5436 | // Remove the switch. | |||
5437 | for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) { | |||
5438 | BasicBlock *Succ = SI->getSuccessor(i); | |||
5439 | ||||
5440 | if (Succ == SI->getDefaultDest()) | |||
5441 | continue; | |||
5442 | Succ->removePredecessor(SI->getParent()); | |||
5443 | } | |||
5444 | SI->eraseFromParent(); | |||
5445 | ||||
5446 | ++NumLookupTables; | |||
5447 | if (NeedMask) | |||
5448 | ++NumLookupTablesHoles; | |||
5449 | return true; | |||
5450 | } | |||
5451 | ||||
5452 | static bool isSwitchDense(ArrayRef<int64_t> Values) { | |||
5453 | // See also SelectionDAGBuilder::isDense(), which this function was based on. | |||
5454 | uint64_t Diff = (uint64_t)Values.back() - (uint64_t)Values.front(); | |||
5455 | uint64_t Range = Diff + 1; | |||
5456 | uint64_t NumCases = Values.size(); | |||
5457 | // 40% is the default density for building a jump table in optsize/minsize mode. | |||
5458 | uint64_t MinDensity = 40; | |||
5459 | ||||
5460 | return NumCases * 100 >= Range * MinDensity; | |||
5461 | } | |||
5462 | ||||
5463 | /// Try to transform a switch that has "holes" in it to a contiguous sequence | |||
5464 | /// of cases. | |||
5465 | /// | |||
5466 | /// A switch such as: switch(i) {case 5: case 9: case 13: case 17:} can be | |||
5467 | /// range-reduced to: switch ((i-5) / 4) {case 0: case 1: case 2: case 3:}. | |||
5468 | /// | |||
5469 | /// This converts a sparse switch into a dense switch which allows better | |||
5470 | /// lowering and could also allow transforming into a lookup table. | |||
5471 | static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder, | |||
5472 | const DataLayout &DL, | |||
5473 | const TargetTransformInfo &TTI) { | |||
5474 | auto *CondTy = cast<IntegerType>(SI->getCondition()->getType()); | |||
5475 | if (CondTy->getIntegerBitWidth() > 64 || | |||
5476 | !DL.fitsInLegalInteger(CondTy->getIntegerBitWidth())) | |||
5477 | return false; | |||
5478 | // Only bother with this optimization if there are more than 3 switch cases; | |||
5479 | // SDAG will only bother creating jump tables for 4 or more cases. | |||
5480 | if (SI->getNumCases() < 4) | |||
5481 | return false; | |||
5482 | ||||
5483 | // This transform is agnostic to the signedness of the input or case values. We | |||
5484 | // can treat the case values as signed or unsigned. We can optimize more common | |||
5485 | // cases such as a sequence crossing zero {-4,0,4,8} if we interpret case values | |||
5486 | // as signed. | |||
5487 | SmallVector<int64_t,4> Values; | |||
5488 | for (auto &C : SI->cases()) | |||
5489 | Values.push_back(C.getCaseValue()->getValue().getSExtValue()); | |||
5490 | std::sort(Values.begin(), Values.end()); | |||
5491 | ||||
5492 | // If the switch is already dense, there's nothing useful to do here. | |||
5493 | if (isSwitchDense(Values)) | |||
5494 | return false; | |||
5495 | ||||
5496 | // First, transform the values such that they start at zero and ascend. | |||
5497 | int64_t Base = Values[0]; | |||
5498 | for (auto &V : Values) | |||
5499 | V -= (uint64_t)(Base); | |||
5500 | ||||
5501 | // Now we have signed numbers that have been shifted so that, given enough | |||
5502 | // precision, there are no negative values. Since the rest of the transform | |||
5503 | // is bitwise only, we switch now to an unsigned representation. | |||
5504 | uint64_t GCD = 0; | |||
5505 | for (auto &V : Values) | |||
5506 | GCD = GreatestCommonDivisor64(GCD, (uint64_t)V); | |||
5507 | ||||
5508 | // This transform can be done speculatively because it is so cheap - it results | |||
5509 | // in a single rotate operation being inserted. This can only happen if the | |||
5510 | // factor extracted is a power of 2. | |||
5511 | // FIXME: If the GCD is an odd number we can multiply by the multiplicative | |||
5512 | // inverse of GCD and then perform this transform. | |||
5513 | // FIXME: It's possible that optimizing a switch on powers of two might also | |||
5514 | // be beneficial - flag values are often powers of two and we could use a CLZ | |||
5515 | // as the key function. | |||
5516 | if (GCD <= 1 || !isPowerOf2_64(GCD)) | |||
5517 | // No common divisor found or too expensive to compute key function. | |||
5518 | return false; | |||
5519 | ||||
5520 | unsigned Shift = Log2_64(GCD); | |||
5521 | for (auto &V : Values) | |||
5522 | V = (int64_t)((uint64_t)V >> Shift); | |||
5523 | ||||
5524 | if (!isSwitchDense(Values)) | |||
5525 | // Transform didn't create a dense switch. | |||
5526 | return false; | |||
5527 | ||||
5528 | // The obvious transform is to shift the switch condition right and emit a | |||
5529 | // check that the condition actually cleanly divided by GCD, i.e. | |||
5530 | // C & (1 << Shift - 1) == 0 | |||
5531 | // inserting a new CFG edge to handle the case where it didn't divide cleanly. | |||
5532 | // | |||
5533 | // A cheaper way of doing this is a simple ROTR(C, Shift). This performs the | |||
5534 | // shift and puts the shifted-off bits in the uppermost bits. If any of these | |||
5535 | // are nonzero then the switch condition will be very large and will hit the | |||
5536 | // default case. | |||
5537 | ||||
5538 | auto *Ty = cast<IntegerType>(SI->getCondition()->getType()); | |||
5539 | Builder.SetInsertPoint(SI); | |||
5540 | auto *ShiftC = ConstantInt::get(Ty, Shift); | |||
5541 | auto *Sub = Builder.CreateSub(SI->getCondition(), ConstantInt::get(Ty, Base)); | |||
5542 | auto *LShr = Builder.CreateLShr(Sub, ShiftC); | |||
5543 | auto *Shl = Builder.CreateShl(Sub, Ty->getBitWidth() - Shift); | |||
5544 | auto *Rot = Builder.CreateOr(LShr, Shl); | |||
5545 | SI->replaceUsesOfWith(SI->getCondition(), Rot); | |||
5546 | ||||
5547 | for (auto Case : SI->cases()) { | |||
5548 | auto *Orig = Case.getCaseValue(); | |||
5549 | auto Sub = Orig->getValue() - APInt(Ty->getBitWidth(), Base); | |||
5550 | Case.setValue( | |||
5551 | cast<ConstantInt>(ConstantInt::get(Ty, Sub.lshr(ShiftC->getValue())))); | |||
5552 | } | |||
5553 | return true; | |||
5554 | } | |||
5555 | ||||
5556 | bool SimplifyCFGOpt::SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) { | |||
5557 | BasicBlock *BB = SI->getParent(); | |||
5558 | ||||
5559 | if (isValueEqualityComparison(SI)) { | |||
5560 | // If we only have one predecessor, and if it is a branch on this value, | |||
5561 | // see if that predecessor totally determines the outcome of this switch. | |||
5562 | if (BasicBlock *OnlyPred = BB->getSinglePredecessor()) | |||
5563 | if (SimplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder)) | |||
5564 | return simplifyCFG(BB, TTI, Options) | true; | |||
5565 | ||||
5566 | Value *Cond = SI->getCondition(); | |||
5567 | if (SelectInst *Select = dyn_cast<SelectInst>(Cond)) | |||
5568 | if (SimplifySwitchOnSelect(SI, Select)) | |||
5569 | return simplifyCFG(BB, TTI, Options) | true; | |||
5570 | ||||
5571 | // If the block only contains the switch, see if we can fold the block | |||
5572 | // away into any preds. | |||
5573 | BasicBlock::iterator BBI = BB->begin(); | |||
5574 | // Ignore dbg intrinsics. | |||
5575 | while (isa<DbgInfoIntrinsic>(BBI)) | |||
5576 | ++BBI; | |||
5577 | if (SI == &*BBI) | |||
5578 | if (FoldValueComparisonIntoPredecessors(SI, Builder)) | |||
5579 | return simplifyCFG(BB, TTI, Options) | true; | |||
5580 | } | |||
5581 | ||||
5582 | // Try to transform the switch into an icmp and a branch. | |||
5583 | if (TurnSwitchRangeIntoICmp(SI, Builder)) | |||
5584 | return simplifyCFG(BB, TTI, Options) | true; | |||
5585 | ||||
5586 | // Remove unreachable cases. | |||
5587 | if (eliminateDeadSwitchCases(SI, Options.AC, DL)) | |||
5588 | return simplifyCFG(BB, TTI, Options) | true; | |||
5589 | ||||
5590 | if (switchToSelect(SI, Builder, DL, TTI)) | |||
5591 | return simplifyCFG(BB, TTI, Options) | true; | |||
5592 | ||||
5593 | if (Options.ForwardSwitchCondToPhi && ForwardSwitchConditionToPHI(SI)) | |||
5594 | return simplifyCFG(BB, TTI, Options) | true; | |||
5595 | ||||
5596 | // The conversion from switch to lookup tables results in difficult-to-analyze | |||
5597 | // code and makes pruning branches much harder. This is a problem if the | |||
5598 | // switch expression itself can still be restricted as a result of inlining or | |||
5599 | // CVP. Therefore, only apply this transformation during late stages of the | |||
5600 | // optimisation pipeline. | |||
5601 | if (Options.ConvertSwitchToLookupTable && | |||
5602 | SwitchToLookupTable(SI, Builder, DL, TTI)) | |||
5603 | return simplifyCFG(BB, TTI, Options) | true; | |||
5604 | ||||
5605 | if (ReduceSwitchRange(SI, Builder, DL, TTI)) | |||
5606 | return simplifyCFG(BB, TTI, Options) | true; | |||
5607 | ||||
5608 | return false; | |||
5609 | } | |||
5610 | ||||
5611 | bool SimplifyCFGOpt::SimplifyIndirectBr(IndirectBrInst *IBI) { | |||
5612 | BasicBlock *BB = IBI->getParent(); | |||
5613 | bool Changed = false; | |||
5614 | ||||
5615 | // Eliminate redundant destinations. | |||
5616 | SmallPtrSet<Value *, 8> Succs; | |||
5617 | for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; ++i) { | |||
5618 | BasicBlock *Dest = IBI->getDestination(i); | |||
5619 | if (!Dest->hasAddressTaken() || !Succs.insert(Dest).second) { | |||
5620 | Dest->removePredecessor(BB); | |||
5621 | IBI->removeDestination(i); | |||
5622 | --i; | |||
5623 | --e; | |||
5624 | Changed = true; | |||
5625 | } | |||
5626 | } | |||
5627 | ||||
5628 | if (IBI->getNumDestinations() == 0) { | |||
5629 | // If the indirectbr has no successors, change it to unreachable. | |||
5630 | new UnreachableInst(IBI->getContext(), IBI); | |||
5631 | EraseTerminatorInstAndDCECond(IBI); | |||
5632 | return true; | |||
5633 | } | |||
5634 | ||||
5635 | if (IBI->getNumDestinations() == 1) { | |||
5636 | // If the indirectbr has one successor, change it to a direct branch. | |||
5637 | BranchInst::Create(IBI->getDestination(0), IBI); | |||
5638 | EraseTerminatorInstAndDCECond(IBI); | |||
5639 | return true; | |||
5640 | } | |||
5641 | ||||
5642 | if (SelectInst *SI = dyn_cast<SelectInst>(IBI->getAddress())) { | |||
5643 | if (SimplifyIndirectBrOnSelect(IBI, SI)) | |||
5644 | return simplifyCFG(BB, TTI, Options) | true; | |||
5645 | } | |||
5646 | return Changed; | |||
5647 | } | |||
5648 | ||||
5649 | /// Given an block with only a single landing pad and a unconditional branch | |||
5650 | /// try to find another basic block which this one can be merged with. This | |||
5651 | /// handles cases where we have multiple invokes with unique landing pads, but | |||
5652 | /// a shared handler. | |||
5653 | /// | |||
5654 | /// We specifically choose to not worry about merging non-empty blocks | |||
5655 | /// here. That is a PRE/scheduling problem and is best solved elsewhere. In | |||
5656 | /// practice, the optimizer produces empty landing pad blocks quite frequently | |||
5657 | /// when dealing with exception dense code. (see: instcombine, gvn, if-else | |||
5658 | /// sinking in this file) | |||
5659 | /// | |||
5660 | /// This is primarily a code size optimization. We need to avoid performing | |||
5661 | /// any transform which might inhibit optimization (such as our ability to | |||
5662 | /// specialize a particular handler via tail commoning). We do this by not | |||
5663 | /// merging any blocks which require us to introduce a phi. Since the same | |||
5664 | /// values are flowing through both blocks, we don't loose any ability to | |||
5665 | /// specialize. If anything, we make such specialization more likely. | |||
5666 | /// | |||
5667 | /// TODO - This transformation could remove entries from a phi in the target | |||
5668 | /// block when the inputs in the phi are the same for the two blocks being | |||
5669 | /// merged. In some cases, this could result in removal of the PHI entirely. | |||
5670 | static bool TryToMergeLandingPad(LandingPadInst *LPad, BranchInst *BI, | |||
5671 | BasicBlock *BB) { | |||
5672 | auto Succ = BB->getUniqueSuccessor(); | |||
5673 | assert(Succ)(static_cast <bool> (Succ) ? void (0) : __assert_fail ( "Succ", "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5673, __extension__ __PRETTY_FUNCTION__)); | |||
5674 | // If there's a phi in the successor block, we'd likely have to introduce | |||
5675 | // a phi into the merged landing pad block. | |||
5676 | if (isa<PHINode>(*Succ->begin())) | |||
5677 | return false; | |||
5678 | ||||
5679 | for (BasicBlock *OtherPred : predecessors(Succ)) { | |||
5680 | if (BB == OtherPred) | |||
5681 | continue; | |||
5682 | BasicBlock::iterator I = OtherPred->begin(); | |||
5683 | LandingPadInst *LPad2 = dyn_cast<LandingPadInst>(I); | |||
5684 | if (!LPad2 || !LPad2->isIdenticalTo(LPad)) | |||
5685 | continue; | |||
5686 | for (++I; isa<DbgInfoIntrinsic>(I); ++I) | |||
5687 | ; | |||
5688 | BranchInst *BI2 = dyn_cast<BranchInst>(I); | |||
5689 | if (!BI2 || !BI2->isIdenticalTo(BI)) | |||
5690 | continue; | |||
5691 | ||||
5692 | // We've found an identical block. Update our predecessors to take that | |||
5693 | // path instead and make ourselves dead. | |||
5694 | SmallSet<BasicBlock *, 16> Preds; | |||
5695 | Preds.insert(pred_begin(BB), pred_end(BB)); | |||
5696 | for (BasicBlock *Pred : Preds) { | |||
5697 | InvokeInst *II = cast<InvokeInst>(Pred->getTerminator()); | |||
5698 | assert(II->getNormalDest() != BB && II->getUnwindDest() == BB &&(static_cast <bool> (II->getNormalDest() != BB && II->getUnwindDest() == BB && "unexpected successor" ) ? void (0) : __assert_fail ("II->getNormalDest() != BB && II->getUnwindDest() == BB && \"unexpected successor\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5699, __extension__ __PRETTY_FUNCTION__)) | |||
5699 | "unexpected successor")(static_cast <bool> (II->getNormalDest() != BB && II->getUnwindDest() == BB && "unexpected successor" ) ? void (0) : __assert_fail ("II->getNormalDest() != BB && II->getUnwindDest() == BB && \"unexpected successor\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5699, __extension__ __PRETTY_FUNCTION__)); | |||
5700 | II->setUnwindDest(OtherPred); | |||
5701 | } | |||
5702 | ||||
5703 | // The debug info in OtherPred doesn't cover the merged control flow that | |||
5704 | // used to go through BB. We need to delete it or update it. | |||
5705 | for (auto I = OtherPred->begin(), E = OtherPred->end(); I != E;) { | |||
5706 | Instruction &Inst = *I; | |||
5707 | I++; | |||
5708 | if (isa<DbgInfoIntrinsic>(Inst)) | |||
5709 | Inst.eraseFromParent(); | |||
5710 | } | |||
5711 | ||||
5712 | SmallSet<BasicBlock *, 16> Succs; | |||
5713 | Succs.insert(succ_begin(BB), succ_end(BB)); | |||
5714 | for (BasicBlock *Succ : Succs) { | |||
5715 | Succ->removePredecessor(BB); | |||
5716 | } | |||
5717 | ||||
5718 | IRBuilder<> Builder(BI); | |||
5719 | Builder.CreateUnreachable(); | |||
5720 | BI->eraseFromParent(); | |||
5721 | return true; | |||
5722 | } | |||
5723 | return false; | |||
5724 | } | |||
5725 | ||||
5726 | bool SimplifyCFGOpt::SimplifyUncondBranch(BranchInst *BI, | |||
5727 | IRBuilder<> &Builder) { | |||
5728 | BasicBlock *BB = BI->getParent(); | |||
5729 | BasicBlock *Succ = BI->getSuccessor(0); | |||
5730 | ||||
5731 | // If the Terminator is the only non-phi instruction, simplify the block. | |||
5732 | // If LoopHeader is provided, check if the block or its successor is a loop | |||
5733 | // header. (This is for early invocations before loop simplify and | |||
5734 | // vectorization to keep canonical loop forms for nested loops. These blocks | |||
5735 | // can be eliminated when the pass is invoked later in the back-end.) | |||
5736 | // Note that if BB has only one predecessor then we do not introduce new | |||
5737 | // backedge, so we can eliminate BB. | |||
5738 | bool NeedCanonicalLoop = | |||
5739 | Options.NeedCanonicalLoop && | |||
5740 | (LoopHeaders && std::distance(pred_begin(BB), pred_end(BB)) > 1 && | |||
5741 | (LoopHeaders->count(BB) || LoopHeaders->count(Succ))); | |||
5742 | BasicBlock::iterator I = BB->getFirstNonPHIOrDbg()->getIterator(); | |||
5743 | if (I->isTerminator() && BB != &BB->getParent()->getEntryBlock() && | |||
5744 | !NeedCanonicalLoop && TryToSimplifyUncondBranchFromEmptyBlock(BB)) | |||
5745 | return true; | |||
5746 | ||||
5747 | // If the only instruction in the block is a seteq/setne comparison against a | |||
5748 | // constant, try to simplify the block. | |||
5749 | if (ICmpInst *ICI = dyn_cast<ICmpInst>(I)) | |||
5750 | if (ICI->isEquality() && isa<ConstantInt>(ICI->getOperand(1))) { | |||
5751 | for (++I; isa<DbgInfoIntrinsic>(I); ++I) | |||
5752 | ; | |||
5753 | if (I->isTerminator() && | |||
5754 | tryToSimplifyUncondBranchWithICmpInIt(ICI, Builder, DL, TTI, Options)) | |||
5755 | return true; | |||
5756 | } | |||
5757 | ||||
5758 | // See if we can merge an empty landing pad block with another which is | |||
5759 | // equivalent. | |||
5760 | if (LandingPadInst *LPad = dyn_cast<LandingPadInst>(I)) { | |||
5761 | for (++I; isa<DbgInfoIntrinsic>(I); ++I) | |||
5762 | ; | |||
5763 | if (I->isTerminator() && TryToMergeLandingPad(LPad, BI, BB)) | |||
5764 | return true; | |||
5765 | } | |||
5766 | ||||
5767 | // If this basic block is ONLY a compare and a branch, and if a predecessor | |||
5768 | // branches to us and our successor, fold the comparison into the | |||
5769 | // predecessor and use logical operations to update the incoming value | |||
5770 | // for PHI nodes in common successor. | |||
5771 | if (FoldBranchToCommonDest(BI, Options.BonusInstThreshold)) | |||
5772 | return simplifyCFG(BB, TTI, Options) | true; | |||
5773 | return false; | |||
5774 | } | |||
5775 | ||||
5776 | static BasicBlock *allPredecessorsComeFromSameSource(BasicBlock *BB) { | |||
5777 | BasicBlock *PredPred = nullptr; | |||
5778 | for (auto *P : predecessors(BB)) { | |||
5779 | BasicBlock *PPred = P->getSinglePredecessor(); | |||
5780 | if (!PPred || (PredPred && PredPred != PPred)) | |||
5781 | return nullptr; | |||
5782 | PredPred = PPred; | |||
5783 | } | |||
5784 | return PredPred; | |||
5785 | } | |||
5786 | ||||
5787 | bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) { | |||
5788 | BasicBlock *BB = BI->getParent(); | |||
5789 | ||||
5790 | // Conditional branch | |||
5791 | if (isValueEqualityComparison(BI)) { | |||
5792 | // If we only have one predecessor, and if it is a branch on this value, | |||
5793 | // see if that predecessor totally determines the outcome of this | |||
5794 | // switch. | |||
5795 | if (BasicBlock *OnlyPred = BB->getSinglePredecessor()) | |||
5796 | if (SimplifyEqualityComparisonWithOnlyPredecessor(BI, OnlyPred, Builder)) | |||
5797 | return simplifyCFG(BB, TTI, Options) | true; | |||
5798 | ||||
5799 | // This block must be empty, except for the setcond inst, if it exists. | |||
5800 | // Ignore dbg intrinsics. | |||
5801 | BasicBlock::iterator I = BB->begin(); | |||
5802 | // Ignore dbg intrinsics. | |||
5803 | while (isa<DbgInfoIntrinsic>(I)) | |||
5804 | ++I; | |||
5805 | if (&*I == BI) { | |||
5806 | if (FoldValueComparisonIntoPredecessors(BI, Builder)) | |||
5807 | return simplifyCFG(BB, TTI, Options) | true; | |||
5808 | } else if (&*I == cast<Instruction>(BI->getCondition())) { | |||
5809 | ++I; | |||
5810 | // Ignore dbg intrinsics. | |||
5811 | while (isa<DbgInfoIntrinsic>(I)) | |||
5812 | ++I; | |||
5813 | if (&*I == BI && FoldValueComparisonIntoPredecessors(BI, Builder)) | |||
5814 | return simplifyCFG(BB, TTI, Options) | true; | |||
5815 | } | |||
5816 | } | |||
5817 | ||||
5818 | // Try to turn "br (X == 0 | X == 1), T, F" into a switch instruction. | |||
5819 | if (SimplifyBranchOnICmpChain(BI, Builder, DL)) | |||
5820 | return true; | |||
5821 | ||||
5822 | // If this basic block has a single dominating predecessor block and the | |||
5823 | // dominating block's condition implies BI's condition, we know the direction | |||
5824 | // of the BI branch. | |||
5825 | if (BasicBlock *Dom = BB->getSinglePredecessor()) { | |||
5826 | auto *PBI = dyn_cast_or_null<BranchInst>(Dom->getTerminator()); | |||
5827 | if (PBI && PBI->isConditional() && | |||
5828 | PBI->getSuccessor(0) != PBI->getSuccessor(1)) { | |||
5829 | assert(PBI->getSuccessor(0) == BB || PBI->getSuccessor(1) == BB)(static_cast <bool> (PBI->getSuccessor(0) == BB || PBI ->getSuccessor(1) == BB) ? void (0) : __assert_fail ("PBI->getSuccessor(0) == BB || PBI->getSuccessor(1) == BB" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5829, __extension__ __PRETTY_FUNCTION__)); | |||
5830 | bool CondIsTrue = PBI->getSuccessor(0) == BB; | |||
5831 | Optional<bool> Implication = isImpliedCondition( | |||
5832 | PBI->getCondition(), BI->getCondition(), DL, CondIsTrue); | |||
5833 | if (Implication) { | |||
5834 | // Turn this into a branch on constant. | |||
5835 | auto *OldCond = BI->getCondition(); | |||
5836 | ConstantInt *CI = *Implication | |||
5837 | ? ConstantInt::getTrue(BB->getContext()) | |||
5838 | : ConstantInt::getFalse(BB->getContext()); | |||
5839 | BI->setCondition(CI); | |||
5840 | RecursivelyDeleteTriviallyDeadInstructions(OldCond); | |||
5841 | return simplifyCFG(BB, TTI, Options) | true; | |||
5842 | } | |||
5843 | } | |||
5844 | } | |||
5845 | ||||
5846 | // If this basic block is ONLY a compare and a branch, and if a predecessor | |||
5847 | // branches to us and one of our successors, fold the comparison into the | |||
5848 | // predecessor and use logical operations to pick the right destination. | |||
5849 | if (FoldBranchToCommonDest(BI, Options.BonusInstThreshold)) | |||
5850 | return simplifyCFG(BB, TTI, Options) | true; | |||
5851 | ||||
5852 | // We have a conditional branch to two blocks that are only reachable | |||
5853 | // from BI. We know that the condbr dominates the two blocks, so see if | |||
5854 | // there is any identical code in the "then" and "else" blocks. If so, we | |||
5855 | // can hoist it up to the branching block. | |||
5856 | if (BI->getSuccessor(0)->getSinglePredecessor()) { | |||
5857 | if (BI->getSuccessor(1)->getSinglePredecessor()) { | |||
5858 | if (HoistThenElseCodeToIf(BI, TTI)) | |||
5859 | return simplifyCFG(BB, TTI, Options) | true; | |||
5860 | } else { | |||
5861 | // If Successor #1 has multiple preds, we may be able to conditionally | |||
5862 | // execute Successor #0 if it branches to Successor #1. | |||
5863 | TerminatorInst *Succ0TI = BI->getSuccessor(0)->getTerminator(); | |||
5864 | if (Succ0TI->getNumSuccessors() == 1 && | |||
5865 | Succ0TI->getSuccessor(0) == BI->getSuccessor(1)) | |||
5866 | if (SpeculativelyExecuteBB(BI, BI->getSuccessor(0), TTI)) | |||
5867 | return simplifyCFG(BB, TTI, Options) | true; | |||
5868 | } | |||
5869 | } else if (BI->getSuccessor(1)->getSinglePredecessor()) { | |||
5870 | // If Successor #0 has multiple preds, we may be able to conditionally | |||
5871 | // execute Successor #1 if it branches to Successor #0. | |||
5872 | TerminatorInst *Succ1TI = BI->getSuccessor(1)->getTerminator(); | |||
5873 | if (Succ1TI->getNumSuccessors() == 1 && | |||
5874 | Succ1TI->getSuccessor(0) == BI->getSuccessor(0)) | |||
5875 | if (SpeculativelyExecuteBB(BI, BI->getSuccessor(1), TTI)) | |||
5876 | return simplifyCFG(BB, TTI, Options) | true; | |||
5877 | } | |||
5878 | ||||
5879 | // If this is a branch on a phi node in the current block, thread control | |||
5880 | // through this block if any PHI node entries are constants. | |||
5881 | if (PHINode *PN = dyn_cast<PHINode>(BI->getCondition())) | |||
5882 | if (PN->getParent() == BI->getParent()) | |||
5883 | if (FoldCondBranchOnPHI(BI, DL, Options.AC)) | |||
5884 | return simplifyCFG(BB, TTI, Options) | true; | |||
5885 | ||||
5886 | // Scan predecessor blocks for conditional branches. | |||
5887 | for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) | |||
5888 | if (BranchInst *PBI = dyn_cast<BranchInst>((*PI)->getTerminator())) | |||
5889 | if (PBI != BI && PBI->isConditional()) | |||
5890 | if (SimplifyCondBranchToCondBranch(PBI, BI, DL)) | |||
5891 | return simplifyCFG(BB, TTI, Options) | true; | |||
5892 | ||||
5893 | // Look for diamond patterns. | |||
5894 | if (MergeCondStores) | |||
5895 | if (BasicBlock *PrevBB = allPredecessorsComeFromSameSource(BB)) | |||
5896 | if (BranchInst *PBI = dyn_cast<BranchInst>(PrevBB->getTerminator())) | |||
5897 | if (PBI != BI && PBI->isConditional()) | |||
5898 | if (mergeConditionalStores(PBI, BI, DL)) | |||
5899 | return simplifyCFG(BB, TTI, Options) | true; | |||
5900 | ||||
5901 | return false; | |||
5902 | } | |||
5903 | ||||
5904 | /// Check if passing a value to an instruction will cause undefined behavior. | |||
5905 | static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I) { | |||
5906 | Constant *C = dyn_cast<Constant>(V); | |||
5907 | if (!C) | |||
5908 | return false; | |||
5909 | ||||
5910 | if (I->use_empty()) | |||
5911 | return false; | |||
5912 | ||||
5913 | if (C->isNullValue() || isa<UndefValue>(C)) { | |||
5914 | // Only look at the first use, avoid hurting compile time with long uselists | |||
5915 | User *Use = *I->user_begin(); | |||
5916 | ||||
5917 | // Now make sure that there are no instructions in between that can alter | |||
5918 | // control flow (eg. calls) | |||
5919 | for (BasicBlock::iterator | |||
5920 | i = ++BasicBlock::iterator(I), | |||
5921 | UI = BasicBlock::iterator(dyn_cast<Instruction>(Use)); | |||
5922 | i != UI; ++i) | |||
5923 | if (i == I->getParent()->end() || i->mayHaveSideEffects()) | |||
5924 | return false; | |||
5925 | ||||
5926 | // Look through GEPs. A load from a GEP derived from NULL is still undefined | |||
5927 | if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Use)) | |||
5928 | if (GEP->getPointerOperand() == I) | |||
5929 | return passingValueIsAlwaysUndefined(V, GEP); | |||
5930 | ||||
5931 | // Look through bitcasts. | |||
5932 | if (BitCastInst *BC = dyn_cast<BitCastInst>(Use)) | |||
5933 | return passingValueIsAlwaysUndefined(V, BC); | |||
5934 | ||||
5935 | // Load from null is undefined. | |||
5936 | if (LoadInst *LI = dyn_cast<LoadInst>(Use)) | |||
5937 | if (!LI->isVolatile()) | |||
5938 | return LI->getPointerAddressSpace() == 0; | |||
5939 | ||||
5940 | // Store to null is undefined. | |||
5941 | if (StoreInst *SI = dyn_cast<StoreInst>(Use)) | |||
5942 | if (!SI->isVolatile()) | |||
5943 | return SI->getPointerAddressSpace() == 0 && | |||
5944 | SI->getPointerOperand() == I; | |||
5945 | ||||
5946 | // A call to null is undefined. | |||
5947 | if (auto CS = CallSite(Use)) | |||
5948 | return CS.getCalledValue() == I; | |||
5949 | } | |||
5950 | return false; | |||
5951 | } | |||
5952 | ||||
5953 | /// If BB has an incoming value that will always trigger undefined behavior | |||
5954 | /// (eg. null pointer dereference), remove the branch leading here. | |||
5955 | static bool removeUndefIntroducingPredecessor(BasicBlock *BB) { | |||
5956 | for (PHINode &PHI : BB->phis()) | |||
5957 | for (unsigned i = 0, e = PHI.getNumIncomingValues(); i != e; ++i) | |||
5958 | if (passingValueIsAlwaysUndefined(PHI.getIncomingValue(i), &PHI)) { | |||
5959 | TerminatorInst *T = PHI.getIncomingBlock(i)->getTerminator(); | |||
5960 | IRBuilder<> Builder(T); | |||
5961 | if (BranchInst *BI = dyn_cast<BranchInst>(T)) { | |||
5962 | BB->removePredecessor(PHI.getIncomingBlock(i)); | |||
5963 | // Turn uncoditional branches into unreachables and remove the dead | |||
5964 | // destination from conditional branches. | |||
5965 | if (BI->isUnconditional()) | |||
5966 | Builder.CreateUnreachable(); | |||
5967 | else | |||
5968 | Builder.CreateBr(BI->getSuccessor(0) == BB ? BI->getSuccessor(1) | |||
5969 | : BI->getSuccessor(0)); | |||
5970 | BI->eraseFromParent(); | |||
5971 | return true; | |||
5972 | } | |||
5973 | // TODO: SwitchInst. | |||
5974 | } | |||
5975 | ||||
5976 | return false; | |||
5977 | } | |||
5978 | ||||
5979 | bool SimplifyCFGOpt::run(BasicBlock *BB) { | |||
5980 | bool Changed = false; | |||
5981 | ||||
5982 | assert(BB && BB->getParent() && "Block not embedded in function!")(static_cast <bool> (BB && BB->getParent() && "Block not embedded in function!") ? void (0) : __assert_fail ("BB && BB->getParent() && \"Block not embedded in function!\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5982, __extension__ __PRETTY_FUNCTION__)); | |||
5983 | assert(BB->getTerminator() && "Degenerate basic block encountered!")(static_cast <bool> (BB->getTerminator() && "Degenerate basic block encountered!" ) ? void (0) : __assert_fail ("BB->getTerminator() && \"Degenerate basic block encountered!\"" , "/build/llvm-toolchain-snapshot-7~svn326551/lib/Transforms/Utils/SimplifyCFG.cpp" , 5983, __extension__ __PRETTY_FUNCTION__)); | |||
5984 | ||||
5985 | // Remove basic blocks that have no predecessors (except the entry block)... | |||
5986 | // or that just have themself as a predecessor. These are unreachable. | |||
5987 | if ((pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) || | |||
5988 | BB->getSinglePredecessor() == BB) { | |||
5989 | DEBUG(dbgs() << "Removing BB: \n" << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("simplifycfg")) { dbgs() << "Removing BB: \n" << *BB; } } while (false); | |||
5990 | DeleteDeadBlock(BB); | |||
5991 | return true; | |||
5992 | } | |||
5993 | ||||
5994 | // Check to see if we can constant propagate this terminator instruction | |||
5995 | // away... | |||
5996 | Changed |= ConstantFoldTerminator(BB, true); | |||
5997 | ||||
5998 | // Check for and eliminate duplicate PHI nodes in this block. | |||
5999 | Changed |= EliminateDuplicatePHINodes(BB); | |||
6000 | ||||
6001 | // Check for and remove branches that will always cause undefined behavior. | |||
6002 | Changed |= removeUndefIntroducingPredecessor(BB); | |||
6003 | ||||
6004 | // Merge basic blocks into their predecessor if there is only one distinct | |||
6005 | // pred, and if there is only one distinct successor of the predecessor, and | |||
6006 | // if there are no PHI nodes. | |||
6007 | if (MergeBlockIntoPredecessor(BB)) | |||
6008 | return true; | |||
6009 | ||||
6010 | if (SinkCommon && Options.SinkCommonInsts) | |||
6011 | Changed |= SinkCommonCodeFromPredecessors(BB); | |||
6012 | ||||
6013 | IRBuilder<> Builder(BB); | |||
6014 | ||||
6015 | // If there is a trivial two-entry PHI node in this basic block, and we can | |||
6016 | // eliminate it, do so now. | |||
6017 | if (auto *PN = dyn_cast<PHINode>(BB->begin())) | |||
6018 | if (PN->getNumIncomingValues() == 2) | |||
6019 | Changed |= FoldTwoEntryPHINode(PN, TTI, DL); | |||
6020 | ||||
6021 | Builder.SetInsertPoint(BB->getTerminator()); | |||
6022 | if (auto *BI = dyn_cast<BranchInst>(BB->getTerminator())) { | |||
6023 | if (BI->isUnconditional()) { | |||
6024 | if (SimplifyUncondBranch(BI, Builder)) | |||
6025 | return true; | |||
6026 | } else { | |||
6027 | if (SimplifyCondBranch(BI, Builder)) | |||
6028 | return true; | |||
6029 | } | |||
6030 | } else if (auto *RI = dyn_cast<ReturnInst>(BB->getTerminator())) { | |||
6031 | if (SimplifyReturn(RI, Builder)) | |||
6032 | return true; | |||
6033 | } else if (auto *RI = dyn_cast<ResumeInst>(BB->getTerminator())) { | |||
6034 | if (SimplifyResume(RI, Builder)) | |||
6035 | return true; | |||
6036 | } else if (auto *RI = dyn_cast<CleanupReturnInst>(BB->getTerminator())) { | |||
6037 | if (SimplifyCleanupReturn(RI)) | |||
6038 | return true; | |||
6039 | } else if (auto *SI = dyn_cast<SwitchInst>(BB->getTerminator())) { | |||
6040 | if (SimplifySwitch(SI, Builder)) | |||
6041 | return true; | |||
6042 | } else if (auto *UI = dyn_cast<UnreachableInst>(BB->getTerminator())) { | |||
6043 | if (SimplifyUnreachable(UI)) | |||
6044 | return true; | |||
6045 | } else if (auto *IBI = dyn_cast<IndirectBrInst>(BB->getTerminator())) { | |||
6046 | if (SimplifyIndirectBr(IBI)) | |||
6047 | return true; | |||
6048 | } | |||
6049 | ||||
6050 | return Changed; | |||
6051 | } | |||
6052 | ||||
6053 | bool llvm::simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI, | |||
6054 | const SimplifyCFGOptions &Options, | |||
6055 | SmallPtrSetImpl<BasicBlock *> *LoopHeaders) { | |||
6056 | return SimplifyCFGOpt(TTI, BB->getModule()->getDataLayout(), LoopHeaders, | |||
6057 | Options) | |||
6058 | .run(BB); | |||
6059 | } |