Bug Summary

File:llvm/lib/Transforms/Utils/SimplifyCFG.cpp
Warning:line 4060, column 24
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SimplifyCFG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/lib/Transforms/Utils -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/lib/Transforms/Utils -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-09-26-161721-17566-1 -x c++ /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
1//===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Peephole optimize the CFG.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APInt.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/Optional.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/ScopeExit.h"
19#include "llvm/ADT/SetOperations.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/Analysis/AssumptionCache.h"
26#include "llvm/Analysis/ConstantFolding.h"
27#include "llvm/Analysis/EHPersonalities.h"
28#include "llvm/Analysis/GuardUtils.h"
29#include "llvm/Analysis/InstructionSimplify.h"
30#include "llvm/Analysis/MemorySSA.h"
31#include "llvm/Analysis/MemorySSAUpdater.h"
32#include "llvm/Analysis/TargetTransformInfo.h"
33#include "llvm/Analysis/ValueTracking.h"
34#include "llvm/IR/Attributes.h"
35#include "llvm/IR/BasicBlock.h"
36#include "llvm/IR/CFG.h"
37#include "llvm/IR/Constant.h"
38#include "llvm/IR/ConstantRange.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DataLayout.h"
41#include "llvm/IR/DerivedTypes.h"
42#include "llvm/IR/Function.h"
43#include "llvm/IR/GlobalValue.h"
44#include "llvm/IR/GlobalVariable.h"
45#include "llvm/IR/IRBuilder.h"
46#include "llvm/IR/InstrTypes.h"
47#include "llvm/IR/Instruction.h"
48#include "llvm/IR/Instructions.h"
49#include "llvm/IR/IntrinsicInst.h"
50#include "llvm/IR/Intrinsics.h"
51#include "llvm/IR/LLVMContext.h"
52#include "llvm/IR/MDBuilder.h"
53#include "llvm/IR/Metadata.h"
54#include "llvm/IR/Module.h"
55#include "llvm/IR/NoFolder.h"
56#include "llvm/IR/Operator.h"
57#include "llvm/IR/PatternMatch.h"
58#include "llvm/IR/Type.h"
59#include "llvm/IR/Use.h"
60#include "llvm/IR/User.h"
61#include "llvm/IR/Value.h"
62#include "llvm/Support/Casting.h"
63#include "llvm/Support/CommandLine.h"
64#include "llvm/Support/Debug.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/KnownBits.h"
67#include "llvm/Support/MathExtras.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Transforms/Utils/BasicBlockUtils.h"
70#include "llvm/Transforms/Utils/Local.h"
71#include "llvm/Transforms/Utils/ValueMapper.h"
72#include <algorithm>
73#include <cassert>
74#include <climits>
75#include <cstddef>
76#include <cstdint>
77#include <iterator>
78#include <map>
79#include <set>
80#include <tuple>
81#include <utility>
82#include <vector>
83
84using namespace llvm;
85using namespace PatternMatch;
86
87#define DEBUG_TYPE"simplifycfg" "simplifycfg"
88
89// Chosen as 2 so as to be cheap, but still to have enough power to fold
90// a select, so the "clamp" idiom (of a min followed by a max) will be caught.
91// To catch this, we need to fold a compare and a select, hence '2' being the
92// minimum reasonable default.
93static cl::opt<unsigned> PHINodeFoldingThreshold(
94 "phi-node-folding-threshold", cl::Hidden, cl::init(2),
95 cl::desc(
96 "Control the amount of phi node folding to perform (default = 2)"));
97
98static cl::opt<unsigned> TwoEntryPHINodeFoldingThreshold(
99 "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4),
100 cl::desc("Control the maximal total instruction cost that we are willing "
101 "to speculatively execute to fold a 2-entry PHI node into a "
102 "select (default = 4)"));
103
104static cl::opt<bool> DupRet(
105 "simplifycfg-dup-ret", cl::Hidden, cl::init(false),
106 cl::desc("Duplicate return instructions into unconditional branches"));
107
108static cl::opt<bool>
109 HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true),
110 cl::desc("Hoist common instructions up to the parent block"));
111
112static cl::opt<bool>
113 SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
114 cl::desc("Sink common instructions down to the end block"));
115
116static cl::opt<bool> HoistCondStores(
117 "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true),
118 cl::desc("Hoist conditional stores if an unconditional store precedes"));
119
120static cl::opt<bool> MergeCondStores(
121 "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true),
122 cl::desc("Hoist conditional stores even if an unconditional store does not "
123 "precede - hoist multiple conditional stores into a single "
124 "predicated store"));
125
126static cl::opt<bool> MergeCondStoresAggressively(
127 "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false),
128 cl::desc("When merging conditional stores, do so even if the resultant "
129 "basic blocks are unlikely to be if-converted as a result"));
130
131static cl::opt<bool> SpeculateOneExpensiveInst(
132 "speculate-one-expensive-inst", cl::Hidden, cl::init(true),
133 cl::desc("Allow exactly one expensive instruction to be speculatively "
134 "executed"));
135
136static cl::opt<unsigned> MaxSpeculationDepth(
137 "max-speculation-depth", cl::Hidden, cl::init(10),
138 cl::desc("Limit maximum recursion depth when calculating costs of "
139 "speculatively executed instructions"));
140
141static cl::opt<int>
142MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden, cl::init(10),
143 cl::desc("Max size of a block which is still considered "
144 "small enough to thread through"));
145
146// Two is chosen to allow one negation and a logical combine.
147static cl::opt<unsigned>
148 BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden,
149 cl::init(2),
150 cl::desc("Maximum cost of combining conditions when "
151 "folding branches"));
152
153STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps")static llvm::Statistic NumBitMaps = {"simplifycfg", "NumBitMaps"
, "Number of switch instructions turned into bitmaps"}
;
154STATISTIC(NumLinearMaps,static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps"
, "Number of switch instructions turned into linear mapping"}
155 "Number of switch instructions turned into linear mapping")static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps"
, "Number of switch instructions turned into linear mapping"}
;
156STATISTIC(NumLookupTables,static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables"
, "Number of switch instructions turned into lookup tables"}
157 "Number of switch instructions turned into lookup tables")static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables"
, "Number of switch instructions turned into lookup tables"}
;
158STATISTIC(static llvm::Statistic NumLookupTablesHoles = {"simplifycfg",
"NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)"
}
159 NumLookupTablesHoles,static llvm::Statistic NumLookupTablesHoles = {"simplifycfg",
"NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)"
}
160 "Number of switch instructions turned into lookup tables (holes checked)")static llvm::Statistic NumLookupTablesHoles = {"simplifycfg",
"NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)"
}
;
161STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares")static llvm::Statistic NumTableCmpReuses = {"simplifycfg", "NumTableCmpReuses"
, "Number of reused switch table lookup compares"}
;
162STATISTIC(static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode"
, "Number of common instruction 'blocks' hoisted up to the begin block"
}
163 NumHoistCommonCode,static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode"
, "Number of common instruction 'blocks' hoisted up to the begin block"
}
164 "Number of common instruction 'blocks' hoisted up to the begin block")static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode"
, "Number of common instruction 'blocks' hoisted up to the begin block"
}
;
165STATISTIC(NumHoistCommonInstrs,static llvm::Statistic NumHoistCommonInstrs = {"simplifycfg",
"NumHoistCommonInstrs", "Number of common instructions hoisted up to the begin block"
}
166 "Number of common instructions hoisted up to the begin block")static llvm::Statistic NumHoistCommonInstrs = {"simplifycfg",
"NumHoistCommonInstrs", "Number of common instructions hoisted up to the begin block"
}
;
167STATISTIC(NumSinkCommonCode,static llvm::Statistic NumSinkCommonCode = {"simplifycfg", "NumSinkCommonCode"
, "Number of common instruction 'blocks' sunk down to the end block"
}
168 "Number of common instruction 'blocks' sunk down to the end block")static llvm::Statistic NumSinkCommonCode = {"simplifycfg", "NumSinkCommonCode"
, "Number of common instruction 'blocks' sunk down to the end block"
}
;
169STATISTIC(NumSinkCommonInstrs,static llvm::Statistic NumSinkCommonInstrs = {"simplifycfg", "NumSinkCommonInstrs"
, "Number of common instructions sunk down to the end block"}
170 "Number of common instructions sunk down to the end block")static llvm::Statistic NumSinkCommonInstrs = {"simplifycfg", "NumSinkCommonInstrs"
, "Number of common instructions sunk down to the end block"}
;
171STATISTIC(NumSpeculations, "Number of speculative executed instructions")static llvm::Statistic NumSpeculations = {"simplifycfg", "NumSpeculations"
, "Number of speculative executed instructions"}
;
172STATISTIC(NumInvokes,static llvm::Statistic NumInvokes = {"simplifycfg", "NumInvokes"
, "Number of invokes with empty resume blocks simplified into calls"
}
173 "Number of invokes with empty resume blocks simplified into calls")static llvm::Statistic NumInvokes = {"simplifycfg", "NumInvokes"
, "Number of invokes with empty resume blocks simplified into calls"
}
;
174
175namespace {
176
177// The first field contains the value that the switch produces when a certain
178// case group is selected, and the second field is a vector containing the
179// cases composing the case group.
180using SwitchCaseResultVectorTy =
181 SmallVector<std::pair<Constant *, SmallVector<ConstantInt *, 4>>, 2>;
182
183// The first field contains the phi node that generates a result of the switch
184// and the second field contains the value generated for a certain case in the
185// switch for that PHI.
186using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
187
188/// ValueEqualityComparisonCase - Represents a case of a switch.
189struct ValueEqualityComparisonCase {
190 ConstantInt *Value;
191 BasicBlock *Dest;
192
193 ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest)
194 : Value(Value), Dest(Dest) {}
195
196 bool operator<(ValueEqualityComparisonCase RHS) const {
197 // Comparing pointers is ok as we only rely on the order for uniquing.
198 return Value < RHS.Value;
199 }
200
201 bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; }
202};
203
204class SimplifyCFGOpt {
205 const TargetTransformInfo &TTI;
206 const DataLayout &DL;
207 SmallPtrSetImpl<BasicBlock *> *LoopHeaders;
208 const SimplifyCFGOptions &Options;
209 bool Resimplify;
210
211 Value *isValueEqualityComparison(Instruction *TI);
212 BasicBlock *GetValueEqualityComparisonCases(
213 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases);
214 bool SimplifyEqualityComparisonWithOnlyPredecessor(Instruction *TI,
215 BasicBlock *Pred,
216 IRBuilder<> &Builder);
217 bool FoldValueComparisonIntoPredecessors(Instruction *TI,
218 IRBuilder<> &Builder);
219
220 bool simplifyReturn(ReturnInst *RI, IRBuilder<> &Builder);
221 bool simplifyResume(ResumeInst *RI, IRBuilder<> &Builder);
222 bool simplifySingleResume(ResumeInst *RI);
223 bool simplifyCommonResume(ResumeInst *RI);
224 bool simplifyCleanupReturn(CleanupReturnInst *RI);
225 bool simplifyUnreachable(UnreachableInst *UI);
226 bool simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder);
227 bool simplifyIndirectBr(IndirectBrInst *IBI);
228 bool simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder);
229 bool simplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder);
230 bool simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder);
231 bool SimplifyCondBranchToTwoReturns(BranchInst *BI, IRBuilder<> &Builder);
232
233 bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
234 IRBuilder<> &Builder);
235
236 bool HoistThenElseCodeToIf(BranchInst *BI, const TargetTransformInfo &TTI);
237 bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
238 const TargetTransformInfo &TTI);
239 bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
240 BasicBlock *TrueBB, BasicBlock *FalseBB,
241 uint32_t TrueWeight, uint32_t FalseWeight);
242 bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder,
243 const DataLayout &DL);
244 bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select);
245 bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI);
246 bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder);
247
248public:
249 SimplifyCFGOpt(const TargetTransformInfo &TTI, const DataLayout &DL,
250 SmallPtrSetImpl<BasicBlock *> *LoopHeaders,
251 const SimplifyCFGOptions &Opts)
252 : TTI(TTI), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) {}
253
254 bool run(BasicBlock *BB);
255 bool simplifyOnce(BasicBlock *BB);
256
257 // Helper to set Resimplify and return change indication.
258 bool requestResimplify() {
259 Resimplify = true;
260 return true;
261 }
262};
263
264} // end anonymous namespace
265
266/// Return true if it is safe to merge these two
267/// terminator instructions together.
268static bool
269SafeToMergeTerminators(Instruction *SI1, Instruction *SI2,
270 SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) {
271 if (SI1 == SI2)
272 return false; // Can't merge with self!
273
274 // It is not safe to merge these two switch instructions if they have a common
275 // successor, and if that successor has a PHI node, and if *that* PHI node has
276 // conflicting incoming values from the two switch blocks.
277 BasicBlock *SI1BB = SI1->getParent();
278 BasicBlock *SI2BB = SI2->getParent();
279
280 SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
281 bool Fail = false;
282 for (BasicBlock *Succ : successors(SI2BB))
283 if (SI1Succs.count(Succ))
284 for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) {
285 PHINode *PN = cast<PHINode>(BBI);
286 if (PN->getIncomingValueForBlock(SI1BB) !=
287 PN->getIncomingValueForBlock(SI2BB)) {
288 if (FailBlocks)
289 FailBlocks->insert(Succ);
290 Fail = true;
291 }
292 }
293
294 return !Fail;
295}
296
297/// Return true if it is safe and profitable to merge these two terminator
298/// instructions together, where SI1 is an unconditional branch. PhiNodes will
299/// store all PHI nodes in common successors.
300static bool
301isProfitableToFoldUnconditional(BranchInst *SI1, BranchInst *SI2,
302 Instruction *Cond,
303 SmallVectorImpl<PHINode *> &PhiNodes) {
304 if (SI1 == SI2)
305 return false; // Can't merge with self!
306 assert(SI1->isUnconditional() && SI2->isConditional())((SI1->isUnconditional() && SI2->isConditional(
)) ? static_cast<void> (0) : __assert_fail ("SI1->isUnconditional() && SI2->isConditional()"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 306, __PRETTY_FUNCTION__))
;
307
308 // We fold the unconditional branch if we can easily update all PHI nodes in
309 // common successors:
310 // 1> We have a constant incoming value for the conditional branch;
311 // 2> We have "Cond" as the incoming value for the unconditional branch;
312 // 3> SI2->getCondition() and Cond have same operands.
313 CmpInst *Ci2 = dyn_cast<CmpInst>(SI2->getCondition());
314 if (!Ci2)
315 return false;
316 if (!(Cond->getOperand(0) == Ci2->getOperand(0) &&
317 Cond->getOperand(1) == Ci2->getOperand(1)) &&
318 !(Cond->getOperand(0) == Ci2->getOperand(1) &&
319 Cond->getOperand(1) == Ci2->getOperand(0)))
320 return false;
321
322 BasicBlock *SI1BB = SI1->getParent();
323 BasicBlock *SI2BB = SI2->getParent();
324 SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
325 for (BasicBlock *Succ : successors(SI2BB))
326 if (SI1Succs.count(Succ))
327 for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) {
328 PHINode *PN = cast<PHINode>(BBI);
329 if (PN->getIncomingValueForBlock(SI1BB) != Cond ||
330 !isa<ConstantInt>(PN->getIncomingValueForBlock(SI2BB)))
331 return false;
332 PhiNodes.push_back(PN);
333 }
334 return true;
335}
336
337/// Update PHI nodes in Succ to indicate that there will now be entries in it
338/// from the 'NewPred' block. The values that will be flowing into the PHI nodes
339/// will be the same as those coming in from ExistPred, an existing predecessor
340/// of Succ.
341static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
342 BasicBlock *ExistPred,
343 MemorySSAUpdater *MSSAU = nullptr) {
344 for (PHINode &PN : Succ->phis())
345 PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred);
346 if (MSSAU)
347 if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ))
348 MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred);
349}
350
351/// Compute an abstract "cost" of speculating the given instruction,
352/// which is assumed to be safe to speculate. TCC_Free means cheap,
353/// TCC_Basic means less cheap, and TCC_Expensive means prohibitively
354/// expensive.
355static unsigned ComputeSpeculationCost(const User *I,
356 const TargetTransformInfo &TTI) {
357 assert(isSafeToSpeculativelyExecute(I) &&((isSafeToSpeculativelyExecute(I) && "Instruction is not safe to speculatively execute!"
) ? static_cast<void> (0) : __assert_fail ("isSafeToSpeculativelyExecute(I) && \"Instruction is not safe to speculatively execute!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 358, __PRETTY_FUNCTION__))
358 "Instruction is not safe to speculatively execute!")((isSafeToSpeculativelyExecute(I) && "Instruction is not safe to speculatively execute!"
) ? static_cast<void> (0) : __assert_fail ("isSafeToSpeculativelyExecute(I) && \"Instruction is not safe to speculatively execute!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 358, __PRETTY_FUNCTION__))
;
359 return TTI.getUserCost(I, TargetTransformInfo::TCK_SizeAndLatency);
360}
361
362/// If we have a merge point of an "if condition" as accepted above,
363/// return true if the specified value dominates the block. We
364/// don't handle the true generality of domination here, just a special case
365/// which works well enough for us.
366///
367/// If AggressiveInsts is non-null, and if V does not dominate BB, we check to
368/// see if V (which must be an instruction) and its recursive operands
369/// that do not dominate BB have a combined cost lower than CostRemaining and
370/// are non-trapping. If both are true, the instruction is inserted into the
371/// set and true is returned.
372///
373/// The cost for most non-trapping instructions is defined as 1 except for
374/// Select whose cost is 2.
375///
376/// After this function returns, CostRemaining is decreased by the cost of
377/// V plus its non-dominating operands. If that cost is greater than
378/// CostRemaining, false is returned and CostRemaining is undefined.
379static bool DominatesMergePoint(Value *V, BasicBlock *BB,
380 SmallPtrSetImpl<Instruction *> &AggressiveInsts,
381 int &BudgetRemaining,
382 const TargetTransformInfo &TTI,
383 unsigned Depth = 0) {
384 // It is possible to hit a zero-cost cycle (phi/gep instructions for example),
385 // so limit the recursion depth.
386 // TODO: While this recursion limit does prevent pathological behavior, it
387 // would be better to track visited instructions to avoid cycles.
388 if (Depth == MaxSpeculationDepth)
389 return false;
390
391 Instruction *I = dyn_cast<Instruction>(V);
392 if (!I) {
393 // Non-instructions all dominate instructions, but not all constantexprs
394 // can be executed unconditionally.
395 if (ConstantExpr *C = dyn_cast<ConstantExpr>(V))
396 if (C->canTrap())
397 return false;
398 return true;
399 }
400 BasicBlock *PBB = I->getParent();
401
402 // We don't want to allow weird loops that might have the "if condition" in
403 // the bottom of this block.
404 if (PBB == BB)
405 return false;
406
407 // If this instruction is defined in a block that contains an unconditional
408 // branch to BB, then it must be in the 'conditional' part of the "if
409 // statement". If not, it definitely dominates the region.
410 BranchInst *BI = dyn_cast<BranchInst>(PBB->getTerminator());
411 if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB)
412 return true;
413
414 // If we have seen this instruction before, don't count it again.
415 if (AggressiveInsts.count(I))
416 return true;
417
418 // Okay, it looks like the instruction IS in the "condition". Check to
419 // see if it's a cheap instruction to unconditionally compute, and if it
420 // only uses stuff defined outside of the condition. If so, hoist it out.
421 if (!isSafeToSpeculativelyExecute(I))
422 return false;
423
424 BudgetRemaining -= ComputeSpeculationCost(I, TTI);
425
426 // Allow exactly one instruction to be speculated regardless of its cost
427 // (as long as it is safe to do so).
428 // This is intended to flatten the CFG even if the instruction is a division
429 // or other expensive operation. The speculation of an expensive instruction
430 // is expected to be undone in CodeGenPrepare if the speculation has not
431 // enabled further IR optimizations.
432 if (BudgetRemaining < 0 &&
433 (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0))
434 return false;
435
436 // Okay, we can only really hoist these out if their operands do
437 // not take us over the cost threshold.
438 for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i)
439 if (!DominatesMergePoint(*i, BB, AggressiveInsts, BudgetRemaining, TTI,
440 Depth + 1))
441 return false;
442 // Okay, it's safe to do this! Remember this instruction.
443 AggressiveInsts.insert(I);
444 return true;
445}
446
447/// Extract ConstantInt from value, looking through IntToPtr
448/// and PointerNullValue. Return NULL if value is not a constant int.
449static ConstantInt *GetConstantInt(Value *V, const DataLayout &DL) {
450 // Normal constant int.
451 ConstantInt *CI = dyn_cast<ConstantInt>(V);
452 if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy())
453 return CI;
454
455 // This is some kind of pointer constant. Turn it into a pointer-sized
456 // ConstantInt if possible.
457 IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType()));
458
459 // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
460 if (isa<ConstantPointerNull>(V))
461 return ConstantInt::get(PtrTy, 0);
462
463 // IntToPtr const int.
464 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
465 if (CE->getOpcode() == Instruction::IntToPtr)
466 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) {
467 // The constant is very likely to have the right type already.
468 if (CI->getType() == PtrTy)
469 return CI;
470 else
471 return cast<ConstantInt>(
472 ConstantExpr::getIntegerCast(CI, PtrTy, /*isSigned=*/false));
473 }
474 return nullptr;
475}
476
477namespace {
478
479/// Given a chain of or (||) or and (&&) comparison of a value against a
480/// constant, this will try to recover the information required for a switch
481/// structure.
482/// It will depth-first traverse the chain of comparison, seeking for patterns
483/// like %a == 12 or %a < 4 and combine them to produce a set of integer
484/// representing the different cases for the switch.
485/// Note that if the chain is composed of '||' it will build the set of elements
486/// that matches the comparisons (i.e. any of this value validate the chain)
487/// while for a chain of '&&' it will build the set elements that make the test
488/// fail.
489struct ConstantComparesGatherer {
490 const DataLayout &DL;
491
492 /// Value found for the switch comparison
493 Value *CompValue = nullptr;
494
495 /// Extra clause to be checked before the switch
496 Value *Extra = nullptr;
497
498 /// Set of integers to match in switch
499 SmallVector<ConstantInt *, 8> Vals;
500
501 /// Number of comparisons matched in the and/or chain
502 unsigned UsedICmps = 0;
503
504 /// Construct and compute the result for the comparison instruction Cond
505 ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) {
506 gather(Cond);
507 }
508
509 ConstantComparesGatherer(const ConstantComparesGatherer &) = delete;
510 ConstantComparesGatherer &
511 operator=(const ConstantComparesGatherer &) = delete;
512
513private:
514 /// Try to set the current value used for the comparison, it succeeds only if
515 /// it wasn't set before or if the new value is the same as the old one
516 bool setValueOnce(Value *NewVal) {
517 if (CompValue && CompValue != NewVal)
518 return false;
519 CompValue = NewVal;
520 return (CompValue != nullptr);
521 }
522
523 /// Try to match Instruction "I" as a comparison against a constant and
524 /// populates the array Vals with the set of values that match (or do not
525 /// match depending on isEQ).
526 /// Return false on failure. On success, the Value the comparison matched
527 /// against is placed in CompValue.
528 /// If CompValue is already set, the function is expected to fail if a match
529 /// is found but the value compared to is different.
530 bool matchInstruction(Instruction *I, bool isEQ) {
531 // If this is an icmp against a constant, handle this as one of the cases.
532 ICmpInst *ICI;
533 ConstantInt *C;
534 if (!((ICI = dyn_cast<ICmpInst>(I)) &&
535 (C = GetConstantInt(I->getOperand(1), DL)))) {
536 return false;
537 }
538
539 Value *RHSVal;
540 const APInt *RHSC;
541
542 // Pattern match a special case
543 // (x & ~2^z) == y --> x == y || x == y|2^z
544 // This undoes a transformation done by instcombine to fuse 2 compares.
545 if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
546 // It's a little bit hard to see why the following transformations are
547 // correct. Here is a CVC3 program to verify them for 64-bit values:
548
549 /*
550 ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63);
551 x : BITVECTOR(64);
552 y : BITVECTOR(64);
553 z : BITVECTOR(64);
554 mask : BITVECTOR(64) = BVSHL(ONE, z);
555 QUERY( (y & ~mask = y) =>
556 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
557 );
558 QUERY( (y | mask = y) =>
559 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
560 );
561 */
562
563 // Please note that each pattern must be a dual implication (<--> or
564 // iff). One directional implication can create spurious matches. If the
565 // implication is only one-way, an unsatisfiable condition on the left
566 // side can imply a satisfiable condition on the right side. Dual
567 // implication ensures that satisfiable conditions are transformed to
568 // other satisfiable conditions and unsatisfiable conditions are
569 // transformed to other unsatisfiable conditions.
570
571 // Here is a concrete example of a unsatisfiable condition on the left
572 // implying a satisfiable condition on the right:
573 //
574 // mask = (1 << z)
575 // (x & ~mask) == y --> (x == y || x == (y | mask))
576 //
577 // Substituting y = 3, z = 0 yields:
578 // (x & -2) == 3 --> (x == 3 || x == 2)
579
580 // Pattern match a special case:
581 /*
582 QUERY( (y & ~mask = y) =>
583 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
584 );
585 */
586 if (match(ICI->getOperand(0),
587 m_And(m_Value(RHSVal), m_APInt(RHSC)))) {
588 APInt Mask = ~*RHSC;
589 if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) {
590 // If we already have a value for the switch, it has to match!
591 if (!setValueOnce(RHSVal))
592 return false;
593
594 Vals.push_back(C);
595 Vals.push_back(
596 ConstantInt::get(C->getContext(),
597 C->getValue() | Mask));
598 UsedICmps++;
599 return true;
600 }
601 }
602
603 // Pattern match a special case:
604 /*
605 QUERY( (y | mask = y) =>
606 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
607 );
608 */
609 if (match(ICI->getOperand(0),
610 m_Or(m_Value(RHSVal), m_APInt(RHSC)))) {
611 APInt Mask = *RHSC;
612 if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) {
613 // If we already have a value for the switch, it has to match!
614 if (!setValueOnce(RHSVal))
615 return false;
616
617 Vals.push_back(C);
618 Vals.push_back(ConstantInt::get(C->getContext(),
619 C->getValue() & ~Mask));
620 UsedICmps++;
621 return true;
622 }
623 }
624
625 // If we already have a value for the switch, it has to match!
626 if (!setValueOnce(ICI->getOperand(0)))
627 return false;
628
629 UsedICmps++;
630 Vals.push_back(C);
631 return ICI->getOperand(0);
632 }
633
634 // If we have "x ult 3", for example, then we can add 0,1,2 to the set.
635 ConstantRange Span = ConstantRange::makeAllowedICmpRegion(
636 ICI->getPredicate(), C->getValue());
637
638 // Shift the range if the compare is fed by an add. This is the range
639 // compare idiom as emitted by instcombine.
640 Value *CandidateVal = I->getOperand(0);
641 if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) {
642 Span = Span.subtract(*RHSC);
643 CandidateVal = RHSVal;
644 }
645
646 // If this is an and/!= check, then we are looking to build the set of
647 // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into
648 // x != 0 && x != 1.
649 if (!isEQ)
650 Span = Span.inverse();
651
652 // If there are a ton of values, we don't want to make a ginormous switch.
653 if (Span.isSizeLargerThan(8) || Span.isEmptySet()) {
654 return false;
655 }
656
657 // If we already have a value for the switch, it has to match!
658 if (!setValueOnce(CandidateVal))
659 return false;
660
661 // Add all values from the range to the set
662 for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp)
663 Vals.push_back(ConstantInt::get(I->getContext(), Tmp));
664
665 UsedICmps++;
666 return true;
667 }
668
669 /// Given a potentially 'or'd or 'and'd together collection of icmp
670 /// eq/ne/lt/gt instructions that compare a value against a constant, extract
671 /// the value being compared, and stick the list constants into the Vals
672 /// vector.
673 /// One "Extra" case is allowed to differ from the other.
674 void gather(Value *V) {
675 bool isEQ = (cast<Instruction>(V)->getOpcode() == Instruction::Or);
676
677 // Keep a stack (SmallVector for efficiency) for depth-first traversal
678 SmallVector<Value *, 8> DFT;
679 SmallPtrSet<Value *, 8> Visited;
680
681 // Initialize
682 Visited.insert(V);
683 DFT.push_back(V);
684
685 while (!DFT.empty()) {
686 V = DFT.pop_back_val();
687
688 if (Instruction *I = dyn_cast<Instruction>(V)) {
689 // If it is a || (or && depending on isEQ), process the operands.
690 if (I->getOpcode() == (isEQ ? Instruction::Or : Instruction::And)) {
691 if (Visited.insert(I->getOperand(1)).second)
692 DFT.push_back(I->getOperand(1));
693 if (Visited.insert(I->getOperand(0)).second)
694 DFT.push_back(I->getOperand(0));
695 continue;
696 }
697
698 // Try to match the current instruction
699 if (matchInstruction(I, isEQ))
700 // Match succeed, continue the loop
701 continue;
702 }
703
704 // One element of the sequence of || (or &&) could not be match as a
705 // comparison against the same value as the others.
706 // We allow only one "Extra" case to be checked before the switch
707 if (!Extra) {
708 Extra = V;
709 continue;
710 }
711 // Failed to parse a proper sequence, abort now
712 CompValue = nullptr;
713 break;
714 }
715 }
716};
717
718} // end anonymous namespace
719
720static void EraseTerminatorAndDCECond(Instruction *TI,
721 MemorySSAUpdater *MSSAU = nullptr) {
722 Instruction *Cond = nullptr;
723 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
724 Cond = dyn_cast<Instruction>(SI->getCondition());
725 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
726 if (BI->isConditional())
727 Cond = dyn_cast<Instruction>(BI->getCondition());
728 } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) {
729 Cond = dyn_cast<Instruction>(IBI->getAddress());
730 }
731
732 TI->eraseFromParent();
733 if (Cond)
734 RecursivelyDeleteTriviallyDeadInstructions(Cond, nullptr, MSSAU);
735}
736
737/// Return true if the specified terminator checks
738/// to see if a value is equal to constant integer value.
739Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) {
740 Value *CV = nullptr;
741 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
742 // Do not permit merging of large switch instructions into their
743 // predecessors unless there is only one predecessor.
744 if (!SI->getParent()->hasNPredecessorsOrMore(128 / SI->getNumSuccessors()))
745 CV = SI->getCondition();
746 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI))
747 if (BI->isConditional() && BI->getCondition()->hasOneUse())
748 if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) {
749 if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL))
750 CV = ICI->getOperand(0);
751 }
752
753 // Unwrap any lossless ptrtoint cast.
754 if (CV) {
755 if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) {
756 Value *Ptr = PTII->getPointerOperand();
757 if (PTII->getType() == DL.getIntPtrType(Ptr->getType()))
758 CV = Ptr;
759 }
760 }
761 return CV;
762}
763
764/// Given a value comparison instruction,
765/// decode all of the 'cases' that it represents and return the 'default' block.
766BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases(
767 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases) {
768 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
769 Cases.reserve(SI->getNumCases());
770 for (auto Case : SI->cases())
771 Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(),
772 Case.getCaseSuccessor()));
773 return SI->getDefaultDest();
774 }
775
776 BranchInst *BI = cast<BranchInst>(TI);
777 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
778 BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE);
779 Cases.push_back(ValueEqualityComparisonCase(
780 GetConstantInt(ICI->getOperand(1), DL), Succ));
781 return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ);
782}
783
784/// Given a vector of bb/value pairs, remove any entries
785/// in the list that match the specified block.
786static void
787EliminateBlockCases(BasicBlock *BB,
788 std::vector<ValueEqualityComparisonCase> &Cases) {
789 Cases.erase(std::remove(Cases.begin(), Cases.end(), BB), Cases.end());
790}
791
792/// Return true if there are any keys in C1 that exist in C2 as well.
793static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
794 std::vector<ValueEqualityComparisonCase> &C2) {
795 std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2;
796
797 // Make V1 be smaller than V2.
798 if (V1->size() > V2->size())
799 std::swap(V1, V2);
800
801 if (V1->empty())
802 return false;
803 if (V1->size() == 1) {
804 // Just scan V2.
805 ConstantInt *TheVal = (*V1)[0].Value;
806 for (unsigned i = 0, e = V2->size(); i != e; ++i)
807 if (TheVal == (*V2)[i].Value)
808 return true;
809 }
810
811 // Otherwise, just sort both lists and compare element by element.
812 array_pod_sort(V1->begin(), V1->end());
813 array_pod_sort(V2->begin(), V2->end());
814 unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size();
815 while (i1 != e1 && i2 != e2) {
816 if ((*V1)[i1].Value == (*V2)[i2].Value)
817 return true;
818 if ((*V1)[i1].Value < (*V2)[i2].Value)
819 ++i1;
820 else
821 ++i2;
822 }
823 return false;
824}
825
826// Set branch weights on SwitchInst. This sets the metadata if there is at
827// least one non-zero weight.
828static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights) {
829 // Check that there is at least one non-zero weight. Otherwise, pass
830 // nullptr to setMetadata which will erase the existing metadata.
831 MDNode *N = nullptr;
832 if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; }))
833 N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights);
834 SI->setMetadata(LLVMContext::MD_prof, N);
835}
836
837// Similar to the above, but for branch and select instructions that take
838// exactly 2 weights.
839static void setBranchWeights(Instruction *I, uint32_t TrueWeight,
840 uint32_t FalseWeight) {
841 assert(isa<BranchInst>(I) || isa<SelectInst>(I))((isa<BranchInst>(I) || isa<SelectInst>(I)) ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(I) || isa<SelectInst>(I)"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 841, __PRETTY_FUNCTION__))
;
842 // Check that there is at least one non-zero weight. Otherwise, pass
843 // nullptr to setMetadata which will erase the existing metadata.
844 MDNode *N = nullptr;
845 if (TrueWeight || FalseWeight)
846 N = MDBuilder(I->getParent()->getContext())
847 .createBranchWeights(TrueWeight, FalseWeight);
848 I->setMetadata(LLVMContext::MD_prof, N);
849}
850
851/// If TI is known to be a terminator instruction and its block is known to
852/// only have a single predecessor block, check to see if that predecessor is
853/// also a value comparison with the same value, and if that comparison
854/// determines the outcome of this comparison. If so, simplify TI. This does a
855/// very limited form of jump threading.
856bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
857 Instruction *TI, BasicBlock *Pred, IRBuilder<> &Builder) {
858 Value *PredVal = isValueEqualityComparison(Pred->getTerminator());
859 if (!PredVal)
860 return false; // Not a value comparison in predecessor.
861
862 Value *ThisVal = isValueEqualityComparison(TI);
863 assert(ThisVal && "This isn't a value comparison!!")((ThisVal && "This isn't a value comparison!!") ? static_cast
<void> (0) : __assert_fail ("ThisVal && \"This isn't a value comparison!!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 863, __PRETTY_FUNCTION__))
;
864 if (ThisVal != PredVal)
865 return false; // Different predicates.
866
867 // TODO: Preserve branch weight metadata, similarly to how
868 // FoldValueComparisonIntoPredecessors preserves it.
869
870 // Find out information about when control will move from Pred to TI's block.
871 std::vector<ValueEqualityComparisonCase> PredCases;
872 BasicBlock *PredDef =
873 GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases);
874 EliminateBlockCases(PredDef, PredCases); // Remove default from cases.
875
876 // Find information about how control leaves this block.
877 std::vector<ValueEqualityComparisonCase> ThisCases;
878 BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases);
879 EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases.
880
881 // If TI's block is the default block from Pred's comparison, potentially
882 // simplify TI based on this knowledge.
883 if (PredDef == TI->getParent()) {
884 // If we are here, we know that the value is none of those cases listed in
885 // PredCases. If there are any cases in ThisCases that are in PredCases, we
886 // can simplify TI.
887 if (!ValuesOverlap(PredCases, ThisCases))
888 return false;
889
890 if (isa<BranchInst>(TI)) {
891 // Okay, one of the successors of this condbr is dead. Convert it to a
892 // uncond br.
893 assert(ThisCases.size() == 1 && "Branch can only have one case!")((ThisCases.size() == 1 && "Branch can only have one case!"
) ? static_cast<void> (0) : __assert_fail ("ThisCases.size() == 1 && \"Branch can only have one case!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 893, __PRETTY_FUNCTION__))
;
894 // Insert the new branch.
895 Instruction *NI = Builder.CreateBr(ThisDef);
896 (void)NI;
897
898 // Remove PHI node entries for the dead edge.
899 ThisCases[0].Dest->removePredecessor(TI->getParent());
900
901 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
902 << "Through successor TI: " << *TI << "Leaving: " << *NIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
903 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
;
904
905 EraseTerminatorAndDCECond(TI);
906 return true;
907 }
908
909 SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI);
910 // Okay, TI has cases that are statically dead, prune them away.
911 SmallPtrSet<Constant *, 16> DeadCases;
912 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
913 DeadCases.insert(PredCases[i].Value);
914
915 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI; } } while (false)
916 << "Through successor TI: " << *TI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI; } } while (false)
;
917
918 for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
919 --i;
920 if (DeadCases.count(i->getCaseValue())) {
921 i->getCaseSuccessor()->removePredecessor(TI->getParent());
922 SI.removeCase(i);
923 }
924 }
925 LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Leaving: " << *TI <<
"\n"; } } while (false)
;
926 return true;
927 }
928
929 // Otherwise, TI's block must correspond to some matched value. Find out
930 // which value (or set of values) this is.
931 ConstantInt *TIV = nullptr;
932 BasicBlock *TIBB = TI->getParent();
933 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
934 if (PredCases[i].Dest == TIBB) {
935 if (TIV)
936 return false; // Cannot handle multiple values coming to this block.
937 TIV = PredCases[i].Value;
938 }
939 assert(TIV && "No edge from pred to succ?")((TIV && "No edge from pred to succ?") ? static_cast<
void> (0) : __assert_fail ("TIV && \"No edge from pred to succ?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 939, __PRETTY_FUNCTION__))
;
940
941 // Okay, we found the one constant that our value can be if we get into TI's
942 // BB. Find out which successor will unconditionally be branched to.
943 BasicBlock *TheRealDest = nullptr;
944 for (unsigned i = 0, e = ThisCases.size(); i != e; ++i)
945 if (ThisCases[i].Value == TIV) {
946 TheRealDest = ThisCases[i].Dest;
947 break;
948 }
949
950 // If not handled by any explicit cases, it is handled by the default case.
951 if (!TheRealDest)
952 TheRealDest = ThisDef;
953
954 // Remove PHI node entries for dead edges.
955 BasicBlock *CheckEdge = TheRealDest;
956 for (BasicBlock *Succ : successors(TIBB))
957 if (Succ != CheckEdge)
958 Succ->removePredecessor(TIBB);
959 else
960 CheckEdge = nullptr;
961
962 // Insert the new branch.
963 Instruction *NI = Builder.CreateBr(TheRealDest);
964 (void)NI;
965
966 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
967 << "Through successor TI: " << *TI << "Leaving: " << *NIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
968 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
;
969
970 EraseTerminatorAndDCECond(TI);
971 return true;
972}
973
974namespace {
975
976/// This class implements a stable ordering of constant
977/// integers that does not depend on their address. This is important for
978/// applications that sort ConstantInt's to ensure uniqueness.
979struct ConstantIntOrdering {
980 bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const {
981 return LHS->getValue().ult(RHS->getValue());
982 }
983};
984
985} // end anonymous namespace
986
987static int ConstantIntSortPredicate(ConstantInt *const *P1,
988 ConstantInt *const *P2) {
989 const ConstantInt *LHS = *P1;
990 const ConstantInt *RHS = *P2;
991 if (LHS == RHS)
992 return 0;
993 return LHS->getValue().ult(RHS->getValue()) ? 1 : -1;
994}
995
996static inline bool HasBranchWeights(const Instruction *I) {
997 MDNode *ProfMD = I->getMetadata(LLVMContext::MD_prof);
998 if (ProfMD && ProfMD->getOperand(0))
999 if (MDString *MDS = dyn_cast<MDString>(ProfMD->getOperand(0)))
1000 return MDS->getString().equals("branch_weights");
1001
1002 return false;
1003}
1004
1005/// Get Weights of a given terminator, the default weight is at the front
1006/// of the vector. If TI is a conditional eq, we need to swap the branch-weight
1007/// metadata.
1008static void GetBranchWeights(Instruction *TI,
1009 SmallVectorImpl<uint64_t> &Weights) {
1010 MDNode *MD = TI->getMetadata(LLVMContext::MD_prof);
1011 assert(MD)((MD) ? static_cast<void> (0) : __assert_fail ("MD", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1011, __PRETTY_FUNCTION__))
;
1012 for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) {
1013 ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(i));
1014 Weights.push_back(CI->getValue().getZExtValue());
1015 }
1016
1017 // If TI is a conditional eq, the default case is the false case,
1018 // and the corresponding branch-weight data is at index 2. We swap the
1019 // default weight to be the first entry.
1020 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
1021 assert(Weights.size() == 2)((Weights.size() == 2) ? static_cast<void> (0) : __assert_fail
("Weights.size() == 2", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1021, __PRETTY_FUNCTION__))
;
1022 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
1023 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
1024 std::swap(Weights.front(), Weights.back());
1025 }
1026}
1027
1028/// Keep halving the weights until all can fit in uint32_t.
1029static void FitWeights(MutableArrayRef<uint64_t> Weights) {
1030 uint64_t Max = *std::max_element(Weights.begin(), Weights.end());
1031 if (Max > UINT_MAX(2147483647 *2U +1U)) {
1032 unsigned Offset = 32 - countLeadingZeros(Max);
1033 for (uint64_t &I : Weights)
1034 I >>= Offset;
1035 }
1036}
1037
1038/// The specified terminator is a value equality comparison instruction
1039/// (either a switch or a branch on "X == c").
1040/// See if any of the predecessors of the terminator block are value comparisons
1041/// on the same value. If so, and if safe to do so, fold them together.
1042bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(Instruction *TI,
1043 IRBuilder<> &Builder) {
1044 BasicBlock *BB = TI->getParent();
1045 Value *CV = isValueEqualityComparison(TI); // CondVal
1046 assert(CV && "Not a comparison?")((CV && "Not a comparison?") ? static_cast<void>
(0) : __assert_fail ("CV && \"Not a comparison?\"", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1046, __PRETTY_FUNCTION__))
;
1047 bool Changed = false;
1048
1049 SmallVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB));
1050 while (!Preds.empty()) {
1051 BasicBlock *Pred = Preds.pop_back_val();
1052
1053 // See if the predecessor is a comparison with the same value.
1054 Instruction *PTI = Pred->getTerminator();
1055 Value *PCV = isValueEqualityComparison(PTI); // PredCondVal
1056
1057 if (PCV == CV && TI != PTI) {
1058 SmallSetVector<BasicBlock*, 4> FailBlocks;
1059 if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) {
1060 for (auto *Succ : FailBlocks) {
1061 if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split"))
1062 return false;
1063 }
1064 }
1065
1066 // Figure out which 'cases' to copy from SI to PSI.
1067 std::vector<ValueEqualityComparisonCase> BBCases;
1068 BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases);
1069
1070 std::vector<ValueEqualityComparisonCase> PredCases;
1071 BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases);
1072
1073 // Based on whether the default edge from PTI goes to BB or not, fill in
1074 // PredCases and PredDefault with the new switch cases we would like to
1075 // build.
1076 SmallVector<BasicBlock *, 8> NewSuccessors;
1077
1078 // Update the branch weight metadata along the way
1079 SmallVector<uint64_t, 8> Weights;
1080 bool PredHasWeights = HasBranchWeights(PTI);
1081 bool SuccHasWeights = HasBranchWeights(TI);
1082
1083 if (PredHasWeights) {
1084 GetBranchWeights(PTI, Weights);
1085 // branch-weight metadata is inconsistent here.
1086 if (Weights.size() != 1 + PredCases.size())
1087 PredHasWeights = SuccHasWeights = false;
1088 } else if (SuccHasWeights)
1089 // If there are no predecessor weights but there are successor weights,
1090 // populate Weights with 1, which will later be scaled to the sum of
1091 // successor's weights
1092 Weights.assign(1 + PredCases.size(), 1);
1093
1094 SmallVector<uint64_t, 8> SuccWeights;
1095 if (SuccHasWeights) {
1096 GetBranchWeights(TI, SuccWeights);
1097 // branch-weight metadata is inconsistent here.
1098 if (SuccWeights.size() != 1 + BBCases.size())
1099 PredHasWeights = SuccHasWeights = false;
1100 } else if (PredHasWeights)
1101 SuccWeights.assign(1 + BBCases.size(), 1);
1102
1103 if (PredDefault == BB) {
1104 // If this is the default destination from PTI, only the edges in TI
1105 // that don't occur in PTI, or that branch to BB will be activated.
1106 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1107 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1108 if (PredCases[i].Dest != BB)
1109 PTIHandled.insert(PredCases[i].Value);
1110 else {
1111 // The default destination is BB, we don't need explicit targets.
1112 std::swap(PredCases[i], PredCases.back());
1113
1114 if (PredHasWeights || SuccHasWeights) {
1115 // Increase weight for the default case.
1116 Weights[0] += Weights[i + 1];
1117 std::swap(Weights[i + 1], Weights.back());
1118 Weights.pop_back();
1119 }
1120
1121 PredCases.pop_back();
1122 --i;
1123 --e;
1124 }
1125
1126 // Reconstruct the new switch statement we will be building.
1127 if (PredDefault != BBDefault) {
1128 PredDefault->removePredecessor(Pred);
1129 PredDefault = BBDefault;
1130 NewSuccessors.push_back(BBDefault);
1131 }
1132
1133 unsigned CasesFromPred = Weights.size();
1134 uint64_t ValidTotalSuccWeight = 0;
1135 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1136 if (!PTIHandled.count(BBCases[i].Value) &&
1137 BBCases[i].Dest != BBDefault) {
1138 PredCases.push_back(BBCases[i]);
1139 NewSuccessors.push_back(BBCases[i].Dest);
1140 if (SuccHasWeights || PredHasWeights) {
1141 // The default weight is at index 0, so weight for the ith case
1142 // should be at index i+1. Scale the cases from successor by
1143 // PredDefaultWeight (Weights[0]).
1144 Weights.push_back(Weights[0] * SuccWeights[i + 1]);
1145 ValidTotalSuccWeight += SuccWeights[i + 1];
1146 }
1147 }
1148
1149 if (SuccHasWeights || PredHasWeights) {
1150 ValidTotalSuccWeight += SuccWeights[0];
1151 // Scale the cases from predecessor by ValidTotalSuccWeight.
1152 for (unsigned i = 1; i < CasesFromPred; ++i)
1153 Weights[i] *= ValidTotalSuccWeight;
1154 // Scale the default weight by SuccDefaultWeight (SuccWeights[0]).
1155 Weights[0] *= SuccWeights[0];
1156 }
1157 } else {
1158 // If this is not the default destination from PSI, only the edges
1159 // in SI that occur in PSI with a destination of BB will be
1160 // activated.
1161 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1162 std::map<ConstantInt *, uint64_t> WeightsForHandled;
1163 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1164 if (PredCases[i].Dest == BB) {
1165 PTIHandled.insert(PredCases[i].Value);
1166
1167 if (PredHasWeights || SuccHasWeights) {
1168 WeightsForHandled[PredCases[i].Value] = Weights[i + 1];
1169 std::swap(Weights[i + 1], Weights.back());
1170 Weights.pop_back();
1171 }
1172
1173 std::swap(PredCases[i], PredCases.back());
1174 PredCases.pop_back();
1175 --i;
1176 --e;
1177 }
1178
1179 // Okay, now we know which constants were sent to BB from the
1180 // predecessor. Figure out where they will all go now.
1181 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1182 if (PTIHandled.count(BBCases[i].Value)) {
1183 // If this is one we are capable of getting...
1184 if (PredHasWeights || SuccHasWeights)
1185 Weights.push_back(WeightsForHandled[BBCases[i].Value]);
1186 PredCases.push_back(BBCases[i]);
1187 NewSuccessors.push_back(BBCases[i].Dest);
1188 PTIHandled.erase(
1189 BBCases[i].Value); // This constant is taken care of
1190 }
1191
1192 // If there are any constants vectored to BB that TI doesn't handle,
1193 // they must go to the default destination of TI.
1194 for (ConstantInt *I : PTIHandled) {
1195 if (PredHasWeights || SuccHasWeights)
1196 Weights.push_back(WeightsForHandled[I]);
1197 PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault));
1198 NewSuccessors.push_back(BBDefault);
1199 }
1200 }
1201
1202 // Okay, at this point, we know which new successor Pred will get. Make
1203 // sure we update the number of entries in the PHI nodes for these
1204 // successors.
1205 for (BasicBlock *NewSuccessor : NewSuccessors)
1206 AddPredecessorToBlock(NewSuccessor, Pred, BB);
1207
1208 Builder.SetInsertPoint(PTI);
1209 // Convert pointer to int before we switch.
1210 if (CV->getType()->isPointerTy()) {
1211 CV = Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()),
1212 "magicptr");
1213 }
1214
1215 // Now that the successors are updated, create the new Switch instruction.
1216 SwitchInst *NewSI =
1217 Builder.CreateSwitch(CV, PredDefault, PredCases.size());
1218 NewSI->setDebugLoc(PTI->getDebugLoc());
1219 for (ValueEqualityComparisonCase &V : PredCases)
1220 NewSI->addCase(V.Value, V.Dest);
1221
1222 if (PredHasWeights || SuccHasWeights) {
1223 // Halve the weights if any of them cannot fit in an uint32_t
1224 FitWeights(Weights);
1225
1226 SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
1227
1228 setBranchWeights(NewSI, MDWeights);
1229 }
1230
1231 EraseTerminatorAndDCECond(PTI);
1232
1233 // Okay, last check. If BB is still a successor of PSI, then we must
1234 // have an infinite loop case. If so, add an infinitely looping block
1235 // to handle the case to preserve the behavior of the code.
1236 BasicBlock *InfLoopBlock = nullptr;
1237 for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i)
1238 if (NewSI->getSuccessor(i) == BB) {
1239 if (!InfLoopBlock) {
1240 // Insert it at the end of the function, because it's either code,
1241 // or it won't matter if it's hot. :)
1242 InfLoopBlock = BasicBlock::Create(BB->getContext(), "infloop",
1243 BB->getParent());
1244 BranchInst::Create(InfLoopBlock, InfLoopBlock);
1245 }
1246 NewSI->setSuccessor(i, InfLoopBlock);
1247 }
1248
1249 Changed = true;
1250 }
1251 }
1252 return Changed;
1253}
1254
1255// If we would need to insert a select that uses the value of this invoke
1256// (comments in HoistThenElseCodeToIf explain why we would need to do this), we
1257// can't hoist the invoke, as there is nowhere to put the select in this case.
1258static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2,
1259 Instruction *I1, Instruction *I2) {
1260 for (BasicBlock *Succ : successors(BB1)) {
1261 for (const PHINode &PN : Succ->phis()) {
1262 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1263 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1264 if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) {
1265 return false;
1266 }
1267 }
1268 }
1269 return true;
1270}
1271
1272static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I);
1273
1274/// Given a conditional branch that goes to BB1 and BB2, hoist any common code
1275/// in the two blocks up into the branch block. The caller of this function
1276/// guarantees that BI's block dominates BB1 and BB2.
1277bool SimplifyCFGOpt::HoistThenElseCodeToIf(BranchInst *BI,
1278 const TargetTransformInfo &TTI) {
1279 // This does very trivial matching, with limited scanning, to find identical
1280 // instructions in the two blocks. In particular, we don't want to get into
1281 // O(M*N) situations here where M and N are the sizes of BB1 and BB2. As
1282 // such, we currently just scan for obviously identical instructions in an
1283 // identical order.
1284 BasicBlock *BB1 = BI->getSuccessor(0); // The true destination.
1285 BasicBlock *BB2 = BI->getSuccessor(1); // The false destination
1286
1287 BasicBlock::iterator BB1_Itr = BB1->begin();
1288 BasicBlock::iterator BB2_Itr = BB2->begin();
1289
1290 Instruction *I1 = &*BB1_Itr++, *I2 = &*BB2_Itr++;
1291 // Skip debug info if it is not identical.
1292 DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1);
1293 DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2);
1294 if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1295 while (isa<DbgInfoIntrinsic>(I1))
1296 I1 = &*BB1_Itr++;
1297 while (isa<DbgInfoIntrinsic>(I2))
1298 I2 = &*BB2_Itr++;
1299 }
1300 // FIXME: Can we define a safety predicate for CallBr?
1301 if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) ||
1302 (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) ||
1303 isa<CallBrInst>(I1))
1304 return false;
1305
1306 BasicBlock *BIParent = BI->getParent();
1307
1308 bool Changed = false;
1309
1310 auto _ = make_scope_exit([&]() {
1311 if (Changed)
1312 ++NumHoistCommonCode;
1313 });
1314
1315 do {
1316 // If we are hoisting the terminator instruction, don't move one (making a
1317 // broken BB), instead clone it, and remove BI.
1318 if (I1->isTerminator())
1319 goto HoistTerminator;
1320
1321 // If we're going to hoist a call, make sure that the two instructions we're
1322 // commoning/hoisting are both marked with musttail, or neither of them is
1323 // marked as such. Otherwise, we might end up in a situation where we hoist
1324 // from a block where the terminator is a `ret` to a block where the terminator
1325 // is a `br`, and `musttail` calls expect to be followed by a return.
1326 auto *C1 = dyn_cast<CallInst>(I1);
1327 auto *C2 = dyn_cast<CallInst>(I2);
1328 if (C1 && C2)
1329 if (C1->isMustTailCall() != C2->isMustTailCall())
1330 return Changed;
1331
1332 if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2))
1333 return Changed;
1334
1335 // If any of the two call sites has nomerge attribute, stop hoisting.
1336 if (const auto *CB1 = dyn_cast<CallBase>(I1))
1337 if (CB1->cannotMerge())
1338 return Changed;
1339 if (const auto *CB2 = dyn_cast<CallBase>(I2))
1340 if (CB2->cannotMerge())
1341 return Changed;
1342
1343 if (isa<DbgInfoIntrinsic>(I1) || isa<DbgInfoIntrinsic>(I2)) {
1344 assert (isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2))((isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic
>(I2)) ? static_cast<void> (0) : __assert_fail ("isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2)"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1344, __PRETTY_FUNCTION__))
;
1345 // The debug location is an integral part of a debug info intrinsic
1346 // and can't be separated from it or replaced. Instead of attempting
1347 // to merge locations, simply hoist both copies of the intrinsic.
1348 BIParent->getInstList().splice(BI->getIterator(),
1349 BB1->getInstList(), I1);
1350 BIParent->getInstList().splice(BI->getIterator(),
1351 BB2->getInstList(), I2);
1352 Changed = true;
1353 } else {
1354 // For a normal instruction, we just move one to right before the branch,
1355 // then replace all uses of the other with the first. Finally, we remove
1356 // the now redundant second instruction.
1357 BIParent->getInstList().splice(BI->getIterator(),
1358 BB1->getInstList(), I1);
1359 if (!I2->use_empty())
1360 I2->replaceAllUsesWith(I1);
1361 I1->andIRFlags(I2);
1362 unsigned KnownIDs[] = {LLVMContext::MD_tbaa,
1363 LLVMContext::MD_range,
1364 LLVMContext::MD_fpmath,
1365 LLVMContext::MD_invariant_load,
1366 LLVMContext::MD_nonnull,
1367 LLVMContext::MD_invariant_group,
1368 LLVMContext::MD_align,
1369 LLVMContext::MD_dereferenceable,
1370 LLVMContext::MD_dereferenceable_or_null,
1371 LLVMContext::MD_mem_parallel_loop_access,
1372 LLVMContext::MD_access_group,
1373 LLVMContext::MD_preserve_access_index};
1374 combineMetadata(I1, I2, KnownIDs, true);
1375
1376 // I1 and I2 are being combined into a single instruction. Its debug
1377 // location is the merged locations of the original instructions.
1378 I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1379
1380 I2->eraseFromParent();
1381 Changed = true;
1382 }
1383 ++NumHoistCommonInstrs;
1384
1385 I1 = &*BB1_Itr++;
1386 I2 = &*BB2_Itr++;
1387 // Skip debug info if it is not identical.
1388 DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1);
1389 DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2);
1390 if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1391 while (isa<DbgInfoIntrinsic>(I1))
1392 I1 = &*BB1_Itr++;
1393 while (isa<DbgInfoIntrinsic>(I2))
1394 I2 = &*BB2_Itr++;
1395 }
1396 } while (I1->isIdenticalToWhenDefined(I2));
1397
1398 return true;
1399
1400HoistTerminator:
1401 // It may not be possible to hoist an invoke.
1402 // FIXME: Can we define a safety predicate for CallBr?
1403 if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))
1404 return Changed;
1405
1406 // TODO: callbr hoisting currently disabled pending further study.
1407 if (isa<CallBrInst>(I1))
1408 return Changed;
1409
1410 for (BasicBlock *Succ : successors(BB1)) {
1411 for (PHINode &PN : Succ->phis()) {
1412 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1413 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1414 if (BB1V == BB2V)
1415 continue;
1416
1417 // Check for passingValueIsAlwaysUndefined here because we would rather
1418 // eliminate undefined control flow then converting it to a select.
1419 if (passingValueIsAlwaysUndefined(BB1V, &PN) ||
1420 passingValueIsAlwaysUndefined(BB2V, &PN))
1421 return Changed;
1422
1423 if (isa<ConstantExpr>(BB1V) && !isSafeToSpeculativelyExecute(BB1V))
1424 return Changed;
1425 if (isa<ConstantExpr>(BB2V) && !isSafeToSpeculativelyExecute(BB2V))
1426 return Changed;
1427 }
1428 }
1429
1430 // Okay, it is safe to hoist the terminator.
1431 Instruction *NT = I1->clone();
1432 BIParent->getInstList().insert(BI->getIterator(), NT);
1433 if (!NT->getType()->isVoidTy()) {
1434 I1->replaceAllUsesWith(NT);
1435 I2->replaceAllUsesWith(NT);
1436 NT->takeName(I1);
1437 }
1438 Changed = true;
1439 ++NumHoistCommonInstrs;
1440
1441 // Ensure terminator gets a debug location, even an unknown one, in case
1442 // it involves inlinable calls.
1443 NT->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1444
1445 // PHIs created below will adopt NT's merged DebugLoc.
1446 IRBuilder<NoFolder> Builder(NT);
1447
1448 // Hoisting one of the terminators from our successor is a great thing.
1449 // Unfortunately, the successors of the if/else blocks may have PHI nodes in
1450 // them. If they do, all PHI entries for BB1/BB2 must agree for all PHI
1451 // nodes, so we insert select instruction to compute the final result.
1452 std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects;
1453 for (BasicBlock *Succ : successors(BB1)) {
1454 for (PHINode &PN : Succ->phis()) {
1455 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1456 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1457 if (BB1V == BB2V)
1458 continue;
1459
1460 // These values do not agree. Insert a select instruction before NT
1461 // that determines the right value.
1462 SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)];
1463 if (!SI) {
1464 // Propagate fast-math-flags from phi node to its replacement select.
1465 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
1466 if (isa<FPMathOperator>(PN))
1467 Builder.setFastMathFlags(PN.getFastMathFlags());
1468
1469 SI = cast<SelectInst>(
1470 Builder.CreateSelect(BI->getCondition(), BB1V, BB2V,
1471 BB1V->getName() + "." + BB2V->getName(), BI));
1472 }
1473
1474 // Make the PHI node use the select for all incoming values for BB1/BB2
1475 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
1476 if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2)
1477 PN.setIncomingValue(i, SI);
1478 }
1479 }
1480
1481 // Update any PHI nodes in our new successors.
1482 for (BasicBlock *Succ : successors(BB1))
1483 AddPredecessorToBlock(Succ, BIParent, BB1);
1484
1485 EraseTerminatorAndDCECond(BI);
1486 return Changed;
1487}
1488
1489// Check lifetime markers.
1490static bool isLifeTimeMarker(const Instruction *I) {
1491 if (auto II = dyn_cast<IntrinsicInst>(I)) {
1492 switch (II->getIntrinsicID()) {
1493 default:
1494 break;
1495 case Intrinsic::lifetime_start:
1496 case Intrinsic::lifetime_end:
1497 return true;
1498 }
1499 }
1500 return false;
1501}
1502
1503// TODO: Refine this. This should avoid cases like turning constant memcpy sizes
1504// into variables.
1505static bool replacingOperandWithVariableIsCheap(const Instruction *I,
1506 int OpIdx) {
1507 return !isa<IntrinsicInst>(I);
1508}
1509
1510// All instructions in Insts belong to different blocks that all unconditionally
1511// branch to a common successor. Analyze each instruction and return true if it
1512// would be possible to sink them into their successor, creating one common
1513// instruction instead. For every value that would be required to be provided by
1514// PHI node (because an operand varies in each input block), add to PHIOperands.
1515static bool canSinkInstructions(
1516 ArrayRef<Instruction *> Insts,
1517 DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) {
1518 // Prune out obviously bad instructions to move. Each instruction must have
1519 // exactly zero or one use, and we check later that use is by a single, common
1520 // PHI instruction in the successor.
1521 bool HasUse = !Insts.front()->user_empty();
1522 for (auto *I : Insts) {
1523 // These instructions may change or break semantics if moved.
1524 if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) ||
1525 I->getType()->isTokenTy())
1526 return false;
1527
1528 // Conservatively return false if I is an inline-asm instruction. Sinking
1529 // and merging inline-asm instructions can potentially create arguments
1530 // that cannot satisfy the inline-asm constraints.
1531 // If the instruction has nomerge attribute, return false.
1532 if (const auto *C = dyn_cast<CallBase>(I))
1533 if (C->isInlineAsm() || C->cannotMerge())
1534 return false;
1535
1536 // Each instruction must have zero or one use.
1537 if (HasUse && !I->hasOneUse())
1538 return false;
1539 if (!HasUse && !I->user_empty())
1540 return false;
1541 }
1542
1543 const Instruction *I0 = Insts.front();
1544 for (auto *I : Insts)
1545 if (!I->isSameOperationAs(I0))
1546 return false;
1547
1548 // All instructions in Insts are known to be the same opcode. If they have a
1549 // use, check that the only user is a PHI or in the same block as the
1550 // instruction, because if a user is in the same block as an instruction we're
1551 // contemplating sinking, it must already be determined to be sinkable.
1552 if (HasUse) {
1553 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1554 auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0);
1555 if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool {
1556 auto *U = cast<Instruction>(*I->user_begin());
1557 return (PNUse &&
1558 PNUse->getParent() == Succ &&
1559 PNUse->getIncomingValueForBlock(I->getParent()) == I) ||
1560 U->getParent() == I->getParent();
1561 }))
1562 return false;
1563 }
1564
1565 // Because SROA can't handle speculating stores of selects, try not to sink
1566 // loads, stores or lifetime markers of allocas when we'd have to create a
1567 // PHI for the address operand. Also, because it is likely that loads or
1568 // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink
1569 // them.
1570 // This can cause code churn which can have unintended consequences down
1571 // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244.
1572 // FIXME: This is a workaround for a deficiency in SROA - see
1573 // https://llvm.org/bugs/show_bug.cgi?id=30188
1574 if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) {
1575 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
1576 }))
1577 return false;
1578 if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) {
1579 return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
1580 }))
1581 return false;
1582 if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) {
1583 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
1584 }))
1585 return false;
1586
1587 for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) {
1588 Value *Op = I0->getOperand(OI);
1589 if (Op->getType()->isTokenTy())
1590 // Don't touch any operand of token type.
1591 return false;
1592
1593 auto SameAsI0 = [&I0, OI](const Instruction *I) {
1594 assert(I->getNumOperands() == I0->getNumOperands())((I->getNumOperands() == I0->getNumOperands()) ? static_cast
<void> (0) : __assert_fail ("I->getNumOperands() == I0->getNumOperands()"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1594, __PRETTY_FUNCTION__))
;
1595 return I->getOperand(OI) == I0->getOperand(OI);
1596 };
1597 if (!all_of(Insts, SameAsI0)) {
1598 if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) ||
1599 !canReplaceOperandWithVariable(I0, OI))
1600 // We can't create a PHI from this GEP.
1601 return false;
1602 // Don't create indirect calls! The called value is the final operand.
1603 if (isa<CallBase>(I0) && OI == OE - 1) {
1604 // FIXME: if the call was *already* indirect, we should do this.
1605 return false;
1606 }
1607 for (auto *I : Insts)
1608 PHIOperands[I].push_back(I->getOperand(OI));
1609 }
1610 }
1611 return true;
1612}
1613
1614// Assuming canSinkLastInstruction(Blocks) has returned true, sink the last
1615// instruction of every block in Blocks to their common successor, commoning
1616// into one instruction.
1617static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
1618 auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0);
1619
1620 // canSinkLastInstruction returning true guarantees that every block has at
1621 // least one non-terminator instruction.
1622 SmallVector<Instruction*,4> Insts;
1623 for (auto *BB : Blocks) {
1624 Instruction *I = BB->getTerminator();
1625 do {
1626 I = I->getPrevNode();
1627 } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front());
1628 if (!isa<DbgInfoIntrinsic>(I))
1629 Insts.push_back(I);
1630 }
1631
1632 // The only checking we need to do now is that all users of all instructions
1633 // are the same PHI node. canSinkLastInstruction should have checked this but
1634 // it is slightly over-aggressive - it gets confused by commutative instructions
1635 // so double-check it here.
1636 Instruction *I0 = Insts.front();
1637 if (!I0->user_empty()) {
1638 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1639 if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool {
1640 auto *U = cast<Instruction>(*I->user_begin());
1641 return U == PNUse;
1642 }))
1643 return false;
1644 }
1645
1646 // We don't need to do any more checking here; canSinkLastInstruction should
1647 // have done it all for us.
1648 SmallVector<Value*, 4> NewOperands;
1649 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) {
1650 // This check is different to that in canSinkLastInstruction. There, we
1651 // cared about the global view once simplifycfg (and instcombine) have
1652 // completed - it takes into account PHIs that become trivially
1653 // simplifiable. However here we need a more local view; if an operand
1654 // differs we create a PHI and rely on instcombine to clean up the very
1655 // small mess we may make.
1656 bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) {
1657 return I->getOperand(O) != I0->getOperand(O);
1658 });
1659 if (!NeedPHI) {
1660 NewOperands.push_back(I0->getOperand(O));
1661 continue;
1662 }
1663
1664 // Create a new PHI in the successor block and populate it.
1665 auto *Op = I0->getOperand(O);
1666 assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!")((!Op->getType()->isTokenTy() && "Can't PHI tokens!"
) ? static_cast<void> (0) : __assert_fail ("!Op->getType()->isTokenTy() && \"Can't PHI tokens!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1666, __PRETTY_FUNCTION__))
;
1667 auto *PN = PHINode::Create(Op->getType(), Insts.size(),
1668 Op->getName() + ".sink", &BBEnd->front());
1669 for (auto *I : Insts)
1670 PN->addIncoming(I->getOperand(O), I->getParent());
1671 NewOperands.push_back(PN);
1672 }
1673
1674 // Arbitrarily use I0 as the new "common" instruction; remap its operands
1675 // and move it to the start of the successor block.
1676 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O)
1677 I0->getOperandUse(O).set(NewOperands[O]);
1678 I0->moveBefore(&*BBEnd->getFirstInsertionPt());
1679
1680 // Update metadata and IR flags, and merge debug locations.
1681 for (auto *I : Insts)
1682 if (I != I0) {
1683 // The debug location for the "common" instruction is the merged locations
1684 // of all the commoned instructions. We start with the original location
1685 // of the "common" instruction and iteratively merge each location in the
1686 // loop below.
1687 // This is an N-way merge, which will be inefficient if I0 is a CallInst.
1688 // However, as N-way merge for CallInst is rare, so we use simplified API
1689 // instead of using complex API for N-way merge.
1690 I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc());
1691 combineMetadataForCSE(I0, I, true);
1692 I0->andIRFlags(I);
1693 }
1694
1695 if (!I0->user_empty()) {
1696 // canSinkLastInstruction checked that all instructions were used by
1697 // one and only one PHI node. Find that now, RAUW it to our common
1698 // instruction and nuke it.
1699 auto *PN = cast<PHINode>(*I0->user_begin());
1700 PN->replaceAllUsesWith(I0);
1701 PN->eraseFromParent();
1702 }
1703
1704 // Finally nuke all instructions apart from the common instruction.
1705 for (auto *I : Insts)
1706 if (I != I0)
1707 I->eraseFromParent();
1708
1709 return true;
1710}
1711
1712namespace {
1713
1714 // LockstepReverseIterator - Iterates through instructions
1715 // in a set of blocks in reverse order from the first non-terminator.
1716 // For example (assume all blocks have size n):
1717 // LockstepReverseIterator I([B1, B2, B3]);
1718 // *I-- = [B1[n], B2[n], B3[n]];
1719 // *I-- = [B1[n-1], B2[n-1], B3[n-1]];
1720 // *I-- = [B1[n-2], B2[n-2], B3[n-2]];
1721 // ...
1722 class LockstepReverseIterator {
1723 ArrayRef<BasicBlock*> Blocks;
1724 SmallVector<Instruction*,4> Insts;
1725 bool Fail;
1726
1727 public:
1728 LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) : Blocks(Blocks) {
1729 reset();
1730 }
1731
1732 void reset() {
1733 Fail = false;
1734 Insts.clear();
1735 for (auto *BB : Blocks) {
1736 Instruction *Inst = BB->getTerminator();
1737 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1738 Inst = Inst->getPrevNode();
1739 if (!Inst) {
1740 // Block wasn't big enough.
1741 Fail = true;
1742 return;
1743 }
1744 Insts.push_back(Inst);
1745 }
1746 }
1747
1748 bool isValid() const {
1749 return !Fail;
1750 }
1751
1752 void operator--() {
1753 if (Fail)
1754 return;
1755 for (auto *&Inst : Insts) {
1756 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1757 Inst = Inst->getPrevNode();
1758 // Already at beginning of block.
1759 if (!Inst) {
1760 Fail = true;
1761 return;
1762 }
1763 }
1764 }
1765
1766 ArrayRef<Instruction*> operator * () const {
1767 return Insts;
1768 }
1769 };
1770
1771} // end anonymous namespace
1772
1773/// Check whether BB's predecessors end with unconditional branches. If it is
1774/// true, sink any common code from the predecessors to BB.
1775/// We also allow one predecessor to end with conditional branch (but no more
1776/// than one).
1777static bool SinkCommonCodeFromPredecessors(BasicBlock *BB) {
1778 // We support two situations:
1779 // (1) all incoming arcs are unconditional
1780 // (2) one incoming arc is conditional
1781 //
1782 // (2) is very common in switch defaults and
1783 // else-if patterns;
1784 //
1785 // if (a) f(1);
1786 // else if (b) f(2);
1787 //
1788 // produces:
1789 //
1790 // [if]
1791 // / \
1792 // [f(1)] [if]
1793 // | | \
1794 // | | |
1795 // | [f(2)]|
1796 // \ | /
1797 // [ end ]
1798 //
1799 // [end] has two unconditional predecessor arcs and one conditional. The
1800 // conditional refers to the implicit empty 'else' arc. This conditional
1801 // arc can also be caused by an empty default block in a switch.
1802 //
1803 // In this case, we attempt to sink code from all *unconditional* arcs.
1804 // If we can sink instructions from these arcs (determined during the scan
1805 // phase below) we insert a common successor for all unconditional arcs and
1806 // connect that to [end], to enable sinking:
1807 //
1808 // [if]
1809 // / \
1810 // [x(1)] [if]
1811 // | | \
1812 // | | \
1813 // | [x(2)] |
1814 // \ / |
1815 // [sink.split] |
1816 // \ /
1817 // [ end ]
1818 //
1819 SmallVector<BasicBlock*,4> UnconditionalPreds;
1820 Instruction *Cond = nullptr;
1821 for (auto *B : predecessors(BB)) {
1822 auto *T = B->getTerminator();
1823 if (isa<BranchInst>(T) && cast<BranchInst>(T)->isUnconditional())
1824 UnconditionalPreds.push_back(B);
1825 else if ((isa<BranchInst>(T) || isa<SwitchInst>(T)) && !Cond)
1826 Cond = T;
1827 else
1828 return false;
1829 }
1830 if (UnconditionalPreds.size() < 2)
1831 return false;
1832
1833 // We take a two-step approach to tail sinking. First we scan from the end of
1834 // each block upwards in lockstep. If the n'th instruction from the end of each
1835 // block can be sunk, those instructions are added to ValuesToSink and we
1836 // carry on. If we can sink an instruction but need to PHI-merge some operands
1837 // (because they're not identical in each instruction) we add these to
1838 // PHIOperands.
1839 unsigned ScanIdx = 0;
1840 SmallPtrSet<Value*,4> InstructionsToSink;
1841 DenseMap<Instruction*, SmallVector<Value*,4>> PHIOperands;
1842 LockstepReverseIterator LRI(UnconditionalPreds);
1843 while (LRI.isValid() &&
1844 canSinkInstructions(*LRI, PHIOperands)) {
1845 LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: instruction can be sunk: "
<< *(*LRI)[0] << "\n"; } } while (false)
1846 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: instruction can be sunk: "
<< *(*LRI)[0] << "\n"; } } while (false)
;
1847 InstructionsToSink.insert((*LRI).begin(), (*LRI).end());
1848 ++ScanIdx;
1849 --LRI;
1850 }
1851
1852 // If no instructions can be sunk, early-return.
1853 if (ScanIdx == 0)
1854 return false;
1855
1856 bool Changed = false;
1857
1858 auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) {
1859 unsigned NumPHIdValues = 0;
1860 for (auto *I : *LRI)
1861 for (auto *V : PHIOperands[I])
1862 if (InstructionsToSink.count(V) == 0)
1863 ++NumPHIdValues;
1864 LLVM_DEBUG(dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: #phid values: " <<
NumPHIdValues << "\n"; } } while (false)
;
1865 unsigned NumPHIInsts = NumPHIdValues / UnconditionalPreds.size();
1866 if ((NumPHIdValues % UnconditionalPreds.size()) != 0)
1867 NumPHIInsts++;
1868
1869 return NumPHIInsts <= 1;
1870 };
1871
1872 if (Cond) {
1873 // Check if we would actually sink anything first! This mutates the CFG and
1874 // adds an extra block. The goal in doing this is to allow instructions that
1875 // couldn't be sunk before to be sunk - obviously, speculatable instructions
1876 // (such as trunc, add) can be sunk and predicated already. So we check that
1877 // we're going to sink at least one non-speculatable instruction.
1878 LRI.reset();
1879 unsigned Idx = 0;
1880 bool Profitable = false;
1881 while (ProfitableToSinkInstruction(LRI) && Idx < ScanIdx) {
1882 if (!isSafeToSpeculativelyExecute((*LRI)[0])) {
1883 Profitable = true;
1884 break;
1885 }
1886 --LRI;
1887 ++Idx;
1888 }
1889 if (!Profitable)
1890 return false;
1891
1892 LLVM_DEBUG(dbgs() << "SINK: Splitting edge\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Splitting edge\n"; }
} while (false)
;
1893 // We have a conditional edge and we're going to sink some instructions.
1894 // Insert a new block postdominating all blocks we're going to sink from.
1895 if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split"))
1896 // Edges couldn't be split.
1897 return false;
1898 Changed = true;
1899 }
1900
1901 // Now that we've analyzed all potential sinking candidates, perform the
1902 // actual sink. We iteratively sink the last non-terminator of the source
1903 // blocks into their common successor unless doing so would require too
1904 // many PHI instructions to be generated (currently only one PHI is allowed
1905 // per sunk instruction).
1906 //
1907 // We can use InstructionsToSink to discount values needing PHI-merging that will
1908 // actually be sunk in a later iteration. This allows us to be more
1909 // aggressive in what we sink. This does allow a false positive where we
1910 // sink presuming a later value will also be sunk, but stop half way through
1911 // and never actually sink it which means we produce more PHIs than intended.
1912 // This is unlikely in practice though.
1913 unsigned SinkIdx = 0;
1914 for (; SinkIdx != ScanIdx; ++SinkIdx) {
1915 LLVM_DEBUG(dbgs() << "SINK: Sink: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds
[0]->getTerminator()->getPrevNode() << "\n"; } } while
(false)
1916 << *UnconditionalPreds[0]->getTerminator()->getPrevNode()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds
[0]->getTerminator()->getPrevNode() << "\n"; } } while
(false)
1917 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds
[0]->getTerminator()->getPrevNode() << "\n"; } } while
(false)
;
1918
1919 // Because we've sunk every instruction in turn, the current instruction to
1920 // sink is always at index 0.
1921 LRI.reset();
1922 if (!ProfitableToSinkInstruction(LRI)) {
1923 // Too many PHIs would be created.
1924 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, too many PHIs would be created!\n"
; } } while (false)
1925 dbgs() << "SINK: stopping here, too many PHIs would be created!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, too many PHIs would be created!\n"
; } } while (false)
;
1926 break;
1927 }
1928
1929 if (!sinkLastInstruction(UnconditionalPreds)) {
1930 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, failed to actually sink instruction!\n"
; } } while (false)
1931 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, failed to actually sink instruction!\n"
; } } while (false)
1932 << "SINK: stopping here, failed to actually sink instruction!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, failed to actually sink instruction!\n"
; } } while (false)
;
1933 break;
1934 }
1935
1936 NumSinkCommonInstrs++;
1937 Changed = true;
1938 }
1939 if (SinkIdx != 0)
1940 ++NumSinkCommonCode;
1941 return Changed;
1942}
1943
1944/// Determine if we can hoist sink a sole store instruction out of a
1945/// conditional block.
1946///
1947/// We are looking for code like the following:
1948/// BrBB:
1949/// store i32 %add, i32* %arrayidx2
1950/// ... // No other stores or function calls (we could be calling a memory
1951/// ... // function).
1952/// %cmp = icmp ult %x, %y
1953/// br i1 %cmp, label %EndBB, label %ThenBB
1954/// ThenBB:
1955/// store i32 %add5, i32* %arrayidx2
1956/// br label EndBB
1957/// EndBB:
1958/// ...
1959/// We are going to transform this into:
1960/// BrBB:
1961/// store i32 %add, i32* %arrayidx2
1962/// ... //
1963/// %cmp = icmp ult %x, %y
1964/// %add.add5 = select i1 %cmp, i32 %add, %add5
1965/// store i32 %add.add5, i32* %arrayidx2
1966/// ...
1967///
1968/// \return The pointer to the value of the previous store if the store can be
1969/// hoisted into the predecessor block. 0 otherwise.
1970static Value *isSafeToSpeculateStore(Instruction *I, BasicBlock *BrBB,
1971 BasicBlock *StoreBB, BasicBlock *EndBB) {
1972 StoreInst *StoreToHoist = dyn_cast<StoreInst>(I);
1973 if (!StoreToHoist)
1974 return nullptr;
1975
1976 // Volatile or atomic.
1977 if (!StoreToHoist->isSimple())
1978 return nullptr;
1979
1980 Value *StorePtr = StoreToHoist->getPointerOperand();
1981
1982 // Look for a store to the same pointer in BrBB.
1983 unsigned MaxNumInstToLookAt = 9;
1984 for (Instruction &CurI : reverse(BrBB->instructionsWithoutDebug())) {
1985 if (!MaxNumInstToLookAt)
1986 break;
1987 --MaxNumInstToLookAt;
1988
1989 // Could be calling an instruction that affects memory like free().
1990 if (CurI.mayHaveSideEffects() && !isa<StoreInst>(CurI))
1991 return nullptr;
1992
1993 if (auto *SI = dyn_cast<StoreInst>(&CurI)) {
1994 // Found the previous store make sure it stores to the same location.
1995 if (SI->getPointerOperand() == StorePtr)
1996 // Found the previous store, return its value operand.
1997 return SI->getValueOperand();
1998 return nullptr; // Unknown store.
1999 }
2000 }
2001
2002 return nullptr;
2003}
2004
2005/// Estimate the cost of the insertion(s) and check that the PHI nodes can be
2006/// converted to selects.
2007static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB,
2008 BasicBlock *EndBB,
2009 unsigned &SpeculatedInstructions,
2010 int &BudgetRemaining,
2011 const TargetTransformInfo &TTI) {
2012 TargetTransformInfo::TargetCostKind CostKind =
2013 BB->getParent()->hasMinSize()
2014 ? TargetTransformInfo::TCK_CodeSize
2015 : TargetTransformInfo::TCK_SizeAndLatency;
2016
2017 bool HaveRewritablePHIs = false;
2018 for (PHINode &PN : EndBB->phis()) {
2019 Value *OrigV = PN.getIncomingValueForBlock(BB);
2020 Value *ThenV = PN.getIncomingValueForBlock(ThenBB);
2021
2022 // FIXME: Try to remove some of the duplication with HoistThenElseCodeToIf.
2023 // Skip PHIs which are trivial.
2024 if (ThenV == OrigV)
2025 continue;
2026
2027 BudgetRemaining -=
2028 TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr,
2029 CostKind);
2030
2031 // Don't convert to selects if we could remove undefined behavior instead.
2032 if (passingValueIsAlwaysUndefined(OrigV, &PN) ||
2033 passingValueIsAlwaysUndefined(ThenV, &PN))
2034 return false;
2035
2036 HaveRewritablePHIs = true;
2037 ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV);
2038 ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV);
2039 if (!OrigCE && !ThenCE)
2040 continue; // Known safe and cheap.
2041
2042 if ((ThenCE && !isSafeToSpeculativelyExecute(ThenCE)) ||
2043 (OrigCE && !isSafeToSpeculativelyExecute(OrigCE)))
2044 return false;
2045 unsigned OrigCost = OrigCE ? ComputeSpeculationCost(OrigCE, TTI) : 0;
2046 unsigned ThenCost = ThenCE ? ComputeSpeculationCost(ThenCE, TTI) : 0;
2047 unsigned MaxCost =
2048 2 * PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2049 if (OrigCost + ThenCost > MaxCost)
2050 return false;
2051
2052 // Account for the cost of an unfolded ConstantExpr which could end up
2053 // getting expanded into Instructions.
2054 // FIXME: This doesn't account for how many operations are combined in the
2055 // constant expression.
2056 ++SpeculatedInstructions;
2057 if (SpeculatedInstructions > 1)
2058 return false;
2059 }
2060
2061 return HaveRewritablePHIs;
2062}
2063
2064/// Speculate a conditional basic block flattening the CFG.
2065///
2066/// Note that this is a very risky transform currently. Speculating
2067/// instructions like this is most often not desirable. Instead, there is an MI
2068/// pass which can do it with full awareness of the resource constraints.
2069/// However, some cases are "obvious" and we should do directly. An example of
2070/// this is speculating a single, reasonably cheap instruction.
2071///
2072/// There is only one distinct advantage to flattening the CFG at the IR level:
2073/// it makes very common but simplistic optimizations such as are common in
2074/// instcombine and the DAG combiner more powerful by removing CFG edges and
2075/// modeling their effects with easier to reason about SSA value graphs.
2076///
2077///
2078/// An illustration of this transform is turning this IR:
2079/// \code
2080/// BB:
2081/// %cmp = icmp ult %x, %y
2082/// br i1 %cmp, label %EndBB, label %ThenBB
2083/// ThenBB:
2084/// %sub = sub %x, %y
2085/// br label BB2
2086/// EndBB:
2087/// %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ]
2088/// ...
2089/// \endcode
2090///
2091/// Into this IR:
2092/// \code
2093/// BB:
2094/// %cmp = icmp ult %x, %y
2095/// %sub = sub %x, %y
2096/// %cond = select i1 %cmp, 0, %sub
2097/// ...
2098/// \endcode
2099///
2100/// \returns true if the conditional block is removed.
2101bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
2102 const TargetTransformInfo &TTI) {
2103 // Be conservative for now. FP select instruction can often be expensive.
2104 Value *BrCond = BI->getCondition();
2105 if (isa<FCmpInst>(BrCond))
2106 return false;
2107
2108 BasicBlock *BB = BI->getParent();
2109 BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0);
2110 int BudgetRemaining =
2111 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2112
2113 // If ThenBB is actually on the false edge of the conditional branch, remember
2114 // to swap the select operands later.
2115 bool Invert = false;
2116 if (ThenBB != BI->getSuccessor(0)) {
2117 assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?")((ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?"
) ? static_cast<void> (0) : __assert_fail ("ThenBB == BI->getSuccessor(1) && \"No edge from 'if' block?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2117, __PRETTY_FUNCTION__))
;
2118 Invert = true;
2119 }
2120 assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block")((EndBB == BI->getSuccessor(!Invert) && "No edge from to end block"
) ? static_cast<void> (0) : __assert_fail ("EndBB == BI->getSuccessor(!Invert) && \"No edge from to end block\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2120, __PRETTY_FUNCTION__))
;
2121
2122 // Keep a count of how many times instructions are used within ThenBB when
2123 // they are candidates for sinking into ThenBB. Specifically:
2124 // - They are defined in BB, and
2125 // - They have no side effects, and
2126 // - All of their uses are in ThenBB.
2127 SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts;
2128
2129 SmallVector<Instruction *, 4> SpeculatedDbgIntrinsics;
2130
2131 unsigned SpeculatedInstructions = 0;
2132 Value *SpeculatedStoreValue = nullptr;
2133 StoreInst *SpeculatedStore = nullptr;
2134 for (BasicBlock::iterator BBI = ThenBB->begin(),
2135 BBE = std::prev(ThenBB->end());
2136 BBI != BBE; ++BBI) {
2137 Instruction *I = &*BBI;
2138 // Skip debug info.
2139 if (isa<DbgInfoIntrinsic>(I)) {
2140 SpeculatedDbgIntrinsics.push_back(I);
2141 continue;
2142 }
2143
2144 // Only speculatively execute a single instruction (not counting the
2145 // terminator) for now.
2146 ++SpeculatedInstructions;
2147 if (SpeculatedInstructions > 1)
2148 return false;
2149
2150 // Don't hoist the instruction if it's unsafe or expensive.
2151 if (!isSafeToSpeculativelyExecute(I) &&
2152 !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore(
2153 I, BB, ThenBB, EndBB))))
2154 return false;
2155 if (!SpeculatedStoreValue &&
2156 ComputeSpeculationCost(I, TTI) >
2157 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic)
2158 return false;
2159
2160 // Store the store speculation candidate.
2161 if (SpeculatedStoreValue)
2162 SpeculatedStore = cast<StoreInst>(I);
2163
2164 // Do not hoist the instruction if any of its operands are defined but not
2165 // used in BB. The transformation will prevent the operand from
2166 // being sunk into the use block.
2167 for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i) {
2168 Instruction *OpI = dyn_cast<Instruction>(*i);
2169 if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects())
2170 continue; // Not a candidate for sinking.
2171
2172 ++SinkCandidateUseCounts[OpI];
2173 }
2174 }
2175
2176 // Consider any sink candidates which are only used in ThenBB as costs for
2177 // speculation. Note, while we iterate over a DenseMap here, we are summing
2178 // and so iteration order isn't significant.
2179 for (SmallDenseMap<Instruction *, unsigned, 4>::iterator
2180 I = SinkCandidateUseCounts.begin(),
2181 E = SinkCandidateUseCounts.end();
2182 I != E; ++I)
2183 if (I->first->hasNUses(I->second)) {
2184 ++SpeculatedInstructions;
2185 if (SpeculatedInstructions > 1)
2186 return false;
2187 }
2188
2189 // Check that we can insert the selects and that it's not too expensive to do
2190 // so.
2191 bool Convert = SpeculatedStore != nullptr;
2192 Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB,
2193 SpeculatedInstructions,
2194 BudgetRemaining, TTI);
2195 if (!Convert || BudgetRemaining < 0)
2196 return false;
2197
2198 // If we get here, we can hoist the instruction and if-convert.
2199 LLVM_DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SPECULATIVELY EXECUTING BB"
<< *ThenBB << "\n";; } } while (false)
;
2200
2201 // Insert a select of the value of the speculated store.
2202 if (SpeculatedStoreValue) {
2203 IRBuilder<NoFolder> Builder(BI);
2204 Value *TrueV = SpeculatedStore->getValueOperand();
2205 Value *FalseV = SpeculatedStoreValue;
2206 if (Invert)
2207 std::swap(TrueV, FalseV);
2208 Value *S = Builder.CreateSelect(
2209 BrCond, TrueV, FalseV, "spec.store.select", BI);
2210 SpeculatedStore->setOperand(0, S);
2211 SpeculatedStore->applyMergedLocation(BI->getDebugLoc(),
2212 SpeculatedStore->getDebugLoc());
2213 }
2214
2215 // Metadata can be dependent on the condition we are hoisting above.
2216 // Conservatively strip all metadata on the instruction. Drop the debug loc
2217 // to avoid making it appear as if the condition is a constant, which would
2218 // be misleading while debugging.
2219 for (auto &I : *ThenBB) {
2220 if (!SpeculatedStoreValue || &I != SpeculatedStore)
2221 I.setDebugLoc(DebugLoc());
2222 I.dropUnknownNonDebugMetadata();
2223 }
2224
2225 // Hoist the instructions.
2226 BB->getInstList().splice(BI->getIterator(), ThenBB->getInstList(),
2227 ThenBB->begin(), std::prev(ThenBB->end()));
2228
2229 // Insert selects and rewrite the PHI operands.
2230 IRBuilder<NoFolder> Builder(BI);
2231 for (PHINode &PN : EndBB->phis()) {
2232 unsigned OrigI = PN.getBasicBlockIndex(BB);
2233 unsigned ThenI = PN.getBasicBlockIndex(ThenBB);
2234 Value *OrigV = PN.getIncomingValue(OrigI);
2235 Value *ThenV = PN.getIncomingValue(ThenI);
2236
2237 // Skip PHIs which are trivial.
2238 if (OrigV == ThenV)
2239 continue;
2240
2241 // Create a select whose true value is the speculatively executed value and
2242 // false value is the pre-existing value. Swap them if the branch
2243 // destinations were inverted.
2244 Value *TrueV = ThenV, *FalseV = OrigV;
2245 if (Invert)
2246 std::swap(TrueV, FalseV);
2247 Value *V = Builder.CreateSelect(BrCond, TrueV, FalseV, "spec.select", BI);
2248 PN.setIncomingValue(OrigI, V);
2249 PN.setIncomingValue(ThenI, V);
2250 }
2251
2252 // Remove speculated dbg intrinsics.
2253 // FIXME: Is it possible to do this in a more elegant way? Moving/merging the
2254 // dbg value for the different flows and inserting it after the select.
2255 for (Instruction *I : SpeculatedDbgIntrinsics)
2256 I->eraseFromParent();
2257
2258 ++NumSpeculations;
2259 return true;
2260}
2261
2262/// Return true if we can thread a branch across this block.
2263static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
2264 int Size = 0;
2265
2266 for (Instruction &I : BB->instructionsWithoutDebug()) {
2267 if (Size > MaxSmallBlockSize)
2268 return false; // Don't clone large BB's.
2269
2270 // Can't fold blocks that contain noduplicate or convergent calls.
2271 if (CallInst *CI = dyn_cast<CallInst>(&I))
2272 if (CI->cannotDuplicate() || CI->isConvergent())
2273 return false;
2274
2275 // We will delete Phis while threading, so Phis should not be accounted in
2276 // block's size
2277 if (!isa<PHINode>(I))
2278 ++Size;
2279
2280 // We can only support instructions that do not define values that are
2281 // live outside of the current basic block.
2282 for (User *U : I.users()) {
2283 Instruction *UI = cast<Instruction>(U);
2284 if (UI->getParent() != BB || isa<PHINode>(UI))
2285 return false;
2286 }
2287
2288 // Looks ok, continue checking.
2289 }
2290
2291 return true;
2292}
2293
2294/// If we have a conditional branch on a PHI node value that is defined in the
2295/// same block as the branch and if any PHI entries are constants, thread edges
2296/// corresponding to that entry to be branches to their ultimate destination.
2297static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout &DL,
2298 AssumptionCache *AC) {
2299 BasicBlock *BB = BI->getParent();
2300 PHINode *PN = dyn_cast<PHINode>(BI->getCondition());
2301 // NOTE: we currently cannot transform this case if the PHI node is used
2302 // outside of the block.
2303 if (!PN || PN->getParent() != BB || !PN->hasOneUse())
2304 return false;
2305
2306 // Degenerate case of a single entry PHI.
2307 if (PN->getNumIncomingValues() == 1) {
2308 FoldSingleEntryPHINodes(PN->getParent());
2309 return true;
2310 }
2311
2312 // Now we know that this block has multiple preds and two succs.
2313 if (!BlockIsSimpleEnoughToThreadThrough(BB))
2314 return false;
2315
2316 // Okay, this is a simple enough basic block. See if any phi values are
2317 // constants.
2318 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
2319 ConstantInt *CB = dyn_cast<ConstantInt>(PN->getIncomingValue(i));
2320 if (!CB || !CB->getType()->isIntegerTy(1))
2321 continue;
2322
2323 // Okay, we now know that all edges from PredBB should be revectored to
2324 // branch to RealDest.
2325 BasicBlock *PredBB = PN->getIncomingBlock(i);
2326 BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue());
2327
2328 if (RealDest == BB)
2329 continue; // Skip self loops.
2330 // Skip if the predecessor's terminator is an indirect branch.
2331 if (isa<IndirectBrInst>(PredBB->getTerminator()))
2332 continue;
2333
2334 // The dest block might have PHI nodes, other predecessors and other
2335 // difficult cases. Instead of being smart about this, just insert a new
2336 // block that jumps to the destination block, effectively splitting
2337 // the edge we are about to create.
2338 BasicBlock *EdgeBB =
2339 BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge",
2340 RealDest->getParent(), RealDest);
2341 BranchInst *CritEdgeBranch = BranchInst::Create(RealDest, EdgeBB);
2342 CritEdgeBranch->setDebugLoc(BI->getDebugLoc());
2343
2344 // Update PHI nodes.
2345 AddPredecessorToBlock(RealDest, EdgeBB, BB);
2346
2347 // BB may have instructions that are being threaded over. Clone these
2348 // instructions into EdgeBB. We know that there will be no uses of the
2349 // cloned instructions outside of EdgeBB.
2350 BasicBlock::iterator InsertPt = EdgeBB->begin();
2351 DenseMap<Value *, Value *> TranslateMap; // Track translated values.
2352 for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
2353 if (PHINode *PN = dyn_cast<PHINode>(BBI)) {
2354 TranslateMap[PN] = PN->getIncomingValueForBlock(PredBB);
2355 continue;
2356 }
2357 // Clone the instruction.
2358 Instruction *N = BBI->clone();
2359 if (BBI->hasName())
2360 N->setName(BBI->getName() + ".c");
2361
2362 // Update operands due to translation.
2363 for (User::op_iterator i = N->op_begin(), e = N->op_end(); i != e; ++i) {
2364 DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(*i);
2365 if (PI != TranslateMap.end())
2366 *i = PI->second;
2367 }
2368
2369 // Check for trivial simplification.
2370 if (Value *V = SimplifyInstruction(N, {DL, nullptr, nullptr, AC})) {
2371 if (!BBI->use_empty())
2372 TranslateMap[&*BBI] = V;
2373 if (!N->mayHaveSideEffects()) {
2374 N->deleteValue(); // Instruction folded away, don't need actual inst
2375 N = nullptr;
2376 }
2377 } else {
2378 if (!BBI->use_empty())
2379 TranslateMap[&*BBI] = N;
2380 }
2381 if (N) {
2382 // Insert the new instruction into its new home.
2383 EdgeBB->getInstList().insert(InsertPt, N);
2384
2385 // Register the new instruction with the assumption cache if necessary.
2386 if (AC && match(N, m_Intrinsic<Intrinsic::assume>()))
2387 AC->registerAssumption(cast<IntrinsicInst>(N));
2388 }
2389 }
2390
2391 // Loop over all of the edges from PredBB to BB, changing them to branch
2392 // to EdgeBB instead.
2393 Instruction *PredBBTI = PredBB->getTerminator();
2394 for (unsigned i = 0, e = PredBBTI->getNumSuccessors(); i != e; ++i)
2395 if (PredBBTI->getSuccessor(i) == BB) {
2396 BB->removePredecessor(PredBB);
2397 PredBBTI->setSuccessor(i, EdgeBB);
2398 }
2399
2400 // Recurse, simplifying any other constants.
2401 return FoldCondBranchOnPHI(BI, DL, AC) || true;
2402 }
2403
2404 return false;
2405}
2406
2407/// Given a BB that starts with the specified two-entry PHI node,
2408/// see if we can eliminate it.
2409static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
2410 const DataLayout &DL) {
2411 // Ok, this is a two entry PHI node. Check to see if this is a simple "if
2412 // statement", which has a very simple dominance structure. Basically, we
2413 // are trying to find the condition that is being branched on, which
2414 // subsequently causes this merge to happen. We really want control
2415 // dependence information for this check, but simplifycfg can't keep it up
2416 // to date, and this catches most of the cases we care about anyway.
2417 BasicBlock *BB = PN->getParent();
2418
2419 BasicBlock *IfTrue, *IfFalse;
2420 Value *IfCond = GetIfCondition(BB, IfTrue, IfFalse);
2421 if (!IfCond ||
2422 // Don't bother if the branch will be constant folded trivially.
2423 isa<ConstantInt>(IfCond))
2424 return false;
2425
2426 // Okay, we found that we can merge this two-entry phi node into a select.
2427 // Doing so would require us to fold *all* two entry phi nodes in this block.
2428 // At some point this becomes non-profitable (particularly if the target
2429 // doesn't support cmov's). Only do this transformation if there are two or
2430 // fewer PHI nodes in this block.
2431 unsigned NumPhis = 0;
2432 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I)
2433 if (NumPhis > 2)
2434 return false;
2435
2436 // Loop over the PHI's seeing if we can promote them all to select
2437 // instructions. While we are at it, keep track of the instructions
2438 // that need to be moved to the dominating block.
2439 SmallPtrSet<Instruction *, 4> AggressiveInsts;
2440 int BudgetRemaining =
2441 TwoEntryPHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2442
2443 bool Changed = false;
2444 for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
2445 PHINode *PN = cast<PHINode>(II++);
2446 if (Value *V = SimplifyInstruction(PN, {DL, PN})) {
2447 PN->replaceAllUsesWith(V);
2448 PN->eraseFromParent();
2449 Changed = true;
2450 continue;
2451 }
2452
2453 if (!DominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts,
2454 BudgetRemaining, TTI) ||
2455 !DominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts,
2456 BudgetRemaining, TTI))
2457 return Changed;
2458 }
2459
2460 // If we folded the first phi, PN dangles at this point. Refresh it. If
2461 // we ran out of PHIs then we simplified them all.
2462 PN = dyn_cast<PHINode>(BB->begin());
2463 if (!PN)
2464 return true;
2465
2466 // Return true if at least one of these is a 'not', and another is either
2467 // a 'not' too, or a constant.
2468 auto CanHoistNotFromBothValues = [](Value *V0, Value *V1) {
2469 if (!match(V0, m_Not(m_Value())))
2470 std::swap(V0, V1);
2471 auto Invertible = m_CombineOr(m_Not(m_Value()), m_AnyIntegralConstant());
2472 return match(V0, m_Not(m_Value())) && match(V1, Invertible);
2473 };
2474
2475 // Don't fold i1 branches on PHIs which contain binary operators, unless one
2476 // of the incoming values is an 'not' and another one is freely invertible.
2477 // These can often be turned into switches and other things.
2478 if (PN->getType()->isIntegerTy(1) &&
2479 (isa<BinaryOperator>(PN->getIncomingValue(0)) ||
2480 isa<BinaryOperator>(PN->getIncomingValue(1)) ||
2481 isa<BinaryOperator>(IfCond)) &&
2482 !CanHoistNotFromBothValues(PN->getIncomingValue(0),
2483 PN->getIncomingValue(1)))
2484 return Changed;
2485
2486 // If all PHI nodes are promotable, check to make sure that all instructions
2487 // in the predecessor blocks can be promoted as well. If not, we won't be able
2488 // to get rid of the control flow, so it's not worth promoting to select
2489 // instructions.
2490 BasicBlock *DomBlock = nullptr;
2491 BasicBlock *IfBlock1 = PN->getIncomingBlock(0);
2492 BasicBlock *IfBlock2 = PN->getIncomingBlock(1);
2493 if (cast<BranchInst>(IfBlock1->getTerminator())->isConditional()) {
2494 IfBlock1 = nullptr;
2495 } else {
2496 DomBlock = *pred_begin(IfBlock1);
2497 for (BasicBlock::iterator I = IfBlock1->begin(); !I->isTerminator(); ++I)
2498 if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I)) {
2499 // This is not an aggressive instruction that we can promote.
2500 // Because of this, we won't be able to get rid of the control flow, so
2501 // the xform is not worth it.
2502 return Changed;
2503 }
2504 }
2505
2506 if (cast<BranchInst>(IfBlock2->getTerminator())->isConditional()) {
2507 IfBlock2 = nullptr;
2508 } else {
2509 DomBlock = *pred_begin(IfBlock2);
2510 for (BasicBlock::iterator I = IfBlock2->begin(); !I->isTerminator(); ++I)
2511 if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I)) {
2512 // This is not an aggressive instruction that we can promote.
2513 // Because of this, we won't be able to get rid of the control flow, so
2514 // the xform is not worth it.
2515 return Changed;
2516 }
2517 }
2518 assert(DomBlock && "Failed to find root DomBlock")((DomBlock && "Failed to find root DomBlock") ? static_cast
<void> (0) : __assert_fail ("DomBlock && \"Failed to find root DomBlock\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2518, __PRETTY_FUNCTION__))
;
2519
2520 LLVM_DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfConddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " <<
*IfCond << " T: " << IfTrue->getName() <<
" F: " << IfFalse->getName() << "\n"; } } while
(false)
2521 << " T: " << IfTrue->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " <<
*IfCond << " T: " << IfTrue->getName() <<
" F: " << IfFalse->getName() << "\n"; } } while
(false)
2522 << " F: " << IfFalse->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " <<
*IfCond << " T: " << IfTrue->getName() <<
" F: " << IfFalse->getName() << "\n"; } } while
(false)
;
2523
2524 // If we can still promote the PHI nodes after this gauntlet of tests,
2525 // do all of the PHI's now.
2526 Instruction *InsertPt = DomBlock->getTerminator();
2527 IRBuilder<NoFolder> Builder(InsertPt);
2528
2529 // Move all 'aggressive' instructions, which are defined in the
2530 // conditional parts of the if's up to the dominating block.
2531 if (IfBlock1)
2532 hoistAllInstructionsInto(DomBlock, InsertPt, IfBlock1);
2533 if (IfBlock2)
2534 hoistAllInstructionsInto(DomBlock, InsertPt, IfBlock2);
2535
2536 // Propagate fast-math-flags from phi nodes to replacement selects.
2537 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2538 while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
2539 if (isa<FPMathOperator>(PN))
2540 Builder.setFastMathFlags(PN->getFastMathFlags());
2541
2542 // Change the PHI node into a select instruction.
2543 Value *TrueVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfFalse);
2544 Value *FalseVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfTrue);
2545
2546 Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", InsertPt);
2547 PN->replaceAllUsesWith(Sel);
2548 Sel->takeName(PN);
2549 PN->eraseFromParent();
2550 }
2551
2552 // At this point, IfBlock1 and IfBlock2 are both empty, so our if statement
2553 // has been flattened. Change DomBlock to jump directly to our new block to
2554 // avoid other simplifycfg's kicking in on the diamond.
2555 Instruction *OldTI = DomBlock->getTerminator();
2556 Builder.SetInsertPoint(OldTI);
2557 Builder.CreateBr(BB);
2558 OldTI->eraseFromParent();
2559 return true;
2560}
2561
2562/// If we found a conditional branch that goes to two returning blocks,
2563/// try to merge them together into one return,
2564/// introducing a select if the return values disagree.
2565bool SimplifyCFGOpt::SimplifyCondBranchToTwoReturns(BranchInst *BI,
2566 IRBuilder<> &Builder) {
2567 assert(BI->isConditional() && "Must be a conditional branch")((BI->isConditional() && "Must be a conditional branch"
) ? static_cast<void> (0) : __assert_fail ("BI->isConditional() && \"Must be a conditional branch\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2567, __PRETTY_FUNCTION__))
;
2568 BasicBlock *TrueSucc = BI->getSuccessor(0);
2569 BasicBlock *FalseSucc = BI->getSuccessor(1);
2570 ReturnInst *TrueRet = cast<ReturnInst>(TrueSucc->getTerminator());
2571 ReturnInst *FalseRet = cast<ReturnInst>(FalseSucc->getTerminator());
2572
2573 // Check to ensure both blocks are empty (just a return) or optionally empty
2574 // with PHI nodes. If there are other instructions, merging would cause extra
2575 // computation on one path or the other.
2576 if (!TrueSucc->getFirstNonPHIOrDbg()->isTerminator())
2577 return false;
2578 if (!FalseSucc->getFirstNonPHIOrDbg()->isTerminator())
2579 return false;
2580
2581 Builder.SetInsertPoint(BI);
2582 // Okay, we found a branch that is going to two return nodes. If
2583 // there is no return value for this function, just change the
2584 // branch into a return.
2585 if (FalseRet->getNumOperands() == 0) {
2586 TrueSucc->removePredecessor(BI->getParent());
2587 FalseSucc->removePredecessor(BI->getParent());
2588 Builder.CreateRetVoid();
2589 EraseTerminatorAndDCECond(BI);
2590 return true;
2591 }
2592
2593 // Otherwise, figure out what the true and false return values are
2594 // so we can insert a new select instruction.
2595 Value *TrueValue = TrueRet->getReturnValue();
2596 Value *FalseValue = FalseRet->getReturnValue();
2597
2598 // Unwrap any PHI nodes in the return blocks.
2599 if (PHINode *TVPN = dyn_cast_or_null<PHINode>(TrueValue))
2600 if (TVPN->getParent() == TrueSucc)
2601 TrueValue = TVPN->getIncomingValueForBlock(BI->getParent());
2602 if (PHINode *FVPN = dyn_cast_or_null<PHINode>(FalseValue))
2603 if (FVPN->getParent() == FalseSucc)
2604 FalseValue = FVPN->getIncomingValueForBlock(BI->getParent());
2605
2606 // In order for this transformation to be safe, we must be able to
2607 // unconditionally execute both operands to the return. This is
2608 // normally the case, but we could have a potentially-trapping
2609 // constant expression that prevents this transformation from being
2610 // safe.
2611 if (ConstantExpr *TCV = dyn_cast_or_null<ConstantExpr>(TrueValue))
2612 if (TCV->canTrap())
2613 return false;
2614 if (ConstantExpr *FCV = dyn_cast_or_null<ConstantExpr>(FalseValue))
2615 if (FCV->canTrap())
2616 return false;
2617
2618 // Okay, we collected all the mapped values and checked them for sanity, and
2619 // defined to really do this transformation. First, update the CFG.
2620 TrueSucc->removePredecessor(BI->getParent());
2621 FalseSucc->removePredecessor(BI->getParent());
2622
2623 // Insert select instructions where needed.
2624 Value *BrCond = BI->getCondition();
2625 if (TrueValue) {
2626 // Insert a select if the results differ.
2627 if (TrueValue == FalseValue || isa<UndefValue>(FalseValue)) {
2628 } else if (isa<UndefValue>(TrueValue)) {
2629 TrueValue = FalseValue;
2630 } else {
2631 TrueValue =
2632 Builder.CreateSelect(BrCond, TrueValue, FalseValue, "retval", BI);
2633 }
2634 }
2635
2636 Value *RI =
2637 !TrueValue ? Builder.CreateRetVoid() : Builder.CreateRet(TrueValue);
2638
2639 (void)RI;
2640
2641 LLVM_DEBUG(dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
<< "\n " << *BI << "\nNewRet = " <<
*RI << "\nTRUEBLOCK: " << *TrueSucc << "\nFALSEBLOCK: "
<< *FalseSucc; } } while (false)
2642 << "\n " << *BI << "\nNewRet = " << *RI << "\nTRUEBLOCK: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
<< "\n " << *BI << "\nNewRet = " <<
*RI << "\nTRUEBLOCK: " << *TrueSucc << "\nFALSEBLOCK: "
<< *FalseSucc; } } while (false)
2643 << *TrueSucc << "\nFALSEBLOCK: " << *FalseSucc)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
<< "\n " << *BI << "\nNewRet = " <<
*RI << "\nTRUEBLOCK: " << *TrueSucc << "\nFALSEBLOCK: "
<< *FalseSucc; } } while (false)
;
2644
2645 EraseTerminatorAndDCECond(BI);
2646
2647 return true;
2648}
2649
2650/// Return true if the given instruction is available
2651/// in its predecessor block. If yes, the instruction will be removed.
2652static bool tryCSEWithPredecessor(Instruction *Inst, BasicBlock *PB) {
2653 if (!isa<BinaryOperator>(Inst) && !isa<CmpInst>(Inst))
2654 return false;
2655 for (Instruction &I : *PB) {
2656 Instruction *PBI = &I;
2657 // Check whether Inst and PBI generate the same value.
2658 if (Inst->isIdenticalTo(PBI)) {
2659 Inst->replaceAllUsesWith(PBI);
2660 Inst->eraseFromParent();
2661 return true;
2662 }
2663 }
2664 return false;
2665}
2666
2667/// Return true if either PBI or BI has branch weight available, and store
2668/// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does
2669/// not have branch weight, use 1:1 as its weight.
2670static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI,
2671 uint64_t &PredTrueWeight,
2672 uint64_t &PredFalseWeight,
2673 uint64_t &SuccTrueWeight,
2674 uint64_t &SuccFalseWeight) {
2675 bool PredHasWeights =
2676 PBI->extractProfMetadata(PredTrueWeight, PredFalseWeight);
2677 bool SuccHasWeights =
2678 BI->extractProfMetadata(SuccTrueWeight, SuccFalseWeight);
2679 if (PredHasWeights || SuccHasWeights) {
2680 if (!PredHasWeights)
2681 PredTrueWeight = PredFalseWeight = 1;
2682 if (!SuccHasWeights)
2683 SuccTrueWeight = SuccFalseWeight = 1;
2684 return true;
2685 } else {
2686 return false;
2687 }
2688}
2689
2690/// If this basic block is simple enough, and if a predecessor branches to us
2691/// and one of our successors, fold the block into the predecessor and use
2692/// logical operations to pick the right destination.
2693bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU,
2694 const TargetTransformInfo *TTI,
2695 unsigned BonusInstThreshold) {
2696 BasicBlock *BB = BI->getParent();
2697
2698 const unsigned PredCount = pred_size(BB);
2699
2700 bool Changed = false;
2701 TargetTransformInfo::TargetCostKind CostKind =
2702 BB->getParent()->hasMinSize() ? TargetTransformInfo::TCK_CodeSize
2703 : TargetTransformInfo::TCK_SizeAndLatency;
2704
2705 Instruction *Cond = nullptr;
2706 if (BI->isConditional())
2707 Cond = dyn_cast<Instruction>(BI->getCondition());
2708 else {
2709 // For unconditional branch, check for a simple CFG pattern, where
2710 // BB has a single predecessor and BB's successor is also its predecessor's
2711 // successor. If such pattern exists, check for CSE between BB and its
2712 // predecessor.
2713 if (BasicBlock *PB = BB->getSinglePredecessor())
2714 if (BranchInst *PBI = dyn_cast<BranchInst>(PB->getTerminator()))
2715 if (PBI->isConditional() &&
2716 (BI->getSuccessor(0) == PBI->getSuccessor(0) ||
2717 BI->getSuccessor(0) == PBI->getSuccessor(1))) {
2718 for (auto I = BB->instructionsWithoutDebug().begin(),
2719 E = BB->instructionsWithoutDebug().end();
2720 I != E;) {
2721 Instruction *Curr = &*I++;
2722 if (isa<CmpInst>(Curr)) {
2723 Cond = Curr;
2724 break;
2725 }
2726 // Quit if we can't remove this instruction.
2727 if (!tryCSEWithPredecessor(Curr, PB))
2728 return Changed;
2729 Changed = true;
2730 }
2731 }
2732
2733 if (!Cond)
2734 return Changed;
2735 }
2736
2737 if (!Cond || (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond)) ||
2738 Cond->getParent() != BB || !Cond->hasOneUse())
2739 return Changed;
2740
2741 // Make sure the instruction after the condition is the cond branch.
2742 BasicBlock::iterator CondIt = ++Cond->getIterator();
2743
2744 // Ignore dbg intrinsics.
2745 while (isa<DbgInfoIntrinsic>(CondIt))
2746 ++CondIt;
2747
2748 if (&*CondIt != BI)
2749 return Changed;
2750
2751 // Only allow this transformation if computing the condition doesn't involve
2752 // too many instructions and these involved instructions can be executed
2753 // unconditionally. We denote all involved instructions except the condition
2754 // as "bonus instructions", and only allow this transformation when the
2755 // number of the bonus instructions we'll need to create when cloning into
2756 // each predecessor does not exceed a certain threshold.
2757 unsigned NumBonusInsts = 0;
2758 for (auto I = BB->begin(); Cond != &*I; ++I) {
2759 // Ignore dbg intrinsics.
2760 if (isa<DbgInfoIntrinsic>(I))
2761 continue;
2762 if (!I->hasOneUse() || !isSafeToSpeculativelyExecute(&*I))
2763 return Changed;
2764 // I has only one use and can be executed unconditionally.
2765 Instruction *User = dyn_cast<Instruction>(I->user_back());
2766 if (User == nullptr || User->getParent() != BB)
2767 return Changed;
2768 // I is used in the same BB. Since BI uses Cond and doesn't have more slots
2769 // to use any other instruction, User must be an instruction between next(I)
2770 // and Cond.
2771
2772 // Account for the cost of duplicating this instruction into each
2773 // predecessor.
2774 NumBonusInsts += PredCount;
2775 // Early exits once we reach the limit.
2776 if (NumBonusInsts > BonusInstThreshold)
2777 return Changed;
2778 }
2779
2780 // Cond is known to be a compare or binary operator. Check to make sure that
2781 // neither operand is a potentially-trapping constant expression.
2782 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(0)))
2783 if (CE->canTrap())
2784 return Changed;
2785 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(1)))
2786 if (CE->canTrap())
2787 return Changed;
2788
2789 // Finally, don't infinitely unroll conditional loops.
2790 BasicBlock *TrueDest = BI->getSuccessor(0);
2791 BasicBlock *FalseDest = (BI->isConditional()) ? BI->getSuccessor(1) : nullptr;
2792 if (TrueDest == BB || FalseDest == BB)
2793 return Changed;
2794
2795 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
2796 BasicBlock *PredBlock = *PI;
2797 BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
2798
2799 // Check that we have two conditional branches. If there is a PHI node in
2800 // the common successor, verify that the same value flows in from both
2801 // blocks.
2802 SmallVector<PHINode *, 4> PHIs;
2803 if (!PBI || PBI->isUnconditional() ||
2804 (BI->isConditional() && !SafeToMergeTerminators(BI, PBI)) ||
2805 (!BI->isConditional() &&
2806 !isProfitableToFoldUnconditional(BI, PBI, Cond, PHIs)))
2807 continue;
2808
2809 // Determine if the two branches share a common destination.
2810 Instruction::BinaryOps Opc = Instruction::BinaryOpsEnd;
2811 bool InvertPredCond = false;
2812
2813 if (BI->isConditional()) {
2814 if (PBI->getSuccessor(0) == TrueDest) {
2815 Opc = Instruction::Or;
2816 } else if (PBI->getSuccessor(1) == FalseDest) {
2817 Opc = Instruction::And;
2818 } else if (PBI->getSuccessor(0) == FalseDest) {
2819 Opc = Instruction::And;
2820 InvertPredCond = true;
2821 } else if (PBI->getSuccessor(1) == TrueDest) {
2822 Opc = Instruction::Or;
2823 InvertPredCond = true;
2824 } else {
2825 continue;
2826 }
2827 } else {
2828 if (PBI->getSuccessor(0) != TrueDest && PBI->getSuccessor(1) != TrueDest)
2829 continue;
2830 }
2831
2832 // Check the cost of inserting the necessary logic before performing the
2833 // transformation.
2834 if (TTI && Opc != Instruction::BinaryOpsEnd) {
2835 Type *Ty = BI->getCondition()->getType();
2836 unsigned Cost = TTI->getArithmeticInstrCost(Opc, Ty, CostKind);
2837 if (InvertPredCond && (!PBI->getCondition()->hasOneUse() ||
2838 !isa<CmpInst>(PBI->getCondition())))
2839 Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind);
2840
2841 if (Cost > BranchFoldThreshold)
2842 continue;
2843 }
2844
2845 LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING BRANCH TO COMMON DEST:\n"
<< *PBI << *BB; } } while (false)
;
2846 Changed = true;
2847
2848 IRBuilder<> Builder(PBI);
2849
2850 // If we need to invert the condition in the pred block to match, do so now.
2851 if (InvertPredCond) {
2852 Value *NewCond = PBI->getCondition();
2853
2854 if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) {
2855 CmpInst *CI = cast<CmpInst>(NewCond);
2856 CI->setPredicate(CI->getInversePredicate());
2857 } else {
2858 NewCond =
2859 Builder.CreateNot(NewCond, PBI->getCondition()->getName() + ".not");
2860 }
2861
2862 PBI->setCondition(NewCond);
2863 PBI->swapSuccessors();
2864 }
2865
2866 // If we have bonus instructions, clone them into the predecessor block.
2867 // Note that there may be multiple predecessor blocks, so we cannot move
2868 // bonus instructions to a predecessor block.
2869 ValueToValueMapTy VMap; // maps original values to cloned values
2870 // We already make sure Cond is the last instruction before BI. Therefore,
2871 // all instructions before Cond other than DbgInfoIntrinsic are bonus
2872 // instructions.
2873 for (auto BonusInst = BB->begin(); Cond != &*BonusInst; ++BonusInst) {
2874 if (isa<DbgInfoIntrinsic>(BonusInst))
2875 continue;
2876 Instruction *NewBonusInst = BonusInst->clone();
2877
2878 // When we fold the bonus instructions we want to make sure we
2879 // reset their debug locations in order to avoid stepping on dead
2880 // code caused by folding dead branches.
2881 NewBonusInst->setDebugLoc(DebugLoc());
2882
2883 RemapInstruction(NewBonusInst, VMap,
2884 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
2885 VMap[&*BonusInst] = NewBonusInst;
2886
2887 // If we moved a load, we cannot any longer claim any knowledge about
2888 // its potential value. The previous information might have been valid
2889 // only given the branch precondition.
2890 // For an analogous reason, we must also drop all the metadata whose
2891 // semantics we don't understand.
2892 NewBonusInst->dropUnknownNonDebugMetadata();
2893
2894 PredBlock->getInstList().insert(PBI->getIterator(), NewBonusInst);
2895 NewBonusInst->takeName(&*BonusInst);
2896 BonusInst->setName(BonusInst->getName() + ".old");
2897 }
2898
2899 // Clone Cond into the predecessor basic block, and or/and the
2900 // two conditions together.
2901 Instruction *CondInPred = Cond->clone();
2902
2903 // Reset the condition debug location to avoid jumping on dead code
2904 // as the result of folding dead branches.
2905 CondInPred->setDebugLoc(DebugLoc());
2906
2907 RemapInstruction(CondInPred, VMap,
2908 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
2909 PredBlock->getInstList().insert(PBI->getIterator(), CondInPred);
2910 CondInPred->takeName(Cond);
2911 Cond->setName(CondInPred->getName() + ".old");
2912
2913 if (BI->isConditional()) {
2914 Instruction *NewCond = cast<Instruction>(
2915 Builder.CreateBinOp(Opc, PBI->getCondition(), CondInPred, "or.cond"));
2916 PBI->setCondition(NewCond);
2917
2918 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
2919 bool HasWeights =
2920 extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
2921 SuccTrueWeight, SuccFalseWeight);
2922 SmallVector<uint64_t, 8> NewWeights;
2923
2924 if (PBI->getSuccessor(0) == BB) {
2925 if (HasWeights) {
2926 // PBI: br i1 %x, BB, FalseDest
2927 // BI: br i1 %y, TrueDest, FalseDest
2928 // TrueWeight is TrueWeight for PBI * TrueWeight for BI.
2929 NewWeights.push_back(PredTrueWeight * SuccTrueWeight);
2930 // FalseWeight is FalseWeight for PBI * TotalWeight for BI +
2931 // TrueWeight for PBI * FalseWeight for BI.
2932 // We assume that total weights of a BranchInst can fit into 32 bits.
2933 // Therefore, we will not have overflow using 64-bit arithmetic.
2934 NewWeights.push_back(PredFalseWeight *
2935 (SuccFalseWeight + SuccTrueWeight) +
2936 PredTrueWeight * SuccFalseWeight);
2937 }
2938 AddPredecessorToBlock(TrueDest, PredBlock, BB, MSSAU);
2939 PBI->setSuccessor(0, TrueDest);
2940 }
2941 if (PBI->getSuccessor(1) == BB) {
2942 if (HasWeights) {
2943 // PBI: br i1 %x, TrueDest, BB
2944 // BI: br i1 %y, TrueDest, FalseDest
2945 // TrueWeight is TrueWeight for PBI * TotalWeight for BI +
2946 // FalseWeight for PBI * TrueWeight for BI.
2947 NewWeights.push_back(PredTrueWeight *
2948 (SuccFalseWeight + SuccTrueWeight) +
2949 PredFalseWeight * SuccTrueWeight);
2950 // FalseWeight is FalseWeight for PBI * FalseWeight for BI.
2951 NewWeights.push_back(PredFalseWeight * SuccFalseWeight);
2952 }
2953 AddPredecessorToBlock(FalseDest, PredBlock, BB, MSSAU);
2954 PBI->setSuccessor(1, FalseDest);
2955 }
2956 if (NewWeights.size() == 2) {
2957 // Halve the weights if any of them cannot fit in an uint32_t
2958 FitWeights(NewWeights);
2959
2960 SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(),
2961 NewWeights.end());
2962 setBranchWeights(PBI, MDWeights[0], MDWeights[1]);
2963 } else
2964 PBI->setMetadata(LLVMContext::MD_prof, nullptr);
2965 } else {
2966 // Update PHI nodes in the common successors.
2967 for (unsigned i = 0, e = PHIs.size(); i != e; ++i) {
2968 ConstantInt *PBI_C = cast<ConstantInt>(
2969 PHIs[i]->getIncomingValueForBlock(PBI->getParent()));
2970 assert(PBI_C->getType()->isIntegerTy(1))((PBI_C->getType()->isIntegerTy(1)) ? static_cast<void
> (0) : __assert_fail ("PBI_C->getType()->isIntegerTy(1)"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2970, __PRETTY_FUNCTION__))
;
2971 Instruction *MergedCond = nullptr;
2972 if (PBI->getSuccessor(0) == TrueDest) {
2973 // Create (PBI_Cond and PBI_C) or (!PBI_Cond and BI_Value)
2974 // PBI_C is true: PBI_Cond or (!PBI_Cond and BI_Value)
2975 // is false: !PBI_Cond and BI_Value
2976 Instruction *NotCond = cast<Instruction>(
2977 Builder.CreateNot(PBI->getCondition(), "not.cond"));
2978 MergedCond = cast<Instruction>(
2979 Builder.CreateBinOp(Instruction::And, NotCond, CondInPred,
2980 "and.cond"));
2981 if (PBI_C->isOne())
2982 MergedCond = cast<Instruction>(Builder.CreateBinOp(
2983 Instruction::Or, PBI->getCondition(), MergedCond, "or.cond"));
2984 } else {
2985 // Create (PBI_Cond and BI_Value) or (!PBI_Cond and PBI_C)
2986 // PBI_C is true: (PBI_Cond and BI_Value) or (!PBI_Cond)
2987 // is false: PBI_Cond and BI_Value
2988 MergedCond = cast<Instruction>(Builder.CreateBinOp(
2989 Instruction::And, PBI->getCondition(), CondInPred, "and.cond"));
2990 if (PBI_C->isOne()) {
2991 Instruction *NotCond = cast<Instruction>(
2992 Builder.CreateNot(PBI->getCondition(), "not.cond"));
2993 MergedCond = cast<Instruction>(Builder.CreateBinOp(
2994 Instruction::Or, NotCond, MergedCond, "or.cond"));
2995 }
2996 }
2997 // Update PHI Node.
2998 PHIs[i]->setIncomingValueForBlock(PBI->getParent(), MergedCond);
2999 }
3000
3001 // PBI is changed to branch to TrueDest below. Remove itself from
3002 // potential phis from all other successors.
3003 if (MSSAU)
3004 MSSAU->changeCondBranchToUnconditionalTo(PBI, TrueDest);
3005
3006 // Change PBI from Conditional to Unconditional.
3007 BranchInst *New_PBI = BranchInst::Create(TrueDest, PBI);
3008 EraseTerminatorAndDCECond(PBI, MSSAU);
3009 PBI = New_PBI;
3010 }
3011
3012 // If BI was a loop latch, it may have had associated loop metadata.
3013 // We need to copy it to the new latch, that is, PBI.
3014 if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop))
3015 PBI->setMetadata(LLVMContext::MD_loop, LoopMD);
3016
3017 // TODO: If BB is reachable from all paths through PredBlock, then we
3018 // could replace PBI's branch probabilities with BI's.
3019
3020 // Copy any debug value intrinsics into the end of PredBlock.
3021 for (Instruction &I : *BB) {
3022 if (isa<DbgInfoIntrinsic>(I)) {
3023 Instruction *NewI = I.clone();
3024 RemapInstruction(NewI, VMap,
3025 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
3026 NewI->insertBefore(PBI);
3027 }
3028 }
3029
3030 return Changed;
3031 }
3032 return Changed;
3033}
3034
3035// If there is only one store in BB1 and BB2, return it, otherwise return
3036// nullptr.
3037static StoreInst *findUniqueStoreInBlocks(BasicBlock *BB1, BasicBlock *BB2) {
3038 StoreInst *S = nullptr;
3039 for (auto *BB : {BB1, BB2}) {
3040 if (!BB)
3041 continue;
3042 for (auto &I : *BB)
3043 if (auto *SI = dyn_cast<StoreInst>(&I)) {
3044 if (S)
3045 // Multiple stores seen.
3046 return nullptr;
3047 else
3048 S = SI;
3049 }
3050 }
3051 return S;
3052}
3053
3054static Value *ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB,
3055 Value *AlternativeV = nullptr) {
3056 // PHI is going to be a PHI node that allows the value V that is defined in
3057 // BB to be referenced in BB's only successor.
3058 //
3059 // If AlternativeV is nullptr, the only value we care about in PHI is V. It
3060 // doesn't matter to us what the other operand is (it'll never get used). We
3061 // could just create a new PHI with an undef incoming value, but that could
3062 // increase register pressure if EarlyCSE/InstCombine can't fold it with some
3063 // other PHI. So here we directly look for some PHI in BB's successor with V
3064 // as an incoming operand. If we find one, we use it, else we create a new
3065 // one.
3066 //
3067 // If AlternativeV is not nullptr, we care about both incoming values in PHI.
3068 // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV]
3069 // where OtherBB is the single other predecessor of BB's only successor.
3070 PHINode *PHI = nullptr;
3071 BasicBlock *Succ = BB->getSingleSuccessor();
3072
3073 for (auto I = Succ->begin(); isa<PHINode>(I); ++I)
3074 if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) {
3075 PHI = cast<PHINode>(I);
3076 if (!AlternativeV)
3077 break;
3078
3079 assert(Succ->hasNPredecessors(2))((Succ->hasNPredecessors(2)) ? static_cast<void> (0)
: __assert_fail ("Succ->hasNPredecessors(2)", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3079, __PRETTY_FUNCTION__))
;
3080 auto PredI = pred_begin(Succ);
3081 BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI;
3082 if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV)
3083 break;
3084 PHI = nullptr;
3085 }
3086 if (PHI)
3087 return PHI;
3088
3089 // If V is not an instruction defined in BB, just return it.
3090 if (!AlternativeV &&
3091 (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB))
3092 return V;
3093
3094 PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge", &Succ->front());
3095 PHI->addIncoming(V, BB);
3096 for (BasicBlock *PredBB : predecessors(Succ))
3097 if (PredBB != BB)
3098 PHI->addIncoming(
3099 AlternativeV ? AlternativeV : UndefValue::get(V->getType()), PredBB);
3100 return PHI;
3101}
3102
3103static bool mergeConditionalStoreToAddress(BasicBlock *PTB, BasicBlock *PFB,
3104 BasicBlock *QTB, BasicBlock *QFB,
3105 BasicBlock *PostBB, Value *Address,
3106 bool InvertPCond, bool InvertQCond,
3107 const DataLayout &DL,
3108 const TargetTransformInfo &TTI) {
3109 // For every pointer, there must be exactly two stores, one coming from
3110 // PTB or PFB, and the other from QTB or QFB. We don't support more than one
3111 // store (to any address) in PTB,PFB or QTB,QFB.
3112 // FIXME: We could relax this restriction with a bit more work and performance
3113 // testing.
3114 StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB);
3115 StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB);
3116 if (!PStore || !QStore)
3117 return false;
3118
3119 // Now check the stores are compatible.
3120 if (!QStore->isUnordered() || !PStore->isUnordered())
3121 return false;
3122
3123 // Check that sinking the store won't cause program behavior changes. Sinking
3124 // the store out of the Q blocks won't change any behavior as we're sinking
3125 // from a block to its unconditional successor. But we're moving a store from
3126 // the P blocks down through the middle block (QBI) and past both QFB and QTB.
3127 // So we need to check that there are no aliasing loads or stores in
3128 // QBI, QTB and QFB. We also need to check there are no conflicting memory
3129 // operations between PStore and the end of its parent block.
3130 //
3131 // The ideal way to do this is to query AliasAnalysis, but we don't
3132 // preserve AA currently so that is dangerous. Be super safe and just
3133 // check there are no other memory operations at all.
3134 for (auto &I : *QFB->getSinglePredecessor())
3135 if (I.mayReadOrWriteMemory())
3136 return false;
3137 for (auto &I : *QFB)
3138 if (&I != QStore && I.mayReadOrWriteMemory())
3139 return false;
3140 if (QTB)
3141 for (auto &I : *QTB)
3142 if (&I != QStore && I.mayReadOrWriteMemory())
3143 return false;
3144 for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end();
3145 I != E; ++I)
3146 if (&*I != PStore && I->mayReadOrWriteMemory())
3147 return false;
3148
3149 // If we're not in aggressive mode, we only optimize if we have some
3150 // confidence that by optimizing we'll allow P and/or Q to be if-converted.
3151 auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) {
3152 if (!BB)
3153 return true;
3154 // Heuristic: if the block can be if-converted/phi-folded and the
3155 // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
3156 // thread this store.
3157 int BudgetRemaining =
3158 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
3159 for (auto &I : BB->instructionsWithoutDebug()) {
3160 // Consider terminator instruction to be free.
3161 if (I.isTerminator())
3162 continue;
3163 // If this is one the stores that we want to speculate out of this BB,
3164 // then don't count it's cost, consider it to be free.
3165 if (auto *S = dyn_cast<StoreInst>(&I))
3166 if (llvm::find(FreeStores, S))
3167 continue;
3168 // Else, we have a white-list of instructions that we are ak speculating.
3169 if (!isa<BinaryOperator>(I) && !isa<GetElementPtrInst>(I))
3170 return false; // Not in white-list - not worthwhile folding.
3171 // And finally, if this is a non-free instruction that we are okay
3172 // speculating, ensure that we consider the speculation budget.
3173 BudgetRemaining -= TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency);
3174 if (BudgetRemaining < 0)
3175 return false; // Eagerly refuse to fold as soon as we're out of budget.
3176 }
3177 assert(BudgetRemaining >= 0 &&((BudgetRemaining >= 0 && "When we run out of budget we will eagerly return from within the "
"per-instruction loop.") ? static_cast<void> (0) : __assert_fail
("BudgetRemaining >= 0 && \"When we run out of budget we will eagerly return from within the \" \"per-instruction loop.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3179, __PRETTY_FUNCTION__))
3178 "When we run out of budget we will eagerly return from within the "((BudgetRemaining >= 0 && "When we run out of budget we will eagerly return from within the "
"per-instruction loop.") ? static_cast<void> (0) : __assert_fail
("BudgetRemaining >= 0 && \"When we run out of budget we will eagerly return from within the \" \"per-instruction loop.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3179, __PRETTY_FUNCTION__))
3179 "per-instruction loop.")((BudgetRemaining >= 0 && "When we run out of budget we will eagerly return from within the "
"per-instruction loop.") ? static_cast<void> (0) : __assert_fail
("BudgetRemaining >= 0 && \"When we run out of budget we will eagerly return from within the \" \"per-instruction loop.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3179, __PRETTY_FUNCTION__))
;
3180 return true;
3181 };
3182
3183 const std::array<StoreInst *, 2> FreeStores = {PStore, QStore};
3184 if (!MergeCondStoresAggressively &&
3185 (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) ||
3186 !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores)))
3187 return false;
3188
3189 // If PostBB has more than two predecessors, we need to split it so we can
3190 // sink the store.
3191 if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) {
3192 // We know that QFB's only successor is PostBB. And QFB has a single
3193 // predecessor. If QTB exists, then its only successor is also PostBB.
3194 // If QTB does not exist, then QFB's only predecessor has a conditional
3195 // branch to QFB and PostBB.
3196 BasicBlock *TruePred = QTB ? QTB : QFB->getSinglePredecessor();
3197 BasicBlock *NewBB = SplitBlockPredecessors(PostBB, { QFB, TruePred},
3198 "condstore.split");
3199 if (!NewBB)
3200 return false;
3201 PostBB = NewBB;
3202 }
3203
3204 // OK, we're going to sink the stores to PostBB. The store has to be
3205 // conditional though, so first create the predicate.
3206 Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator())
3207 ->getCondition();
3208 Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator())
3209 ->getCondition();
3210
3211 Value *PPHI = ensureValueAvailableInSuccessor(PStore->getValueOperand(),
3212 PStore->getParent());
3213 Value *QPHI = ensureValueAvailableInSuccessor(QStore->getValueOperand(),
3214 QStore->getParent(), PPHI);
3215
3216 IRBuilder<> QB(&*PostBB->getFirstInsertionPt());
3217
3218 Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond);
3219 Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond);
3220
3221 if (InvertPCond)
3222 PPred = QB.CreateNot(PPred);
3223 if (InvertQCond)
3224 QPred = QB.CreateNot(QPred);
3225 Value *CombinedPred = QB.CreateOr(PPred, QPred);
3226
3227 auto *T =
3228 SplitBlockAndInsertIfThen(CombinedPred, &*QB.GetInsertPoint(), false);
3229 QB.SetInsertPoint(T);
3230 StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address));
3231 AAMDNodes AAMD;
3232 PStore->getAAMetadata(AAMD, /*Merge=*/false);
3233 PStore->getAAMetadata(AAMD, /*Merge=*/true);
3234 SI->setAAMetadata(AAMD);
3235 // Choose the minimum alignment. If we could prove both stores execute, we
3236 // could use biggest one. In this case, though, we only know that one of the
3237 // stores executes. And we don't know it's safe to take the alignment from a
3238 // store that doesn't execute.
3239 SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign()));
3240
3241 QStore->eraseFromParent();
3242 PStore->eraseFromParent();
3243
3244 return true;
3245}
3246
3247static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI,
3248 const DataLayout &DL,
3249 const TargetTransformInfo &TTI) {
3250 // The intention here is to find diamonds or triangles (see below) where each
3251 // conditional block contains a store to the same address. Both of these
3252 // stores are conditional, so they can't be unconditionally sunk. But it may
3253 // be profitable to speculatively sink the stores into one merged store at the
3254 // end, and predicate the merged store on the union of the two conditions of
3255 // PBI and QBI.
3256 //
3257 // This can reduce the number of stores executed if both of the conditions are
3258 // true, and can allow the blocks to become small enough to be if-converted.
3259 // This optimization will also chain, so that ladders of test-and-set
3260 // sequences can be if-converted away.
3261 //
3262 // We only deal with simple diamonds or triangles:
3263 //
3264 // PBI or PBI or a combination of the two
3265 // / \ | \
3266 // PTB PFB | PFB
3267 // \ / | /
3268 // QBI QBI
3269 // / \ | \
3270 // QTB QFB | QFB
3271 // \ / | /
3272 // PostBB PostBB
3273 //
3274 // We model triangles as a type of diamond with a nullptr "true" block.
3275 // Triangles are canonicalized so that the fallthrough edge is represented by
3276 // a true condition, as in the diagram above.
3277 BasicBlock *PTB = PBI->getSuccessor(0);
3278 BasicBlock *PFB = PBI->getSuccessor(1);
3279 BasicBlock *QTB = QBI->getSuccessor(0);
3280 BasicBlock *QFB = QBI->getSuccessor(1);
3281 BasicBlock *PostBB = QFB->getSingleSuccessor();
3282
3283 // Make sure we have a good guess for PostBB. If QTB's only successor is
3284 // QFB, then QFB is a better PostBB.
3285 if (QTB->getSingleSuccessor() == QFB)
3286 PostBB = QFB;
3287
3288 // If we couldn't find a good PostBB, stop.
3289 if (!PostBB)
3290 return false;
3291
3292 bool InvertPCond = false, InvertQCond = false;
3293 // Canonicalize fallthroughs to the true branches.
3294 if (PFB == QBI->getParent()) {
3295 std::swap(PFB, PTB);
3296 InvertPCond = true;
3297 }
3298 if (QFB == PostBB) {
3299 std::swap(QFB, QTB);
3300 InvertQCond = true;
3301 }
3302
3303 // From this point on we can assume PTB or QTB may be fallthroughs but PFB
3304 // and QFB may not. Model fallthroughs as a nullptr block.
3305 if (PTB == QBI->getParent())
3306 PTB = nullptr;
3307 if (QTB == PostBB)
3308 QTB = nullptr;
3309
3310 // Legality bailouts. We must have at least the non-fallthrough blocks and
3311 // the post-dominating block, and the non-fallthroughs must only have one
3312 // predecessor.
3313 auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) {
3314 return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S;
3315 };
3316 if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) ||
3317 !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB))
3318 return false;
3319 if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) ||
3320 (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB)))
3321 return false;
3322 if (!QBI->getParent()->hasNUses(2))
3323 return false;
3324
3325 // OK, this is a sequence of two diamonds or triangles.
3326 // Check if there are stores in PTB or PFB that are repeated in QTB or QFB.
3327 SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses;
3328 for (auto *BB : {PTB, PFB}) {
3329 if (!BB)
3330 continue;
3331 for (auto &I : *BB)
3332 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3333 PStoreAddresses.insert(SI->getPointerOperand());
3334 }
3335 for (auto *BB : {QTB, QFB}) {
3336 if (!BB)
3337 continue;
3338 for (auto &I : *BB)
3339 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3340 QStoreAddresses.insert(SI->getPointerOperand());
3341 }
3342
3343 set_intersect(PStoreAddresses, QStoreAddresses);
3344 // set_intersect mutates PStoreAddresses in place. Rename it here to make it
3345 // clear what it contains.
3346 auto &CommonAddresses = PStoreAddresses;
3347
3348 bool Changed = false;
3349 for (auto *Address : CommonAddresses)
3350 Changed |= mergeConditionalStoreToAddress(
3351 PTB, PFB, QTB, QFB, PostBB, Address, InvertPCond, InvertQCond, DL, TTI);
3352 return Changed;
3353}
3354
3355
3356/// If the previous block ended with a widenable branch, determine if reusing
3357/// the target block is profitable and legal. This will have the effect of
3358/// "widening" PBI, but doesn't require us to reason about hosting safety.
3359static bool tryWidenCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
3360 // TODO: This can be generalized in two important ways:
3361 // 1) We can allow phi nodes in IfFalseBB and simply reuse all the input
3362 // values from the PBI edge.
3363 // 2) We can sink side effecting instructions into BI's fallthrough
3364 // successor provided they doesn't contribute to computation of
3365 // BI's condition.
3366 Value *CondWB, *WC;
3367 BasicBlock *IfTrueBB, *IfFalseBB;
3368 if (!parseWidenableBranch(PBI, CondWB, WC, IfTrueBB, IfFalseBB) ||
3369 IfTrueBB != BI->getParent() || !BI->getParent()->getSinglePredecessor())
3370 return false;
3371 if (!IfFalseBB->phis().empty())
3372 return false; // TODO
3373 // Use lambda to lazily compute expensive condition after cheap ones.
3374 auto NoSideEffects = [](BasicBlock &BB) {
3375 return !llvm::any_of(BB, [](const Instruction &I) {
3376 return I.mayWriteToMemory() || I.mayHaveSideEffects();
3377 });
3378 };
3379 if (BI->getSuccessor(1) != IfFalseBB && // no inf looping
3380 BI->getSuccessor(1)->getTerminatingDeoptimizeCall() && // profitability
3381 NoSideEffects(*BI->getParent())) {
3382 BI->getSuccessor(1)->removePredecessor(BI->getParent());
3383 BI->setSuccessor(1, IfFalseBB);
3384 return true;
3385 }
3386 if (BI->getSuccessor(0) != IfFalseBB && // no inf looping
3387 BI->getSuccessor(0)->getTerminatingDeoptimizeCall() && // profitability
3388 NoSideEffects(*BI->getParent())) {
3389 BI->getSuccessor(0)->removePredecessor(BI->getParent());
3390 BI->setSuccessor(0, IfFalseBB);
3391 return true;
3392 }
3393 return false;
3394}
3395
3396/// If we have a conditional branch as a predecessor of another block,
3397/// this function tries to simplify it. We know
3398/// that PBI and BI are both conditional branches, and BI is in one of the
3399/// successor blocks of PBI - PBI branches to BI.
3400static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
3401 const DataLayout &DL,
3402 const TargetTransformInfo &TTI) {
3403 assert(PBI->isConditional() && BI->isConditional())((PBI->isConditional() && BI->isConditional()) ?
static_cast<void> (0) : __assert_fail ("PBI->isConditional() && BI->isConditional()"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3403, __PRETTY_FUNCTION__))
;
3404 BasicBlock *BB = BI->getParent();
3405
3406 // If this block ends with a branch instruction, and if there is a
3407 // predecessor that ends on a branch of the same condition, make
3408 // this conditional branch redundant.
3409 if (PBI->getCondition() == BI->getCondition() &&
3410 PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3411 // Okay, the outcome of this conditional branch is statically
3412 // knowable. If this block had a single pred, handle specially.
3413 if (BB->getSinglePredecessor()) {
3414 // Turn this into a branch on constant.
3415 bool CondIsTrue = PBI->getSuccessor(0) == BB;
3416 BI->setCondition(
3417 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue));
3418 return true; // Nuke the branch on constant.
3419 }
3420
3421 // Otherwise, if there are multiple predecessors, insert a PHI that merges
3422 // in the constant and simplify the block result. Subsequent passes of
3423 // simplifycfg will thread the block.
3424 if (BlockIsSimpleEnoughToThreadThrough(BB)) {
3425 pred_iterator PB = pred_begin(BB), PE = pred_end(BB);
3426 PHINode *NewPN = PHINode::Create(
3427 Type::getInt1Ty(BB->getContext()), std::distance(PB, PE),
3428 BI->getCondition()->getName() + ".pr", &BB->front());
3429 // Okay, we're going to insert the PHI node. Since PBI is not the only
3430 // predecessor, compute the PHI'd conditional value for all of the preds.
3431 // Any predecessor where the condition is not computable we keep symbolic.
3432 for (pred_iterator PI = PB; PI != PE; ++PI) {
3433 BasicBlock *P = *PI;
3434 if ((PBI = dyn_cast<BranchInst>(P->getTerminator())) && PBI != BI &&
3435 PBI->isConditional() && PBI->getCondition() == BI->getCondition() &&
3436 PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3437 bool CondIsTrue = PBI->getSuccessor(0) == BB;
3438 NewPN->addIncoming(
3439 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue),
3440 P);
3441 } else {
3442 NewPN->addIncoming(BI->getCondition(), P);
3443 }
3444 }
3445
3446 BI->setCondition(NewPN);
3447 return true;
3448 }
3449 }
3450
3451 // If the previous block ended with a widenable branch, determine if reusing
3452 // the target block is profitable and legal. This will have the effect of
3453 // "widening" PBI, but doesn't require us to reason about hosting safety.
3454 if (tryWidenCondBranchToCondBranch(PBI, BI))
3455 return true;
3456
3457 if (auto *CE = dyn_cast<ConstantExpr>(BI->getCondition()))
3458 if (CE->canTrap())
3459 return false;
3460
3461 // If both branches are conditional and both contain stores to the same
3462 // address, remove the stores from the conditionals and create a conditional
3463 // merged store at the end.
3464 if (MergeCondStores && mergeConditionalStores(PBI, BI, DL, TTI))
3465 return true;
3466
3467 // If this is a conditional branch in an empty block, and if any
3468 // predecessors are a conditional branch to one of our destinations,
3469 // fold the conditions into logical ops and one cond br.
3470
3471 // Ignore dbg intrinsics.
3472 if (&*BB->instructionsWithoutDebug().begin() != BI)
3473 return false;
3474
3475 int PBIOp, BIOp;
3476 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
3477 PBIOp = 0;
3478 BIOp = 0;
3479 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
3480 PBIOp = 0;
3481 BIOp = 1;
3482 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
3483 PBIOp = 1;
3484 BIOp = 0;
3485 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
3486 PBIOp = 1;
3487 BIOp = 1;
3488 } else {
3489 return false;
3490 }
3491
3492 // Check to make sure that the other destination of this branch
3493 // isn't BB itself. If so, this is an infinite loop that will
3494 // keep getting unwound.
3495 if (PBI->getSuccessor(PBIOp) == BB)
3496 return false;
3497
3498 // Do not perform this transformation if it would require
3499 // insertion of a large number of select instructions. For targets
3500 // without predication/cmovs, this is a big pessimization.
3501
3502 // Also do not perform this transformation if any phi node in the common
3503 // destination block can trap when reached by BB or PBB (PR17073). In that
3504 // case, it would be unsafe to hoist the operation into a select instruction.
3505
3506 BasicBlock *CommonDest = PBI->getSuccessor(PBIOp);
3507 unsigned NumPhis = 0;
3508 for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II);
3509 ++II, ++NumPhis) {
3510 if (NumPhis > 2) // Disable this xform.
3511 return false;
3512
3513 PHINode *PN = cast<PHINode>(II);
3514 Value *BIV = PN->getIncomingValueForBlock(BB);
3515 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(BIV))
3516 if (CE->canTrap())
3517 return false;
3518
3519 unsigned PBBIdx = PN->getBasicBlockIndex(PBI->getParent());
3520 Value *PBIV = PN->getIncomingValue(PBBIdx);
3521 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(PBIV))
3522 if (CE->canTrap())
3523 return false;
3524 }
3525
3526 // Finally, if everything is ok, fold the branches to logical ops.
3527 BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1);
3528
3529 LLVM_DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING BRs:" << *PBI
->getParent() << "AND: " << *BI->getParent(
); } } while (false)
3530 << "AND: " << *BI->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING BRs:" << *PBI
->getParent() << "AND: " << *BI->getParent(
); } } while (false)
;
3531
3532 // If OtherDest *is* BB, then BB is a basic block with a single conditional
3533 // branch in it, where one edge (OtherDest) goes back to itself but the other
3534 // exits. We don't *know* that the program avoids the infinite loop
3535 // (even though that seems likely). If we do this xform naively, we'll end up
3536 // recursively unpeeling the loop. Since we know that (after the xform is
3537 // done) that the block *is* infinite if reached, we just make it an obviously
3538 // infinite loop with no cond branch.
3539 if (OtherDest == BB) {
3540 // Insert it at the end of the function, because it's either code,
3541 // or it won't matter if it's hot. :)
3542 BasicBlock *InfLoopBlock =
3543 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
3544 BranchInst::Create(InfLoopBlock, InfLoopBlock);
3545 OtherDest = InfLoopBlock;
3546 }
3547
3548 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << *PBI->getParent()->getParent
(); } } while (false)
;
3549
3550 // BI may have other predecessors. Because of this, we leave
3551 // it alone, but modify PBI.
3552
3553 // Make sure we get to CommonDest on True&True directions.
3554 Value *PBICond = PBI->getCondition();
3555 IRBuilder<NoFolder> Builder(PBI);
3556 if (PBIOp)
3557 PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not");
3558
3559 Value *BICond = BI->getCondition();
3560 if (BIOp)
3561 BICond = Builder.CreateNot(BICond, BICond->getName() + ".not");
3562
3563 // Merge the conditions.
3564 Value *Cond = Builder.CreateOr(PBICond, BICond, "brmerge");
3565
3566 // Modify PBI to branch on the new condition to the new dests.
3567 PBI->setCondition(Cond);
3568 PBI->setSuccessor(0, CommonDest);
3569 PBI->setSuccessor(1, OtherDest);
3570
3571 // Update branch weight for PBI.
3572 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
3573 uint64_t PredCommon, PredOther, SuccCommon, SuccOther;
3574 bool HasWeights =
3575 extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
3576 SuccTrueWeight, SuccFalseWeight);
3577 if (HasWeights) {
3578 PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3579 PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3580 SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3581 SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3582 // The weight to CommonDest should be PredCommon * SuccTotal +
3583 // PredOther * SuccCommon.
3584 // The weight to OtherDest should be PredOther * SuccOther.
3585 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) +
3586 PredOther * SuccCommon,
3587 PredOther * SuccOther};
3588 // Halve the weights if any of them cannot fit in an uint32_t
3589 FitWeights(NewWeights);
3590
3591 setBranchWeights(PBI, NewWeights[0], NewWeights[1]);
3592 }
3593
3594 // OtherDest may have phi nodes. If so, add an entry from PBI's
3595 // block that are identical to the entries for BI's block.
3596 AddPredecessorToBlock(OtherDest, PBI->getParent(), BB);
3597
3598 // We know that the CommonDest already had an edge from PBI to
3599 // it. If it has PHIs though, the PHIs may have different
3600 // entries for BB and PBI's BB. If so, insert a select to make
3601 // them agree.
3602 for (PHINode &PN : CommonDest->phis()) {
3603 Value *BIV = PN.getIncomingValueForBlock(BB);
3604 unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent());
3605 Value *PBIV = PN.getIncomingValue(PBBIdx);
3606 if (BIV != PBIV) {
3607 // Insert a select in PBI to pick the right value.
3608 SelectInst *NV = cast<SelectInst>(
3609 Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux"));
3610 PN.setIncomingValue(PBBIdx, NV);
3611 // Although the select has the same condition as PBI, the original branch
3612 // weights for PBI do not apply to the new select because the select's
3613 // 'logical' edges are incoming edges of the phi that is eliminated, not
3614 // the outgoing edges of PBI.
3615 if (HasWeights) {
3616 uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3617 uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3618 uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3619 uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3620 // The weight to PredCommonDest should be PredCommon * SuccTotal.
3621 // The weight to PredOtherDest should be PredOther * SuccCommon.
3622 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther),
3623 PredOther * SuccCommon};
3624
3625 FitWeights(NewWeights);
3626
3627 setBranchWeights(NV, NewWeights[0], NewWeights[1]);
3628 }
3629 }
3630 }
3631
3632 LLVM_DEBUG(dbgs() << "INTO: " << *PBI->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "INTO: " << *PBI->
getParent(); } } while (false)
;
3633 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << *PBI->getParent()->getParent
(); } } while (false)
;
3634
3635 // This basic block is probably dead. We know it has at least
3636 // one fewer predecessor.
3637 return true;
3638}
3639
3640// Simplifies a terminator by replacing it with a branch to TrueBB if Cond is
3641// true or to FalseBB if Cond is false.
3642// Takes care of updating the successors and removing the old terminator.
3643// Also makes sure not to introduce new successors by assuming that edges to
3644// non-successor TrueBBs and FalseBBs aren't reachable.
3645bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm,
3646 Value *Cond, BasicBlock *TrueBB,
3647 BasicBlock *FalseBB,
3648 uint32_t TrueWeight,
3649 uint32_t FalseWeight) {
3650 // Remove any superfluous successor edges from the CFG.
3651 // First, figure out which successors to preserve.
3652 // If TrueBB and FalseBB are equal, only try to preserve one copy of that
3653 // successor.
3654 BasicBlock *KeepEdge1 = TrueBB;
3655 BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr;
3656
3657 // Then remove the rest.
3658 for (BasicBlock *Succ : successors(OldTerm)) {
3659 // Make sure only to keep exactly one copy of each edge.
3660 if (Succ == KeepEdge1)
3661 KeepEdge1 = nullptr;
3662 else if (Succ == KeepEdge2)
3663 KeepEdge2 = nullptr;
3664 else
3665 Succ->removePredecessor(OldTerm->getParent(),
3666 /*KeepOneInputPHIs=*/true);
3667 }
3668
3669 IRBuilder<> Builder(OldTerm);
3670 Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc());
3671
3672 // Insert an appropriate new terminator.
3673 if (!KeepEdge1 && !KeepEdge2) {
3674 if (TrueBB == FalseBB)
3675 // We were only looking for one successor, and it was present.
3676 // Create an unconditional branch to it.
3677 Builder.CreateBr(TrueBB);
3678 else {
3679 // We found both of the successors we were looking for.
3680 // Create a conditional branch sharing the condition of the select.
3681 BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
3682 if (TrueWeight != FalseWeight)
3683 setBranchWeights(NewBI, TrueWeight, FalseWeight);
3684 }
3685 } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
3686 // Neither of the selected blocks were successors, so this
3687 // terminator must be unreachable.
3688 new UnreachableInst(OldTerm->getContext(), OldTerm);
3689 } else {
3690 // One of the selected values was a successor, but the other wasn't.
3691 // Insert an unconditional branch to the one that was found;
3692 // the edge to the one that wasn't must be unreachable.
3693 if (!KeepEdge1)
3694 // Only TrueBB was found.
3695 Builder.CreateBr(TrueBB);
3696 else
3697 // Only FalseBB was found.
3698 Builder.CreateBr(FalseBB);
3699 }
3700
3701 EraseTerminatorAndDCECond(OldTerm);
3702 return true;
3703}
3704
3705// Replaces
3706// (switch (select cond, X, Y)) on constant X, Y
3707// with a branch - conditional if X and Y lead to distinct BBs,
3708// unconditional otherwise.
3709bool SimplifyCFGOpt::SimplifySwitchOnSelect(SwitchInst *SI,
3710 SelectInst *Select) {
3711 // Check for constant integer values in the select.
3712 ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue());
3713 ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue());
3714 if (!TrueVal || !FalseVal)
3715 return false;
3716
3717 // Find the relevant condition and destinations.
3718 Value *Condition = Select->getCondition();
3719 BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor();
3720 BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor();
3721
3722 // Get weight for TrueBB and FalseBB.
3723 uint32_t TrueWeight = 0, FalseWeight = 0;
3724 SmallVector<uint64_t, 8> Weights;
3725 bool HasWeights = HasBranchWeights(SI);
3726 if (HasWeights) {
3727 GetBranchWeights(SI, Weights);
3728 if (Weights.size() == 1 + SI->getNumCases()) {
3729 TrueWeight =
3730 (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()];
3731 FalseWeight =
3732 (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()];
3733 }
3734 }
3735
3736 // Perform the actual simplification.
3737 return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight,
3738 FalseWeight);
3739}
3740
3741// Replaces
3742// (indirectbr (select cond, blockaddress(@fn, BlockA),
3743// blockaddress(@fn, BlockB)))
3744// with
3745// (br cond, BlockA, BlockB).
3746bool SimplifyCFGOpt::SimplifyIndirectBrOnSelect(IndirectBrInst *IBI,
3747 SelectInst *SI) {
3748 // Check that both operands of the select are block addresses.
3749 BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue());
3750 BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue());
3751 if (!TBA || !FBA)
3752 return false;
3753
3754 // Extract the actual blocks.
3755 BasicBlock *TrueBB = TBA->getBasicBlock();
3756 BasicBlock *FalseBB = FBA->getBasicBlock();
3757
3758 // Perform the actual simplification.
3759 return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0,
3760 0);
3761}
3762
3763/// This is called when we find an icmp instruction
3764/// (a seteq/setne with a constant) as the only instruction in a
3765/// block that ends with an uncond branch. We are looking for a very specific
3766/// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In
3767/// this case, we merge the first two "or's of icmp" into a switch, but then the
3768/// default value goes to an uncond block with a seteq in it, we get something
3769/// like:
3770///
3771/// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ]
3772/// DEFAULT:
3773/// %tmp = icmp eq i8 %A, 92
3774/// br label %end
3775/// end:
3776/// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ]
3777///
3778/// We prefer to split the edge to 'end' so that there is a true/false entry to
3779/// the PHI, merging the third icmp into the switch.
3780bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt(
3781 ICmpInst *ICI, IRBuilder<> &Builder) {
3782 BasicBlock *BB = ICI->getParent();
3783
3784 // If the block has any PHIs in it or the icmp has multiple uses, it is too
3785 // complex.
3786 if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse())
3787 return false;
3788
3789 Value *V = ICI->getOperand(0);
3790 ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1));
3791
3792 // The pattern we're looking for is where our only predecessor is a switch on
3793 // 'V' and this block is the default case for the switch. In this case we can
3794 // fold the compared value into the switch to simplify things.
3795 BasicBlock *Pred = BB->getSinglePredecessor();
3796 if (!Pred || !isa<SwitchInst>(Pred->getTerminator()))
3797 return false;
3798
3799 SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator());
3800 if (SI->getCondition() != V)
3801 return false;
3802
3803 // If BB is reachable on a non-default case, then we simply know the value of
3804 // V in this block. Substitute it and constant fold the icmp instruction
3805 // away.
3806 if (SI->getDefaultDest() != BB) {
3807 ConstantInt *VVal = SI->findCaseDest(BB);
3808 assert(VVal && "Should have a unique destination value")((VVal && "Should have a unique destination value") ?
static_cast<void> (0) : __assert_fail ("VVal && \"Should have a unique destination value\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3808, __PRETTY_FUNCTION__))
;
3809 ICI->setOperand(0, VVal);
3810
3811 if (Value *V = SimplifyInstruction(ICI, {DL, ICI})) {
3812 ICI->replaceAllUsesWith(V);
3813 ICI->eraseFromParent();
3814 }
3815 // BB is now empty, so it is likely to simplify away.
3816 return requestResimplify();
3817 }
3818
3819 // Ok, the block is reachable from the default dest. If the constant we're
3820 // comparing exists in one of the other edges, then we can constant fold ICI
3821 // and zap it.
3822 if (SI->findCaseValue(Cst) != SI->case_default()) {
3823 Value *V;
3824 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
3825 V = ConstantInt::getFalse(BB->getContext());
3826 else
3827 V = ConstantInt::getTrue(BB->getContext());
3828
3829 ICI->replaceAllUsesWith(V);
3830 ICI->eraseFromParent();
3831 // BB is now empty, so it is likely to simplify away.
3832 return requestResimplify();
3833 }
3834
3835 // The use of the icmp has to be in the 'end' block, by the only PHI node in
3836 // the block.
3837 BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0);
3838 PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back());
3839 if (PHIUse == nullptr || PHIUse != &SuccBlock->front() ||
3840 isa<PHINode>(++BasicBlock::iterator(PHIUse)))
3841 return false;
3842
3843 // If the icmp is a SETEQ, then the default dest gets false, the new edge gets
3844 // true in the PHI.
3845 Constant *DefaultCst = ConstantInt::getTrue(BB->getContext());
3846 Constant *NewCst = ConstantInt::getFalse(BB->getContext());
3847
3848 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
3849 std::swap(DefaultCst, NewCst);
3850
3851 // Replace ICI (which is used by the PHI for the default value) with true or
3852 // false depending on if it is EQ or NE.
3853 ICI->replaceAllUsesWith(DefaultCst);
3854 ICI->eraseFromParent();
3855
3856 // Okay, the switch goes to this block on a default value. Add an edge from
3857 // the switch to the merge point on the compared value.
3858 BasicBlock *NewBB =
3859 BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB);
3860 {
3861 SwitchInstProfUpdateWrapper SIW(*SI);
3862 auto W0 = SIW.getSuccessorWeight(0);
3863 SwitchInstProfUpdateWrapper::CaseWeightOpt NewW;
3864 if (W0) {
3865 NewW = ((uint64_t(*W0) + 1) >> 1);
3866 SIW.setSuccessorWeight(0, *NewW);
3867 }
3868 SIW.addCase(Cst, NewBB, NewW);
3869 }
3870
3871 // NewBB branches to the phi block, add the uncond branch and the phi entry.
3872 Builder.SetInsertPoint(NewBB);
3873 Builder.SetCurrentDebugLocation(SI->getDebugLoc());
3874 Builder.CreateBr(SuccBlock);
3875 PHIUse->addIncoming(NewCst, NewBB);
3876 return true;
3877}
3878
3879/// The specified branch is a conditional branch.
3880/// Check to see if it is branching on an or/and chain of icmp instructions, and
3881/// fold it into a switch instruction if so.
3882bool SimplifyCFGOpt::SimplifyBranchOnICmpChain(BranchInst *BI,
3883 IRBuilder<> &Builder,
3884 const DataLayout &DL) {
3885 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
3886 if (!Cond)
3887 return false;
3888
3889 // Change br (X == 0 | X == 1), T, F into a switch instruction.
3890 // If this is a bunch of seteq's or'd together, or if it's a bunch of
3891 // 'setne's and'ed together, collect them.
3892
3893 // Try to gather values from a chain of and/or to be turned into a switch
3894 ConstantComparesGatherer ConstantCompare(Cond, DL);
3895 // Unpack the result
3896 SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals;
3897 Value *CompVal = ConstantCompare.CompValue;
3898 unsigned UsedICmps = ConstantCompare.UsedICmps;
3899 Value *ExtraCase = ConstantCompare.Extra;
3900
3901 // If we didn't have a multiply compared value, fail.
3902 if (!CompVal)
3903 return false;
3904
3905 // Avoid turning single icmps into a switch.
3906 if (UsedICmps <= 1)
3907 return false;
3908
3909 bool TrueWhenEqual = (Cond->getOpcode() == Instruction::Or);
3910
3911 // There might be duplicate constants in the list, which the switch
3912 // instruction can't handle, remove them now.
3913 array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate);
3914 Values.erase(std::unique(Values.begin(), Values.end()), Values.end());
3915
3916 // If Extra was used, we require at least two switch values to do the
3917 // transformation. A switch with one value is just a conditional branch.
3918 if (ExtraCase && Values.size() < 2)
3919 return false;
3920
3921 // TODO: Preserve branch weight metadata, similarly to how
3922 // FoldValueComparisonIntoPredecessors preserves it.
3923
3924 // Figure out which block is which destination.
3925 BasicBlock *DefaultBB = BI->getSuccessor(1);
3926 BasicBlock *EdgeBB = BI->getSuccessor(0);
3927 if (!TrueWhenEqual)
3928 std::swap(DefaultBB, EdgeBB);
3929
3930 BasicBlock *BB = BI->getParent();
3931
3932 // MSAN does not like undefs as branch condition which can be introduced
3933 // with "explicit branch".
3934 if (ExtraCase && BB->getParent()->hasFnAttribute(Attribute::SanitizeMemory))
3935 return false;
3936
3937 LLVM_DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Converting 'icmp' chain with "
<< Values.size() << " cases into SWITCH. BB is:\n"
<< *BB; } } while (false)
3938 << " cases into SWITCH. BB is:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Converting 'icmp' chain with "
<< Values.size() << " cases into SWITCH. BB is:\n"
<< *BB; } } while (false)
3939 << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Converting 'icmp' chain with "
<< Values.size() << " cases into SWITCH. BB is:\n"
<< *BB; } } while (false)
;
3940
3941 // If there are any extra values that couldn't be folded into the switch
3942 // then we evaluate them with an explicit branch first. Split the block
3943 // right before the condbr to handle it.
3944 if (ExtraCase) {
3945 BasicBlock *NewBB =
3946 BB->splitBasicBlock(BI->getIterator(), "switch.early.test");
3947 // Remove the uncond branch added to the old block.
3948 Instruction *OldTI = BB->getTerminator();
3949 Builder.SetInsertPoint(OldTI);
3950
3951 if (TrueWhenEqual)
3952 Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB);
3953 else
3954 Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB);
3955
3956 OldTI->eraseFromParent();
3957
3958 // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
3959 // for the edge we just added.
3960 AddPredecessorToBlock(EdgeBB, BB, NewBB);
3961
3962 LLVM_DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCasedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << " ** 'icmp' chain unhandled condition: "
<< *ExtraCase << "\nEXTRABB = " << *BB; } }
while (false)
3963 << "\nEXTRABB = " << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << " ** 'icmp' chain unhandled condition: "
<< *ExtraCase << "\nEXTRABB = " << *BB; } }
while (false)
;
3964 BB = NewBB;
3965 }
3966
3967 Builder.SetInsertPoint(BI);
3968 // Convert pointer to int before we switch.
3969 if (CompVal->getType()->isPointerTy()) {
3970 CompVal = Builder.CreatePtrToInt(
3971 CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr");
3972 }
3973
3974 // Create the new switch instruction now.
3975 SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size());
3976
3977 // Add all of the 'cases' to the switch instruction.
3978 for (unsigned i = 0, e = Values.size(); i != e; ++i)
3979 New->addCase(Values[i], EdgeBB);
3980
3981 // We added edges from PI to the EdgeBB. As such, if there were any
3982 // PHI nodes in EdgeBB, they need entries to be added corresponding to
3983 // the number of edges added.
3984 for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) {
3985 PHINode *PN = cast<PHINode>(BBI);
3986 Value *InVal = PN->getIncomingValueForBlock(BB);
3987 for (unsigned i = 0, e = Values.size() - 1; i != e; ++i)
3988 PN->addIncoming(InVal, BB);
3989 }
3990
3991 // Erase the old branch instruction.
3992 EraseTerminatorAndDCECond(BI);
3993
3994 LLVM_DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << " ** 'icmp' chain result is:\n"
<< *BB << '\n'; } } while (false)
;
3995 return true;
3996}
3997
3998bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
3999 if (isa<PHINode>(RI->getValue()))
18
Assuming the object is a 'PHINode'
19
Taking true branch
4000 return simplifyCommonResume(RI);
20
Calling 'SimplifyCFGOpt::simplifyCommonResume'
4001 else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) &&
4002 RI->getValue() == RI->getParent()->getFirstNonPHI())
4003 // The resume must unwind the exception that caused control to branch here.
4004 return simplifySingleResume(RI);
4005
4006 return false;
4007}
4008
4009// Check if cleanup block is empty
4010static bool isCleanupBlockEmpty(iterator_range<BasicBlock::iterator> R) {
4011 for (Instruction &I : R) {
4012 auto *II = dyn_cast<IntrinsicInst>(&I);
4013 if (!II)
4014 return false;
4015
4016 Intrinsic::ID IntrinsicID = II->getIntrinsicID();
4017 switch (IntrinsicID) {
4018 case Intrinsic::dbg_declare:
4019 case Intrinsic::dbg_value:
4020 case Intrinsic::dbg_label:
4021 case Intrinsic::lifetime_end:
4022 break;
4023 default:
4024 return false;
4025 }
4026 }
4027 return true;
22
Returning the value 1, which participates in a condition later
4028}
4029
4030// Simplify resume that is shared by several landing pads (phi of landing pad).
4031bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
4032 BasicBlock *BB = RI->getParent();
4033
4034 // Check that there are no other instructions except for debug and lifetime
4035 // intrinsics between the phi's and resume instruction.
4036 if (!isCleanupBlockEmpty(
21
Calling 'isCleanupBlockEmpty'
23
Returning from 'isCleanupBlockEmpty'
24
Taking false branch
4037 make_range(RI->getParent()->getFirstNonPHI(), BB->getTerminator())))
4038 return false;
4039
4040 SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks;
4041 auto *PhiLPInst = cast<PHINode>(RI->getValue());
25
The object is a 'PHINode'
4042
4043 // Check incoming blocks to see if any of them are trivial.
4044 for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End;
26
Assuming 'Idx' is not equal to 'End'
27
Loop condition is true. Entering loop body
4045 Idx++) {
4046 auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx);
4047 auto *IncomingValue = PhiLPInst->getIncomingValue(Idx);
4048
4049 // If the block has other successors, we can not delete it because
4050 // it has other dependents.
4051 if (IncomingBB->getUniqueSuccessor() != BB)
28
Assuming the condition is false
29
Taking false branch
4052 continue;
4053
4054 auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI());
30
Assuming the object is not a 'LandingPadInst'
31
'LandingPad' initialized to a null pointer value
4055 // Not the landing pad that caused the control to branch here.
4056 if (IncomingValue
31.1
'IncomingValue' is equal to 'LandingPad'
!= LandingPad)
32
Taking false branch
4057 continue;
4058
4059 if (isCleanupBlockEmpty(
4060 make_range(LandingPad->getNextNode(), IncomingBB->getTerminator())))
33
Called C++ object pointer is null
4061 TrivialUnwindBlocks.insert(IncomingBB);
4062 }
4063
4064 // If no trivial unwind blocks, don't do any simplifications.
4065 if (TrivialUnwindBlocks.empty())
4066 return false;
4067
4068 // Turn all invokes that unwind here into calls.
4069 for (auto *TrivialBB : TrivialUnwindBlocks) {
4070 // Blocks that will be simplified should be removed from the phi node.
4071 // Note there could be multiple edges to the resume block, and we need
4072 // to remove them all.
4073 while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1)
4074 BB->removePredecessor(TrivialBB, true);
4075
4076 for (pred_iterator PI = pred_begin(TrivialBB), PE = pred_end(TrivialBB);
4077 PI != PE;) {
4078 BasicBlock *Pred = *PI++;
4079 removeUnwindEdge(Pred);
4080 ++NumInvokes;
4081 }
4082
4083 // In each SimplifyCFG run, only the current processed block can be erased.
4084 // Otherwise, it will break the iteration of SimplifyCFG pass. So instead
4085 // of erasing TrivialBB, we only remove the branch to the common resume
4086 // block so that we can later erase the resume block since it has no
4087 // predecessors.
4088 TrivialBB->getTerminator()->eraseFromParent();
4089 new UnreachableInst(RI->getContext(), TrivialBB);
4090 }
4091
4092 // Delete the resume block if all its predecessors have been removed.
4093 if (pred_empty(BB))
4094 BB->eraseFromParent();
4095
4096 return !TrivialUnwindBlocks.empty();
4097}
4098
4099// Simplify resume that is only used by a single (non-phi) landing pad.
4100bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) {
4101 BasicBlock *BB = RI->getParent();
4102 auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI());
4103 assert(RI->getValue() == LPInst &&((RI->getValue() == LPInst && "Resume must unwind the exception that caused control to here"
) ? static_cast<void> (0) : __assert_fail ("RI->getValue() == LPInst && \"Resume must unwind the exception that caused control to here\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4104, __PRETTY_FUNCTION__))
4104 "Resume must unwind the exception that caused control to here")((RI->getValue() == LPInst && "Resume must unwind the exception that caused control to here"
) ? static_cast<void> (0) : __assert_fail ("RI->getValue() == LPInst && \"Resume must unwind the exception that caused control to here\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4104, __PRETTY_FUNCTION__))
;
4105
4106 // Check that there are no other instructions except for debug intrinsics.
4107 if (!isCleanupBlockEmpty(
4108 make_range<Instruction *>(LPInst->getNextNode(), RI)))
4109 return false;
4110
4111 // Turn all invokes that unwind here into calls and delete the basic block.
4112 for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE;) {
4113 BasicBlock *Pred = *PI++;
4114 removeUnwindEdge(Pred);
4115 ++NumInvokes;
4116 }
4117
4118 // The landingpad is now unreachable. Zap it.
4119 if (LoopHeaders)
4120 LoopHeaders->erase(BB);
4121 BB->eraseFromParent();
4122 return true;
4123}
4124
4125static bool removeEmptyCleanup(CleanupReturnInst *RI) {
4126 // If this is a trivial cleanup pad that executes no instructions, it can be
4127 // eliminated. If the cleanup pad continues to the caller, any predecessor
4128 // that is an EH pad will be updated to continue to the caller and any
4129 // predecessor that terminates with an invoke instruction will have its invoke
4130 // instruction converted to a call instruction. If the cleanup pad being
4131 // simplified does not continue to the caller, each predecessor will be
4132 // updated to continue to the unwind destination of the cleanup pad being
4133 // simplified.
4134 BasicBlock *BB = RI->getParent();
4135 CleanupPadInst *CPInst = RI->getCleanupPad();
4136 if (CPInst->getParent() != BB)
4137 // This isn't an empty cleanup.
4138 return false;
4139
4140 // We cannot kill the pad if it has multiple uses. This typically arises
4141 // from unreachable basic blocks.
4142 if (!CPInst->hasOneUse())
4143 return false;
4144
4145 // Check that there are no other instructions except for benign intrinsics.
4146 if (!isCleanupBlockEmpty(
4147 make_range<Instruction *>(CPInst->getNextNode(), RI)))
4148 return false;
4149
4150 // If the cleanup return we are simplifying unwinds to the caller, this will
4151 // set UnwindDest to nullptr.
4152 BasicBlock *UnwindDest = RI->getUnwindDest();
4153 Instruction *DestEHPad = UnwindDest ? UnwindDest->getFirstNonPHI() : nullptr;
4154
4155 // We're about to remove BB from the control flow. Before we do, sink any
4156 // PHINodes into the unwind destination. Doing this before changing the
4157 // control flow avoids some potentially slow checks, since we can currently
4158 // be certain that UnwindDest and BB have no common predecessors (since they
4159 // are both EH pads).
4160 if (UnwindDest) {
4161 // First, go through the PHI nodes in UnwindDest and update any nodes that
4162 // reference the block we are removing
4163 for (BasicBlock::iterator I = UnwindDest->begin(),
4164 IE = DestEHPad->getIterator();
4165 I != IE; ++I) {
4166 PHINode *DestPN = cast<PHINode>(I);
4167
4168 int Idx = DestPN->getBasicBlockIndex(BB);
4169 // Since BB unwinds to UnwindDest, it has to be in the PHI node.
4170 assert(Idx != -1)((Idx != -1) ? static_cast<void> (0) : __assert_fail ("Idx != -1"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4170, __PRETTY_FUNCTION__))
;
4171 // This PHI node has an incoming value that corresponds to a control
4172 // path through the cleanup pad we are removing. If the incoming
4173 // value is in the cleanup pad, it must be a PHINode (because we
4174 // verified above that the block is otherwise empty). Otherwise, the
4175 // value is either a constant or a value that dominates the cleanup
4176 // pad being removed.
4177 //
4178 // Because BB and UnwindDest are both EH pads, all of their
4179 // predecessors must unwind to these blocks, and since no instruction
4180 // can have multiple unwind destinations, there will be no overlap in
4181 // incoming blocks between SrcPN and DestPN.
4182 Value *SrcVal = DestPN->getIncomingValue(Idx);
4183 PHINode *SrcPN = dyn_cast<PHINode>(SrcVal);
4184
4185 // Remove the entry for the block we are deleting.
4186 DestPN->removeIncomingValue(Idx, false);
4187
4188 if (SrcPN && SrcPN->getParent() == BB) {
4189 // If the incoming value was a PHI node in the cleanup pad we are
4190 // removing, we need to merge that PHI node's incoming values into
4191 // DestPN.
4192 for (unsigned SrcIdx = 0, SrcE = SrcPN->getNumIncomingValues();
4193 SrcIdx != SrcE; ++SrcIdx) {
4194 DestPN->addIncoming(SrcPN->getIncomingValue(SrcIdx),
4195 SrcPN->getIncomingBlock(SrcIdx));
4196 }
4197 } else {
4198 // Otherwise, the incoming value came from above BB and
4199 // so we can just reuse it. We must associate all of BB's
4200 // predecessors with this value.
4201 for (auto *pred : predecessors(BB)) {
4202 DestPN->addIncoming(SrcVal, pred);
4203 }
4204 }
4205 }
4206
4207 // Sink any remaining PHI nodes directly into UnwindDest.
4208 Instruction *InsertPt = DestEHPad;
4209 for (BasicBlock::iterator I = BB->begin(),
4210 IE = BB->getFirstNonPHI()->getIterator();
4211 I != IE;) {
4212 // The iterator must be incremented here because the instructions are
4213 // being moved to another block.
4214 PHINode *PN = cast<PHINode>(I++);
4215 if (PN->use_empty() || !PN->isUsedOutsideOfBlock(BB))
4216 // If the PHI node has no uses or all of its uses are in this basic
4217 // block (meaning they are debug or lifetime intrinsics), just leave
4218 // it. It will be erased when we erase BB below.
4219 continue;
4220
4221 // Otherwise, sink this PHI node into UnwindDest.
4222 // Any predecessors to UnwindDest which are not already represented
4223 // must be back edges which inherit the value from the path through
4224 // BB. In this case, the PHI value must reference itself.
4225 for (auto *pred : predecessors(UnwindDest))
4226 if (pred != BB)
4227 PN->addIncoming(PN, pred);
4228 PN->moveBefore(InsertPt);
4229 }
4230 }
4231
4232 for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE;) {
4233 // The iterator must be updated here because we are removing this pred.
4234 BasicBlock *PredBB = *PI++;
4235 if (UnwindDest == nullptr) {
4236 removeUnwindEdge(PredBB);
4237 ++NumInvokes;
4238 } else {
4239 Instruction *TI = PredBB->getTerminator();
4240 TI->replaceUsesOfWith(BB, UnwindDest);
4241 }
4242 }
4243
4244 // The cleanup pad is now unreachable. Zap it.
4245 BB->eraseFromParent();
4246 return true;
4247}
4248
4249// Try to merge two cleanuppads together.
4250static bool mergeCleanupPad(CleanupReturnInst *RI) {
4251 // Skip any cleanuprets which unwind to caller, there is nothing to merge
4252 // with.
4253 BasicBlock *UnwindDest = RI->getUnwindDest();
4254 if (!UnwindDest)
4255 return false;
4256
4257 // This cleanupret isn't the only predecessor of this cleanuppad, it wouldn't
4258 // be safe to merge without code duplication.
4259 if (UnwindDest->getSinglePredecessor() != RI->getParent())
4260 return false;
4261
4262 // Verify that our cleanuppad's unwind destination is another cleanuppad.
4263 auto *SuccessorCleanupPad = dyn_cast<CleanupPadInst>(&UnwindDest->front());
4264 if (!SuccessorCleanupPad)
4265 return false;
4266
4267 CleanupPadInst *PredecessorCleanupPad = RI->getCleanupPad();
4268 // Replace any uses of the successor cleanupad with the predecessor pad
4269 // The only cleanuppad uses should be this cleanupret, it's cleanupret and
4270 // funclet bundle operands.
4271 SuccessorCleanupPad->replaceAllUsesWith(PredecessorCleanupPad);
4272 // Remove the old cleanuppad.
4273 SuccessorCleanupPad->eraseFromParent();
4274 // Now, we simply replace the cleanupret with a branch to the unwind
4275 // destination.
4276 BranchInst::Create(UnwindDest, RI->getParent());
4277 RI->eraseFromParent();
4278
4279 return true;
4280}
4281
4282bool SimplifyCFGOpt::simplifyCleanupReturn(CleanupReturnInst *RI) {
4283 // It is possible to transiantly have an undef cleanuppad operand because we
4284 // have deleted some, but not all, dead blocks.
4285 // Eventually, this block will be deleted.
4286 if (isa<UndefValue>(RI->getOperand(0)))
4287 return false;
4288
4289 if (mergeCleanupPad(RI))
4290 return true;
4291
4292 if (removeEmptyCleanup(RI))
4293 return true;
4294
4295 return false;
4296}
4297
4298bool SimplifyCFGOpt::simplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
4299 BasicBlock *BB = RI->getParent();
4300 if (!BB->getFirstNonPHIOrDbg()->isTerminator())
4301 return false;
4302
4303 // Find predecessors that end with branches.
4304 SmallVector<BasicBlock *, 8> UncondBranchPreds;
4305 SmallVector<BranchInst *, 8> CondBranchPreds;
4306 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
4307 BasicBlock *P = *PI;
4308 Instruction *PTI = P->getTerminator();
4309 if (BranchInst *BI = dyn_cast<BranchInst>(PTI)) {
4310 if (BI->isUnconditional())
4311 UncondBranchPreds.push_back(P);
4312 else
4313 CondBranchPreds.push_back(BI);
4314 }
4315 }
4316
4317 // If we found some, do the transformation!
4318 if (!UncondBranchPreds.empty() && DupRet) {
4319 while (!UncondBranchPreds.empty()) {
4320 BasicBlock *Pred = UncondBranchPreds.pop_back_val();
4321 LLVM_DEBUG(dbgs() << "FOLDING: " << *BBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING: " << *BB <<
"INTO UNCOND BRANCH PRED: " << *Pred; } } while (false
)
4322 << "INTO UNCOND BRANCH PRED: " << *Pred)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING: " << *BB <<
"INTO UNCOND BRANCH PRED: " << *Pred; } } while (false
)
;
4323 (void)FoldReturnIntoUncondBranch(RI, BB, Pred);
4324 }
4325
4326 // If we eliminated all predecessors of the block, delete the block now.
4327 if (pred_empty(BB)) {
4328 // We know there are no successors, so just nuke the block.
4329 if (LoopHeaders)
4330 LoopHeaders->erase(BB);
4331 BB->eraseFromParent();
4332 }
4333
4334 return true;
4335 }
4336
4337 // Check out all of the conditional branches going to this return
4338 // instruction. If any of them just select between returns, change the
4339 // branch itself into a select/return pair.
4340 while (!CondBranchPreds.empty()) {
4341 BranchInst *BI = CondBranchPreds.pop_back_val();
4342
4343 // Check to see if the non-BB successor is also a return block.
4344 if (isa<ReturnInst>(BI->getSuccessor(0)->getTerminator()) &&
4345 isa<ReturnInst>(BI->getSuccessor(1)->getTerminator()) &&
4346 SimplifyCondBranchToTwoReturns(BI, Builder))
4347 return true;
4348 }
4349 return false;
4350}
4351
4352bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
4353 BasicBlock *BB = UI->getParent();
4354
4355 bool Changed = false;
4356
4357 // If there are any instructions immediately before the unreachable that can
4358 // be removed, do so.
4359 while (UI->getIterator() != BB->begin()) {
4360 BasicBlock::iterator BBI = UI->getIterator();
4361 --BBI;
4362 // Do not delete instructions that can have side effects which might cause
4363 // the unreachable to not be reachable; specifically, calls and volatile
4364 // operations may have this effect.
4365 if (isa<CallInst>(BBI) && !isa<DbgInfoIntrinsic>(BBI))
4366 break;
4367
4368 if (BBI->mayHaveSideEffects()) {
4369 if (auto *SI = dyn_cast<StoreInst>(BBI)) {
4370 if (SI->isVolatile())
4371 break;
4372 } else if (auto *LI = dyn_cast<LoadInst>(BBI)) {
4373 if (LI->isVolatile())
4374 break;
4375 } else if (auto *RMWI = dyn_cast<AtomicRMWInst>(BBI)) {
4376 if (RMWI->isVolatile())
4377 break;
4378 } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(BBI)) {
4379 if (CXI->isVolatile())
4380 break;
4381 } else if (isa<CatchPadInst>(BBI)) {
4382 // A catchpad may invoke exception object constructors and such, which
4383 // in some languages can be arbitrary code, so be conservative by
4384 // default.
4385 // For CoreCLR, it just involves a type test, so can be removed.
4386 if (classifyEHPersonality(BB->getParent()->getPersonalityFn()) !=
4387 EHPersonality::CoreCLR)
4388 break;
4389 } else if (!isa<FenceInst>(BBI) && !isa<VAArgInst>(BBI) &&
4390 !isa<LandingPadInst>(BBI)) {
4391 break;
4392 }
4393 // Note that deleting LandingPad's here is in fact okay, although it
4394 // involves a bit of subtle reasoning. If this inst is a LandingPad,
4395 // all the predecessors of this block will be the unwind edges of Invokes,
4396 // and we can therefore guarantee this block will be erased.
4397 }
4398
4399 // Delete this instruction (any uses are guaranteed to be dead)
4400 if (!BBI->use_empty())
4401 BBI->replaceAllUsesWith(UndefValue::get(BBI->getType()));
4402 BBI->eraseFromParent();
4403 Changed = true;
4404 }
4405
4406 // If the unreachable instruction is the first in the block, take a gander
4407 // at all of the predecessors of this instruction, and simplify them.
4408 if (&BB->front() != UI)
4409 return Changed;
4410
4411 SmallVector<BasicBlock *, 8> Preds(pred_begin(BB), pred_end(BB));
4412 for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
4413 Instruction *TI = Preds[i]->getTerminator();
4414 IRBuilder<> Builder(TI);
4415 if (auto *BI = dyn_cast<BranchInst>(TI)) {
4416 if (BI->isUnconditional()) {
4417 assert(BI->getSuccessor(0) == BB && "Incorrect CFG")((BI->getSuccessor(0) == BB && "Incorrect CFG") ? static_cast
<void> (0) : __assert_fail ("BI->getSuccessor(0) == BB && \"Incorrect CFG\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4417, __PRETTY_FUNCTION__))
;
4418 new UnreachableInst(TI->getContext(), TI);
4419 TI->eraseFromParent();
4420 Changed = true;
4421 } else {
4422 Value* Cond = BI->getCondition();
4423 if (BI->getSuccessor(0) == BB) {
4424 Builder.CreateAssumption(Builder.CreateNot(Cond));
4425 Builder.CreateBr(BI->getSuccessor(1));
4426 } else {
4427 assert(BI->getSuccessor(1) == BB && "Incorrect CFG")((BI->getSuccessor(1) == BB && "Incorrect CFG") ? static_cast
<void> (0) : __assert_fail ("BI->getSuccessor(1) == BB && \"Incorrect CFG\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4427, __PRETTY_FUNCTION__))
;
4428 Builder.CreateAssumption(Cond);
4429 Builder.CreateBr(BI->getSuccessor(0));
4430 }
4431 EraseTerminatorAndDCECond(BI);
4432 Changed = true;
4433 }
4434 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
4435 SwitchInstProfUpdateWrapper SU(*SI);
4436 for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) {
4437 if (i->getCaseSuccessor() != BB) {
4438 ++i;
4439 continue;
4440 }
4441 BB->removePredecessor(SU->getParent());
4442 i = SU.removeCase(i);
4443 e = SU->case_end();
4444 Changed = true;
4445 }
4446 } else if (auto *II = dyn_cast<InvokeInst>(TI)) {
4447 if (II->getUnwindDest() == BB) {
4448 removeUnwindEdge(TI->getParent());
4449 Changed = true;
4450 }
4451 } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) {
4452 if (CSI->getUnwindDest() == BB) {
4453 removeUnwindEdge(TI->getParent());
4454 Changed = true;
4455 continue;
4456 }
4457
4458 for (CatchSwitchInst::handler_iterator I = CSI->handler_begin(),
4459 E = CSI->handler_end();
4460 I != E; ++I) {
4461 if (*I == BB) {
4462 CSI->removeHandler(I);
4463 --I;
4464 --E;
4465 Changed = true;
4466 }
4467 }
4468 if (CSI->getNumHandlers() == 0) {
4469 BasicBlock *CatchSwitchBB = CSI->getParent();
4470 if (CSI->hasUnwindDest()) {
4471 // Redirect preds to the unwind dest
4472 CatchSwitchBB->replaceAllUsesWith(CSI->getUnwindDest());
4473 } else {
4474 // Rewrite all preds to unwind to caller (or from invoke to call).
4475 SmallVector<BasicBlock *, 8> EHPreds(predecessors(CatchSwitchBB));
4476 for (BasicBlock *EHPred : EHPreds)
4477 removeUnwindEdge(EHPred);
4478 }
4479 // The catchswitch is no longer reachable.
4480 new UnreachableInst(CSI->getContext(), CSI);
4481 CSI->eraseFromParent();
4482 Changed = true;
4483 }
4484 } else if (isa<CleanupReturnInst>(TI)) {
4485 new UnreachableInst(TI->getContext(), TI);
4486 TI->eraseFromParent();
4487 Changed = true;
4488 }
4489 }
4490
4491 // If this block is now dead, remove it.
4492 if (pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) {
4493 // We know there are no successors, so just nuke the block.
4494 if (LoopHeaders)
4495 LoopHeaders->erase(BB);
4496 BB->eraseFromParent();
4497 return true;
4498 }
4499
4500 return Changed;
4501}
4502
4503static bool CasesAreContiguous(SmallVectorImpl<ConstantInt *> &Cases) {
4504 assert(Cases.size() >= 1)((Cases.size() >= 1) ? static_cast<void> (0) : __assert_fail
("Cases.size() >= 1", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4504, __PRETTY_FUNCTION__))
;
4505
4506 array_pod_sort(Cases.begin(), Cases.end(), ConstantIntSortPredicate);
4507 for (size_t I = 1, E = Cases.size(); I != E; ++I) {
4508 if (Cases[I - 1]->getValue() != Cases[I]->getValue() + 1)
4509 return false;
4510 }
4511 return true;
4512}
4513
4514static void createUnreachableSwitchDefault(SwitchInst *Switch) {
4515 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch default is dead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SimplifyCFG: switch default is dead.\n"
; } } while (false)
;
4516 BasicBlock *NewDefaultBlock =
4517 SplitBlockPredecessors(Switch->getDefaultDest(), Switch->getParent(), "");
4518 Switch->setDefaultDest(&*NewDefaultBlock);
4519 SplitBlock(&*NewDefaultBlock, &NewDefaultBlock->front());
4520 auto *NewTerminator = NewDefaultBlock->getTerminator();
4521 new UnreachableInst(Switch->getContext(), NewTerminator);
4522 EraseTerminatorAndDCECond(NewTerminator);
4523}
4524
4525/// Turn a switch with two reachable destinations into an integer range
4526/// comparison and branch.
4527bool SimplifyCFGOpt::TurnSwitchRangeIntoICmp(SwitchInst *SI,
4528 IRBuilder<> &Builder) {
4529 assert(SI->getNumCases() > 1 && "Degenerate switch?")((SI->getNumCases() > 1 && "Degenerate switch?"
) ? static_cast<void> (0) : __assert_fail ("SI->getNumCases() > 1 && \"Degenerate switch?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4529, __PRETTY_FUNCTION__))
;
4530
4531 bool HasDefault =
4532 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
4533
4534 // Partition the cases into two sets with different destinations.
4535 BasicBlock *DestA = HasDefault ? SI->getDefaultDest() : nullptr;
4536 BasicBlock *DestB = nullptr;
4537 SmallVector<ConstantInt *, 16> CasesA;
4538 SmallVector<ConstantInt *, 16> CasesB;
4539
4540 for (auto Case : SI->cases()) {
4541 BasicBlock *Dest = Case.getCaseSuccessor();
4542 if (!DestA)
4543 DestA = Dest;
4544 if (Dest == DestA) {
4545 CasesA.push_back(Case.getCaseValue());
4546 continue;
4547 }
4548 if (!DestB)
4549 DestB = Dest;
4550 if (Dest == DestB) {
4551 CasesB.push_back(Case.getCaseValue());
4552 continue;
4553 }
4554 return false; // More than two destinations.
4555 }
4556
4557 assert(DestA && DestB &&((DestA && DestB && "Single-destination switch should have been folded."
) ? static_cast<void> (0) : __assert_fail ("DestA && DestB && \"Single-destination switch should have been folded.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4558, __PRETTY_FUNCTION__))
4558 "Single-destination switch should have been folded.")((DestA && DestB && "Single-destination switch should have been folded."
) ? static_cast<void> (0) : __assert_fail ("DestA && DestB && \"Single-destination switch should have been folded.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4558, __PRETTY_FUNCTION__))
;
4559 assert(DestA != DestB)((DestA != DestB) ? static_cast<void> (0) : __assert_fail
("DestA != DestB", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4559, __PRETTY_FUNCTION__))
;
4560 assert(DestB != SI->getDefaultDest())((DestB != SI->getDefaultDest()) ? static_cast<void>
(0) : __assert_fail ("DestB != SI->getDefaultDest()", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4560, __PRETTY_FUNCTION__))
;
4561 assert(!CasesB.empty() && "There must be non-default cases.")((!CasesB.empty() && "There must be non-default cases."
) ? static_cast<void> (0) : __assert_fail ("!CasesB.empty() && \"There must be non-default cases.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4561, __PRETTY_FUNCTION__))
;
4562 assert(!CasesA.empty() || HasDefault)((!CasesA.empty() || HasDefault) ? static_cast<void> (0
) : __assert_fail ("!CasesA.empty() || HasDefault", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4562, __PRETTY_FUNCTION__))
;
4563
4564 // Figure out if one of the sets of cases form a contiguous range.
4565 SmallVectorImpl<ConstantInt *> *ContiguousCases = nullptr;
4566 BasicBlock *ContiguousDest = nullptr;
4567 BasicBlock *OtherDest = nullptr;
4568 if (!CasesA.empty() && CasesAreContiguous(CasesA)) {
4569 ContiguousCases = &CasesA;
4570 ContiguousDest = DestA;
4571 OtherDest = DestB;
4572 } else if (CasesAreContiguous(CasesB)) {
4573 ContiguousCases = &CasesB;
4574 ContiguousDest = DestB;
4575 OtherDest = DestA;
4576 } else
4577 return false;
4578
4579 // Start building the compare and branch.
4580
4581 Constant *Offset = ConstantExpr::getNeg(ContiguousCases->back());
4582 Constant *NumCases =
4583 ConstantInt::get(Offset->getType(), ContiguousCases->size());
4584
4585 Value *Sub = SI->getCondition();
4586 if (!Offset->isNullValue())
4587 Sub = Builder.CreateAdd(Sub, Offset, Sub->getName() + ".off");
4588
4589 Value *Cmp;
4590 // If NumCases overflowed, then all possible values jump to the successor.
4591 if (NumCases->isNullValue() && !ContiguousCases->empty())
4592 Cmp = ConstantInt::getTrue(SI->getContext());
4593 else
4594 Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch");
4595 BranchInst *NewBI = Builder.CreateCondBr(Cmp, ContiguousDest, OtherDest);
4596
4597 // Update weight for the newly-created conditional branch.
4598 if (HasBranchWeights(SI)) {
4599 SmallVector<uint64_t, 8> Weights;
4600 GetBranchWeights(SI, Weights);
4601 if (Weights.size() == 1 + SI->getNumCases()) {
4602 uint64_t TrueWeight = 0;
4603 uint64_t FalseWeight = 0;
4604 for (size_t I = 0, E = Weights.size(); I != E; ++I) {
4605 if (SI->getSuccessor(I) == ContiguousDest)
4606 TrueWeight += Weights[I];
4607 else
4608 FalseWeight += Weights[I];
4609 }
4610 while (TrueWeight > UINT32_MAX(4294967295U) || FalseWeight > UINT32_MAX(4294967295U)) {
4611 TrueWeight /= 2;
4612 FalseWeight /= 2;
4613 }
4614 setBranchWeights(NewBI, TrueWeight, FalseWeight);
4615 }
4616 }
4617
4618 // Prune obsolete incoming values off the successors' PHI nodes.
4619 for (auto BBI = ContiguousDest->begin(); isa<PHINode>(BBI); ++BBI) {
4620 unsigned PreviousEdges = ContiguousCases->size();
4621 if (ContiguousDest == SI->getDefaultDest())
4622 ++PreviousEdges;
4623 for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4624 cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4625 }
4626 for (auto BBI = OtherDest->begin(); isa<PHINode>(BBI); ++BBI) {
4627 unsigned PreviousEdges = SI->getNumCases() - ContiguousCases->size();
4628 if (OtherDest == SI->getDefaultDest())
4629 ++PreviousEdges;
4630 for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4631 cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4632 }
4633
4634 // Clean up the default block - it may have phis or other instructions before
4635 // the unreachable terminator.
4636 if (!HasDefault)
4637 createUnreachableSwitchDefault(SI);
4638
4639 // Drop the switch.
4640 SI->eraseFromParent();
4641
4642 return true;
4643}
4644
4645/// Compute masked bits for the condition of a switch
4646/// and use it to remove dead cases.
4647static bool eliminateDeadSwitchCases(SwitchInst *SI, AssumptionCache *AC,
4648 const DataLayout &DL) {
4649 Value *Cond = SI->getCondition();
4650 unsigned Bits = Cond->getType()->getIntegerBitWidth();
4651 KnownBits Known = computeKnownBits(Cond, DL, 0, AC, SI);
4652
4653 // We can also eliminate cases by determining that their values are outside of
4654 // the limited range of the condition based on how many significant (non-sign)
4655 // bits are in the condition value.
4656 unsigned ExtraSignBits = ComputeNumSignBits(Cond, DL, 0, AC, SI) - 1;
4657 unsigned MaxSignificantBitsInCond = Bits - ExtraSignBits;
4658
4659 // Gather dead cases.
4660 SmallVector<ConstantInt *, 8> DeadCases;
4661 for (auto &Case : SI->cases()) {
4662 const APInt &CaseVal = Case.getCaseValue()->getValue();
4663 if (Known.Zero.intersects(CaseVal) || !Known.One.isSubsetOf(CaseVal) ||
4664 (CaseVal.getMinSignedBits() > MaxSignificantBitsInCond)) {
4665 DeadCases.push_back(Case.getCaseValue());
4666 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseValdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SimplifyCFG: switch case "
<< CaseVal << " is dead.\n"; } } while (false)
4667 << " is dead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SimplifyCFG: switch case "
<< CaseVal << " is dead.\n"; } } while (false)
;
4668 }
4669 }
4670
4671 // If we can prove that the cases must cover all possible values, the
4672 // default destination becomes dead and we can remove it. If we know some
4673 // of the bits in the value, we can use that to more precisely compute the
4674 // number of possible unique case values.
4675 bool HasDefault =
4676 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
4677 const unsigned NumUnknownBits =
4678 Bits - (Known.Zero | Known.One).countPopulation();
4679 assert(NumUnknownBits <= Bits)((NumUnknownBits <= Bits) ? static_cast<void> (0) : __assert_fail
("NumUnknownBits <= Bits", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4679, __PRETTY_FUNCTION__))
;
4680 if (HasDefault && DeadCases.empty() &&
4681 NumUnknownBits < 64 /* avoid overflow */ &&
4682 SI->getNumCases() == (1ULL << NumUnknownBits)) {
4683 createUnreachableSwitchDefault(SI);
4684 return true;
4685 }
4686
4687 if (DeadCases.empty())
4688 return false;
4689
4690 SwitchInstProfUpdateWrapper SIW(*SI);
4691 for (ConstantInt *DeadCase : DeadCases) {
4692 SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase);
4693 assert(CaseI != SI->case_default() &&((CaseI != SI->case_default() && "Case was not found. Probably mistake in DeadCases forming."
) ? static_cast<void> (0) : __assert_fail ("CaseI != SI->case_default() && \"Case was not found. Probably mistake in DeadCases forming.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4694, __PRETTY_FUNCTION__))
4694 "Case was not found. Probably mistake in DeadCases forming.")((CaseI != SI->case_default() && "Case was not found. Probably mistake in DeadCases forming."
) ? static_cast<void> (0) : __assert_fail ("CaseI != SI->case_default() && \"Case was not found. Probably mistake in DeadCases forming.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4694, __PRETTY_FUNCTION__))
;
4695 // Prune unused values from PHI nodes.
4696 CaseI->getCaseSuccessor()->removePredecessor(SI->getParent());
4697 SIW.removeCase(CaseI);
4698 }
4699
4700 return true;
4701}
4702
4703/// If BB would be eligible for simplification by
4704/// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated
4705/// by an unconditional branch), look at the phi node for BB in the successor
4706/// block and see if the incoming value is equal to CaseValue. If so, return
4707/// the phi node, and set PhiIndex to BB's index in the phi node.
4708static PHINode *FindPHIForConditionForwarding(ConstantInt *CaseValue,
4709 BasicBlock *BB, int *PhiIndex) {
4710 if (BB->getFirstNonPHIOrDbg() != BB->getTerminator())
4711 return nullptr; // BB must be empty to be a candidate for simplification.
4712 if (!BB->getSinglePredecessor())
4713 return nullptr; // BB must be dominated by the switch.
4714
4715 BranchInst *Branch = dyn_cast<BranchInst>(BB->getTerminator());
4716 if (!Branch || !Branch->isUnconditional())
4717 return nullptr; // Terminator must be unconditional branch.
4718
4719 BasicBlock *Succ = Branch->getSuccessor(0);
4720
4721 for (PHINode &PHI : Succ->phis()) {
4722 int Idx = PHI.getBasicBlockIndex(BB);
4723 assert(Idx >= 0 && "PHI has no entry for predecessor?")((Idx >= 0 && "PHI has no entry for predecessor?")
? static_cast<void> (0) : __assert_fail ("Idx >= 0 && \"PHI has no entry for predecessor?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4723, __PRETTY_FUNCTION__))
;
4724
4725 Value *InValue = PHI.getIncomingValue(Idx);
4726 if (InValue != CaseValue)
4727 continue;
4728
4729 *PhiIndex = Idx;
4730 return &PHI;
4731 }
4732
4733 return nullptr;
4734}
4735
4736/// Try to forward the condition of a switch instruction to a phi node
4737/// dominated by the switch, if that would mean that some of the destination
4738/// blocks of the switch can be folded away. Return true if a change is made.
4739static bool ForwardSwitchConditionToPHI(SwitchInst *SI) {
4740 using ForwardingNodesMap = DenseMap<PHINode *, SmallVector<int, 4>>;
4741
4742 ForwardingNodesMap ForwardingNodes;
4743 BasicBlock *SwitchBlock = SI->getParent();
4744 bool Changed = false;
4745 for (auto &Case : SI->cases()) {
4746 ConstantInt *CaseValue = Case.getCaseValue();
4747 BasicBlock *CaseDest = Case.getCaseSuccessor();
4748
4749 // Replace phi operands in successor blocks that are using the constant case
4750 // value rather than the switch condition variable:
4751 // switchbb:
4752 // switch i32 %x, label %default [
4753 // i32 17, label %succ
4754 // ...
4755 // succ:
4756 // %r = phi i32 ... [ 17, %switchbb ] ...
4757 // -->
4758 // %r = phi i32 ... [ %x, %switchbb ] ...
4759
4760 for (PHINode &Phi : CaseDest->phis()) {
4761 // This only works if there is exactly 1 incoming edge from the switch to
4762 // a phi. If there is >1, that means multiple cases of the switch map to 1
4763 // value in the phi, and that phi value is not the switch condition. Thus,
4764 // this transform would not make sense (the phi would be invalid because
4765 // a phi can't have different incoming values from the same block).
4766 int SwitchBBIdx = Phi.getBasicBlockIndex(SwitchBlock);
4767 if (Phi.getIncomingValue(SwitchBBIdx) == CaseValue &&
4768 count(Phi.blocks(), SwitchBlock) == 1) {
4769 Phi.setIncomingValue(SwitchBBIdx, SI->getCondition());
4770 Changed = true;
4771 }
4772 }
4773
4774 // Collect phi nodes that are indirectly using this switch's case constants.
4775 int PhiIdx;
4776 if (auto *Phi = FindPHIForConditionForwarding(CaseValue, CaseDest, &PhiIdx))
4777 ForwardingNodes[Phi].push_back(PhiIdx);
4778 }
4779
4780 for (auto &ForwardingNode : ForwardingNodes) {
4781 PHINode *Phi = ForwardingNode.first;
4782 SmallVectorImpl<int> &Indexes = ForwardingNode.second;
4783 if (Indexes.size() < 2)
4784 continue;
4785
4786 for (int Index : Indexes)
4787 Phi->setIncomingValue(Index, SI->getCondition());
4788 Changed = true;
4789 }
4790
4791 return Changed;
4792}
4793
4794/// Return true if the backend will be able to handle
4795/// initializing an array of constants like C.
4796static bool ValidLookupTableConstant(Constant *C, const TargetTransformInfo &TTI) {
4797 if (C->isThreadDependent())
4798 return false;
4799 if (C->isDLLImportDependent())
4800 return false;
4801
4802 if (!isa<ConstantFP>(C) && !isa<ConstantInt>(C) &&
4803 !isa<ConstantPointerNull>(C) && !isa<GlobalValue>(C) &&
4804 !isa<UndefValue>(C) && !isa<ConstantExpr>(C))
4805 return false;
4806
4807 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
4808 if (!CE->isGEPWithNoNotionalOverIndexing())
4809 return false;
4810 if (!ValidLookupTableConstant(CE->getOperand(0), TTI))
4811 return false;
4812 }
4813
4814 if (!TTI.shouldBuildLookupTablesForConstant(C))
4815 return false;
4816
4817 return true;
4818}
4819
4820/// If V is a Constant, return it. Otherwise, try to look up
4821/// its constant value in ConstantPool, returning 0 if it's not there.
4822static Constant *
4823LookupConstant(Value *V,
4824 const SmallDenseMap<Value *, Constant *> &ConstantPool) {
4825 if (Constant *C = dyn_cast<Constant>(V))
4826 return C;
4827 return ConstantPool.lookup(V);
4828}
4829
4830/// Try to fold instruction I into a constant. This works for
4831/// simple instructions such as binary operations where both operands are
4832/// constant or can be replaced by constants from the ConstantPool. Returns the
4833/// resulting constant on success, 0 otherwise.
4834static Constant *
4835ConstantFold(Instruction *I, const DataLayout &DL,
4836 const SmallDenseMap<Value *, Constant *> &ConstantPool) {
4837 if (SelectInst *Select = dyn_cast<SelectInst>(I)) {
4838 Constant *A = LookupConstant(Select->getCondition(), ConstantPool);
4839 if (!A)
4840 return nullptr;
4841 if (A->isAllOnesValue())
4842 return LookupConstant(Select->getTrueValue(), ConstantPool);
4843 if (A->isNullValue())
4844 return LookupConstant(Select->getFalseValue(), ConstantPool);
4845 return nullptr;
4846 }
4847
4848 SmallVector<Constant *, 4> COps;
4849 for (unsigned N = 0, E = I->getNumOperands(); N != E; ++N) {
4850 if (Constant *A = LookupConstant(I->getOperand(N), ConstantPool))
4851 COps.push_back(A);
4852 else
4853 return nullptr;
4854 }
4855
4856 if (CmpInst *Cmp = dyn_cast<CmpInst>(I)) {
4857 return ConstantFoldCompareInstOperands(Cmp->getPredicate(), COps[0],
4858 COps[1], DL);
4859 }
4860
4861 return ConstantFoldInstOperands(I, COps, DL);
4862}
4863
4864/// Try to determine the resulting constant values in phi nodes
4865/// at the common destination basic block, *CommonDest, for one of the case
4866/// destionations CaseDest corresponding to value CaseVal (0 for the default
4867/// case), of a switch instruction SI.
4868static bool
4869GetCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest,
4870 BasicBlock **CommonDest,
4871 SmallVectorImpl<std::pair<PHINode *, Constant *>> &Res,
4872 const DataLayout &DL, const TargetTransformInfo &TTI) {
4873 // The block from which we enter the common destination.
4874 BasicBlock *Pred = SI->getParent();
4875
4876 // If CaseDest is empty except for some side-effect free instructions through
4877 // which we can constant-propagate the CaseVal, continue to its successor.
4878 SmallDenseMap<Value *, Constant *> ConstantPool;
4879 ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal));
4880 for (Instruction &I :CaseDest->instructionsWithoutDebug()) {
4881 if (I.isTerminator()) {
4882 // If the terminator is a simple branch, continue to the next block.
4883 if (I.getNumSuccessors() != 1 || I.isExceptionalTerminator())
4884 return false;
4885 Pred = CaseDest;
4886 CaseDest = I.getSuccessor(0);
4887 } else if (Constant *C = ConstantFold(&I, DL, ConstantPool)) {
4888 // Instruction is side-effect free and constant.
4889
4890 // If the instruction has uses outside this block or a phi node slot for
4891 // the block, it is not safe to bypass the instruction since it would then
4892 // no longer dominate all its uses.
4893 for (auto &Use : I.uses()) {
4894 User *User = Use.getUser();
4895 if (Instruction *I = dyn_cast<Instruction>(User))
4896 if (I->getParent() == CaseDest)
4897 continue;
4898 if (PHINode *Phi = dyn_cast<PHINode>(User))
4899 if (Phi->getIncomingBlock(Use) == CaseDest)
4900 continue;
4901 return false;
4902 }
4903
4904 ConstantPool.insert(std::make_pair(&I, C));
4905 } else {
4906 break;
4907 }
4908 }
4909
4910 // If we did not have a CommonDest before, use the current one.
4911 if (!*CommonDest)
4912 *CommonDest = CaseDest;
4913 // If the destination isn't the common one, abort.
4914 if (CaseDest != *CommonDest)
4915 return false;
4916
4917 // Get the values for this case from phi nodes in the destination block.
4918 for (PHINode &PHI : (*CommonDest)->phis()) {
4919 int Idx = PHI.getBasicBlockIndex(Pred);
4920 if (Idx == -1)
4921 continue;
4922
4923 Constant *ConstVal =
4924 LookupConstant(PHI.getIncomingValue(Idx), ConstantPool);
4925 if (!ConstVal)
4926 return false;
4927
4928 // Be conservative about which kinds of constants we support.
4929 if (!ValidLookupTableConstant(ConstVal, TTI))
4930 return false;
4931
4932 Res.push_back(std::make_pair(&PHI, ConstVal));
4933 }
4934
4935 return Res.size() > 0;
4936}
4937
4938// Helper function used to add CaseVal to the list of cases that generate
4939// Result. Returns the updated number of cases that generate this result.
4940static uintptr_t MapCaseToResult(ConstantInt *CaseVal,
4941 SwitchCaseResultVectorTy &UniqueResults,
4942 Constant *Result) {
4943 for (auto &I : UniqueResults) {
4944 if (I.first == Result) {
4945 I.second.push_back(CaseVal);
4946 return I.second.size();
4947 }
4948 }
4949 UniqueResults.push_back(
4950 std::make_pair(Result, SmallVector<ConstantInt *, 4>(1, CaseVal)));
4951 return 1;
4952}
4953
4954// Helper function that initializes a map containing
4955// results for the PHI node of the common destination block for a switch
4956// instruction. Returns false if multiple PHI nodes have been found or if
4957// there is not a common destination block for the switch.
4958static bool
4959InitializeUniqueCases(SwitchInst *SI, PHINode *&PHI, BasicBlock *&CommonDest,
4960 SwitchCaseResultVectorTy &UniqueResults,
4961 Constant *&DefaultResult, const DataLayout &DL,
4962 const TargetTransformInfo &TTI,
4963 uintptr_t MaxUniqueResults, uintptr_t MaxCasesPerResult) {
4964 for (auto &I : SI->cases()) {
4965 ConstantInt *CaseVal = I.getCaseValue();
4966
4967 // Resulting value at phi nodes for this case value.
4968 SwitchCaseResultsTy Results;
4969 if (!GetCaseResults(SI, CaseVal, I.getCaseSuccessor(), &CommonDest, Results,
4970 DL, TTI))
4971 return false;
4972
4973 // Only one value per case is permitted.
4974 if (Results.size() > 1)
4975 return false;
4976
4977 // Add the case->result mapping to UniqueResults.
4978 const uintptr_t NumCasesForResult =
4979 MapCaseToResult(CaseVal, UniqueResults, Results.begin()->second);
4980
4981 // Early out if there are too many cases for this result.
4982 if (NumCasesForResult > MaxCasesPerResult)
4983 return false;
4984
4985 // Early out if there are too many unique results.
4986 if (UniqueResults.size() > MaxUniqueResults)
4987 return false;
4988
4989 // Check the PHI consistency.
4990 if (!PHI)
4991 PHI = Results[0].first;
4992 else if (PHI != Results[0].first)
4993 return false;
4994 }
4995 // Find the default result value.
4996 SmallVector<std::pair<PHINode *, Constant *>, 1> DefaultResults;
4997 BasicBlock *DefaultDest = SI->getDefaultDest();
4998 GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, DefaultResults,
4999 DL, TTI);
5000 // If the default value is not found abort unless the default destination
5001 // is unreachable.
5002 DefaultResult =
5003 DefaultResults.size() == 1 ? DefaultResults.begin()->second : nullptr;
5004 if ((!DefaultResult &&
5005 !isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg())))
5006 return false;
5007
5008 return true;
5009}
5010
5011// Helper function that checks if it is possible to transform a switch with only
5012// two cases (or two cases + default) that produces a result into a select.
5013// Example:
5014// switch (a) {
5015// case 10: %0 = icmp eq i32 %a, 10
5016// return 10; %1 = select i1 %0, i32 10, i32 4
5017// case 20: ----> %2 = icmp eq i32 %a, 20
5018// return 2; %3 = select i1 %2, i32 2, i32 %1
5019// default:
5020// return 4;
5021// }
5022static Value *ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector,
5023 Constant *DefaultResult, Value *Condition,
5024 IRBuilder<> &Builder) {
5025 assert(ResultVector.size() == 2 &&((ResultVector.size() == 2 && "We should have exactly two unique results at this point"
) ? static_cast<void> (0) : __assert_fail ("ResultVector.size() == 2 && \"We should have exactly two unique results at this point\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5026, __PRETTY_FUNCTION__))
5026 "We should have exactly two unique results at this point")((ResultVector.size() == 2 && "We should have exactly two unique results at this point"
) ? static_cast<void> (0) : __assert_fail ("ResultVector.size() == 2 && \"We should have exactly two unique results at this point\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5026, __PRETTY_FUNCTION__))
;
5027 // If we are selecting between only two cases transform into a simple
5028 // select or a two-way select if default is possible.
5029 if (ResultVector[0].second.size() == 1 &&
5030 ResultVector[1].second.size() == 1) {
5031 ConstantInt *const FirstCase = ResultVector[0].second[0];
5032 ConstantInt *const SecondCase = ResultVector[1].second[0];
5033
5034 bool DefaultCanTrigger = DefaultResult;
5035 Value *SelectValue = ResultVector[1].first;
5036 if (DefaultCanTrigger) {
5037 Value *const ValueCompare =
5038 Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp");
5039 SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first,
5040 DefaultResult, "switch.select");
5041 }
5042 Value *const ValueCompare =
5043 Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp");
5044 return Builder.CreateSelect(ValueCompare, ResultVector[0].first,
5045 SelectValue, "switch.select");
5046 }
5047
5048 return nullptr;
5049}
5050
5051// Helper function to cleanup a switch instruction that has been converted into
5052// a select, fixing up PHI nodes and basic blocks.
5053static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI,
5054 Value *SelectValue,
5055 IRBuilder<> &Builder) {
5056 BasicBlock *SelectBB = SI->getParent();
5057 while (PHI->getBasicBlockIndex(SelectBB) >= 0)
5058 PHI->removeIncomingValue(SelectBB);
5059 PHI->addIncoming(SelectValue, SelectBB);
5060
5061 Builder.CreateBr(PHI->getParent());
5062
5063 // Remove the switch.
5064 for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
5065 BasicBlock *Succ = SI->getSuccessor(i);
5066
5067 if (Succ == PHI->getParent())
5068 continue;
5069 Succ->removePredecessor(SelectBB);
5070 }
5071 SI->eraseFromParent();
5072}
5073
5074/// If the switch is only used to initialize one or more
5075/// phi nodes in a common successor block with only two different
5076/// constant values, replace the switch with select.
5077static bool switchToSelect(SwitchInst *SI, IRBuilder<> &Builder,
5078 const DataLayout &DL,
5079 const TargetTransformInfo &TTI) {
5080 Value *const Cond = SI->getCondition();
5081 PHINode *PHI = nullptr;
5082 BasicBlock *CommonDest = nullptr;
5083 Constant *DefaultResult;
5084 SwitchCaseResultVectorTy UniqueResults;
5085 // Collect all the cases that will deliver the same value from the switch.
5086 if (!InitializeUniqueCases(SI, PHI, CommonDest, UniqueResults, DefaultResult,
5087 DL, TTI, 2, 1))
5088 return false;
5089 // Selects choose between maximum two values.
5090 if (UniqueResults.size() != 2)
5091 return false;
5092 assert(PHI != nullptr && "PHI for value select not found")((PHI != nullptr && "PHI for value select not found")
? static_cast<void> (0) : __assert_fail ("PHI != nullptr && \"PHI for value select not found\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5092, __PRETTY_FUNCTION__))
;
5093
5094 Builder.SetInsertPoint(SI);
5095 Value *SelectValue =
5096 ConvertTwoCaseSwitch(UniqueResults, DefaultResult, Cond, Builder);
5097 if (SelectValue) {
5098 RemoveSwitchAfterSelectConversion(SI, PHI, SelectValue, Builder);
5099 return true;
5100 }
5101 // The switch couldn't be converted into a select.
5102 return false;
5103}
5104
5105namespace {
5106
5107/// This class represents a lookup table that can be used to replace a switch.
5108class SwitchLookupTable {
5109public:
5110 /// Create a lookup table to use as a switch replacement with the contents
5111 /// of Values, using DefaultValue to fill any holes in the table.
5112 SwitchLookupTable(
5113 Module &M, uint64_t TableSize, ConstantInt *Offset,
5114 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
5115 Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName);
5116
5117 /// Build instructions with Builder to retrieve the value at
5118 /// the position given by Index in the lookup table.
5119 Value *BuildLookup(Value *Index, IRBuilder<> &Builder);
5120
5121 /// Return true if a table with TableSize elements of
5122 /// type ElementType would fit in a target-legal register.
5123 static bool WouldFitInRegister(const DataLayout &DL, uint64_t TableSize,
5124 Type *ElementType);
5125
5126private:
5127 // Depending on the contents of the table, it can be represented in
5128 // different ways.
5129 enum {
5130 // For tables where each element contains the same value, we just have to
5131 // store that single value and return it for each lookup.
5132 SingleValueKind,
5133
5134 // For tables where there is a linear relationship between table index
5135 // and values. We calculate the result with a simple multiplication
5136 // and addition instead of a table lookup.
5137 LinearMapKind,
5138
5139 // For small tables with integer elements, we can pack them into a bitmap
5140 // that fits into a target-legal register. Values are retrieved by
5141 // shift and mask operations.
5142 BitMapKind,
5143
5144 // The table is stored as an array of values. Values are retrieved by load
5145 // instructions from the table.
5146 ArrayKind
5147 } Kind;
5148
5149 // For SingleValueKind, this is the single value.
5150 Constant *SingleValue = nullptr;
5151
5152 // For BitMapKind, this is the bitmap.
5153 ConstantInt *BitMap = nullptr;
5154 IntegerType *BitMapElementTy = nullptr;
5155
5156 // For LinearMapKind, these are the constants used to derive the value.
5157 ConstantInt *LinearOffset = nullptr;
5158 ConstantInt *LinearMultiplier = nullptr;
5159
5160 // For ArrayKind, this is the array.
5161 GlobalVariable *Array = nullptr;
5162};
5163
5164} // end anonymous namespace
5165
5166SwitchLookupTable::SwitchLookupTable(
5167 Module &M, uint64_t TableSize, ConstantInt *Offset,
5168 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
5169 Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName) {
5170 assert(Values.size() && "Can't build lookup table without values!")((Values.size() && "Can't build lookup table without values!"
) ? static_cast<void> (0) : __assert_fail ("Values.size() && \"Can't build lookup table without values!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5170, __PRETTY_FUNCTION__))
;
5171 assert(TableSize >= Values.size() && "Can't fit values in table!")((TableSize >= Values.size() && "Can't fit values in table!"
) ? static_cast<void> (0) : __assert_fail ("TableSize >= Values.size() && \"Can't fit values in table!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5171, __PRETTY_FUNCTION__))
;
5172
5173 // If all values in the table are equal, this is that value.
5174 SingleValue = Values.begin()->second;
5175
5176 Type *ValueType = Values.begin()->second->getType();
5177
5178 // Build up the table contents.
5179 SmallVector<Constant *, 64> TableContents(TableSize);
5180 for (size_t I = 0, E = Values.size(); I != E; ++I) {
5181 ConstantInt *CaseVal = Values[I].first;
5182 Constant *CaseRes = Values[I].second;
5183 assert(CaseRes->getType() == ValueType)((CaseRes->getType() == ValueType) ? static_cast<void>
(0) : __assert_fail ("CaseRes->getType() == ValueType", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5183, __PRETTY_FUNCTION__))
;
5184
5185 uint64_t Idx = (CaseVal->getValue() - Offset->getValue()).getLimitedValue();
5186 TableContents[Idx] = CaseRes;
5187
5188 if (CaseRes != SingleValue)
5189 SingleValue = nullptr;
5190 }
5191
5192 // Fill in any holes in the table with the default result.
5193 if (Values.size() < TableSize) {
5194 assert(DefaultValue &&((DefaultValue && "Need a default value to fill the lookup table holes."
) ? static_cast<void> (0) : __assert_fail ("DefaultValue && \"Need a default value to fill the lookup table holes.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5195, __PRETTY_FUNCTION__))
5195 "Need a default value to fill the lookup table holes.")((DefaultValue && "Need a default value to fill the lookup table holes."
) ? static_cast<void> (0) : __assert_fail ("DefaultValue && \"Need a default value to fill the lookup table holes.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5195, __PRETTY_FUNCTION__))
;
5196 assert(DefaultValue->getType() == ValueType)((DefaultValue->getType() == ValueType) ? static_cast<void
> (0) : __assert_fail ("DefaultValue->getType() == ValueType"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5196, __PRETTY_FUNCTION__))
;
5197 for (uint64_t I = 0; I < TableSize; ++I) {
5198 if (!TableContents[I])
5199 TableContents[I] = DefaultValue;
5200 }
5201
5202 if (DefaultValue != SingleValue)
5203 SingleValue = nullptr;
5204 }
5205
5206 // If each element in the table contains the same value, we only need to store
5207 // that single value.
5208 if (SingleValue) {
5209 Kind = SingleValueKind;
5210 return;
5211 }
5212
5213 // Check if we can derive the value with a linear transformation from the
5214 // table index.
5215 if (isa<IntegerType>(ValueType)) {
5216 bool LinearMappingPossible = true;
5217 APInt PrevVal;
5218 APInt DistToPrev;
5219 assert(TableSize >= 2 && "Should be a SingleValue table.")((TableSize >= 2 && "Should be a SingleValue table."
) ? static_cast<void> (0) : __assert_fail ("TableSize >= 2 && \"Should be a SingleValue table.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5219, __PRETTY_FUNCTION__))
;
5220 // Check if there is the same distance between two consecutive values.
5221 for (uint64_t I = 0; I < TableSize; ++I) {
5222 ConstantInt *ConstVal = dyn_cast<ConstantInt>(TableContents[I]);
5223 if (!ConstVal) {
5224 // This is an undef. We could deal with it, but undefs in lookup tables
5225 // are very seldom. It's probably not worth the additional complexity.
5226 LinearMappingPossible = false;
5227 break;
5228 }
5229 const APInt &Val = ConstVal->getValue();
5230 if (I != 0) {
5231 APInt Dist = Val - PrevVal;
5232 if (I == 1) {
5233 DistToPrev = Dist;
5234 } else if (Dist != DistToPrev) {
5235 LinearMappingPossible = false;
5236 break;
5237 }
5238 }
5239 PrevVal = Val;
5240 }
5241 if (LinearMappingPossible) {
5242 LinearOffset = cast<ConstantInt>(TableContents[0]);
5243 LinearMultiplier = ConstantInt::get(M.getContext(), DistToPrev);
5244 Kind = LinearMapKind;
5245 ++NumLinearMaps;
5246 return;
5247 }
5248 }
5249
5250 // If the type is integer and the table fits in a register, build a bitmap.
5251 if (WouldFitInRegister(DL, TableSize, ValueType)) {
5252 IntegerType *IT = cast<IntegerType>(ValueType);
5253 APInt TableInt(TableSize * IT->getBitWidth(), 0);
5254 for (uint64_t I = TableSize; I > 0; --I) {
5255 TableInt <<= IT->getBitWidth();
5256 // Insert values into the bitmap. Undef values are set to zero.
5257 if (!isa<UndefValue>(TableContents[I - 1])) {
5258 ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]);
5259 TableInt |= Val->getValue().zext(TableInt.getBitWidth());
5260 }
5261 }
5262 BitMap = ConstantInt::get(M.getContext(), TableInt);
5263 BitMapElementTy = IT;
5264 Kind = BitMapKind;
5265 ++NumBitMaps;
5266 return;
5267 }
5268
5269 // Store the table in an array.
5270 ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize);
5271 Constant *Initializer = ConstantArray::get(ArrayTy, TableContents);
5272
5273 Array = new GlobalVariable(M, ArrayTy, /*isConstant=*/true,
5274 GlobalVariable::PrivateLinkage, Initializer,
5275 "switch.table." + FuncName);
5276 Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
5277 // Set the alignment to that of an array items. We will be only loading one
5278 // value out of it.
5279 Array->setAlignment(Align(DL.getPrefTypeAlignment(ValueType)));
5280 Kind = ArrayKind;
5281}
5282
5283Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) {
5284 switch (Kind) {
5285 case SingleValueKind:
5286 return SingleValue;
5287 case LinearMapKind: {
5288 // Derive the result value from the input value.
5289 Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(),
5290 false, "switch.idx.cast");
5291 if (!LinearMultiplier->isOne())
5292 Result = Builder.CreateMul(Result, LinearMultiplier, "switch.idx.mult");
5293 if (!LinearOffset->isZero())
5294 Result = Builder.CreateAdd(Result, LinearOffset, "switch.offset");
5295 return Result;
5296 }
5297 case BitMapKind: {
5298 // Type of the bitmap (e.g. i59).
5299 IntegerType *MapTy = BitMap->getType();
5300
5301 // Cast Index to the same type as the bitmap.
5302 // Note: The Index is <= the number of elements in the table, so
5303 // truncating it to the width of the bitmask is safe.
5304 Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast");
5305
5306 // Multiply the shift amount by the element width.
5307 ShiftAmt = Builder.CreateMul(
5308 ShiftAmt, ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()),
5309 "switch.shiftamt");
5310
5311 // Shift down.
5312 Value *DownShifted =
5313 Builder.CreateLShr(BitMap, ShiftAmt, "switch.downshift");
5314 // Mask off.
5315 return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked");
5316 }
5317 case ArrayKind: {
5318 // Make sure the table index will not overflow when treated as signed.
5319 IntegerType *IT = cast<IntegerType>(Index->getType());
5320 uint64_t TableSize =
5321 Array->getInitializer()->getType()->getArrayNumElements();
5322 if (TableSize > (1ULL << (IT->getBitWidth() - 1)))
5323 Index = Builder.CreateZExt(
5324 Index, IntegerType::get(IT->getContext(), IT->getBitWidth() + 1),
5325 "switch.tableidx.zext");
5326
5327 Value *GEPIndices[] = {Builder.getInt32(0), Index};
5328 Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array,
5329 GEPIndices, "switch.gep");
5330 return Builder.CreateLoad(
5331 cast<ArrayType>(Array->getValueType())->getElementType(), GEP,
5332 "switch.load");
5333 }
5334 }
5335 llvm_unreachable("Unknown lookup table kind!")::llvm::llvm_unreachable_internal("Unknown lookup table kind!"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5335)
;
5336}
5337
5338bool SwitchLookupTable::WouldFitInRegister(const DataLayout &DL,
5339 uint64_t TableSize,
5340 Type *ElementType) {
5341 auto *IT = dyn_cast<IntegerType>(ElementType);
5342 if (!IT)
5343 return false;
5344 // FIXME: If the type is wider than it needs to be, e.g. i8 but all values
5345 // are <= 15, we could try to narrow the type.
5346
5347 // Avoid overflow, fitsInLegalInteger uses unsigned int for the width.
5348 if (TableSize >= UINT_MAX(2147483647 *2U +1U) / IT->getBitWidth())
5349 return false;
5350 return DL.fitsInLegalInteger(TableSize * IT->getBitWidth());
5351}
5352
5353/// Determine whether a lookup table should be built for this switch, based on
5354/// the number of cases, size of the table, and the types of the results.
5355static bool
5356ShouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize,
5357 const TargetTransformInfo &TTI, const DataLayout &DL,
5358 const SmallDenseMap<PHINode *, Type *> &ResultTypes) {
5359 if (SI->getNumCases() > TableSize || TableSize >= UINT64_MAX(18446744073709551615UL) / 10)
5360 return false; // TableSize overflowed, or mul below might overflow.
5361
5362 bool AllTablesFitInRegister = true;
5363 bool HasIllegalType = false;
5364 for (const auto &I : ResultTypes) {
5365 Type *Ty = I.second;
5366
5367 // Saturate this flag to true.
5368 HasIllegalType = HasIllegalType || !TTI.isTypeLegal(Ty);
5369
5370 // Saturate this flag to false.
5371 AllTablesFitInRegister =
5372 AllTablesFitInRegister &&
5373 SwitchLookupTable::WouldFitInRegister(DL, TableSize, Ty);
5374
5375 // If both flags saturate, we're done. NOTE: This *only* works with
5376 // saturating flags, and all flags have to saturate first due to the
5377 // non-deterministic behavior of iterating over a dense map.
5378 if (HasIllegalType && !AllTablesFitInRegister)
5379 break;
5380 }
5381
5382 // If each table would fit in a register, we should build it anyway.
5383 if (AllTablesFitInRegister)
5384 return true;
5385
5386 // Don't build a table that doesn't fit in-register if it has illegal types.
5387 if (HasIllegalType)
5388 return false;
5389
5390 // The table density should be at least 40%. This is the same criterion as for
5391 // jump tables, see SelectionDAGBuilder::handleJTSwitchCase.
5392 // FIXME: Find the best cut-off.
5393 return SI->getNumCases() * 10 >= TableSize * 4;
5394}
5395
5396/// Try to reuse the switch table index compare. Following pattern:
5397/// \code
5398/// if (idx < tablesize)
5399/// r = table[idx]; // table does not contain default_value
5400/// else
5401/// r = default_value;
5402/// if (r != default_value)
5403/// ...
5404/// \endcode
5405/// Is optimized to:
5406/// \code
5407/// cond = idx < tablesize;
5408/// if (cond)
5409/// r = table[idx];
5410/// else
5411/// r = default_value;
5412/// if (cond)
5413/// ...
5414/// \endcode
5415/// Jump threading will then eliminate the second if(cond).
5416static void reuseTableCompare(
5417 User *PhiUser, BasicBlock *PhiBlock, BranchInst *RangeCheckBranch,
5418 Constant *DefaultValue,
5419 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values) {
5420 ICmpInst *CmpInst = dyn_cast<ICmpInst>(PhiUser);
5421 if (!CmpInst)
5422 return;
5423
5424 // We require that the compare is in the same block as the phi so that jump
5425 // threading can do its work afterwards.
5426 if (CmpInst->getParent() != PhiBlock)
5427 return;
5428
5429 Constant *CmpOp1 = dyn_cast<Constant>(CmpInst->getOperand(1));
5430 if (!CmpOp1)
5431 return;
5432
5433 Value *RangeCmp = RangeCheckBranch->getCondition();
5434 Constant *TrueConst = ConstantInt::getTrue(RangeCmp->getType());
5435 Constant *FalseConst = ConstantInt::getFalse(RangeCmp->getType());
5436
5437 // Check if the compare with the default value is constant true or false.
5438 Constant *DefaultConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
5439 DefaultValue, CmpOp1, true);
5440 if (DefaultConst != TrueConst && DefaultConst != FalseConst)
5441 return;
5442
5443 // Check if the compare with the case values is distinct from the default
5444 // compare result.
5445 for (auto ValuePair : Values) {
5446 Constant *CaseConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
5447 ValuePair.second, CmpOp1, true);
5448 if (!CaseConst || CaseConst == DefaultConst || isa<UndefValue>(CaseConst))
5449 return;
5450 assert((CaseConst == TrueConst || CaseConst == FalseConst) &&(((CaseConst == TrueConst || CaseConst == FalseConst) &&
"Expect true or false as compare result.") ? static_cast<
void> (0) : __assert_fail ("(CaseConst == TrueConst || CaseConst == FalseConst) && \"Expect true or false as compare result.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5451, __PRETTY_FUNCTION__))
5451 "Expect true or false as compare result.")(((CaseConst == TrueConst || CaseConst == FalseConst) &&
"Expect true or false as compare result.") ? static_cast<
void> (0) : __assert_fail ("(CaseConst == TrueConst || CaseConst == FalseConst) && \"Expect true or false as compare result.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5451, __PRETTY_FUNCTION__))
;
5452 }
5453
5454 // Check if the branch instruction dominates the phi node. It's a simple
5455 // dominance check, but sufficient for our needs.
5456 // Although this check is invariant in the calling loops, it's better to do it
5457 // at this late stage. Practically we do it at most once for a switch.
5458 BasicBlock *BranchBlock = RangeCheckBranch->getParent();
5459 for (auto PI = pred_begin(PhiBlock), E = pred_end(PhiBlock); PI != E; ++PI) {
5460 BasicBlock *Pred = *PI;
5461 if (Pred != BranchBlock && Pred->getUniquePredecessor() != BranchBlock)
5462 return;
5463 }
5464
5465 if (DefaultConst == FalseConst) {
5466 // The compare yields the same result. We can replace it.
5467 CmpInst->replaceAllUsesWith(RangeCmp);
5468 ++NumTableCmpReuses;
5469 } else {
5470 // The compare yields the same result, just inverted. We can replace it.
5471 Value *InvertedTableCmp = BinaryOperator::CreateXor(
5472 RangeCmp, ConstantInt::get(RangeCmp->getType(), 1), "inverted.cmp",
5473 RangeCheckBranch);
5474 CmpInst->replaceAllUsesWith(InvertedTableCmp);
5475 ++NumTableCmpReuses;
5476 }
5477}
5478
5479/// If the switch is only used to initialize one or more phi nodes in a common
5480/// successor block with different constant values, replace the switch with
5481/// lookup tables.
5482static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
5483 const DataLayout &DL,
5484 const TargetTransformInfo &TTI) {
5485 assert(SI->getNumCases() > 1 && "Degenerate switch?")((SI->getNumCases() > 1 && "Degenerate switch?"
) ? static_cast<void> (0) : __assert_fail ("SI->getNumCases() > 1 && \"Degenerate switch?\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5485, __PRETTY_FUNCTION__))
;
5486
5487 Function *Fn = SI->getParent()->getParent();
5488 // Only build lookup table when we have a target that supports it or the
5489 // attribute is not set.
5490 if (!TTI.shouldBuildLookupTables() ||
5491 (Fn->getFnAttribute("no-jump-tables").getValueAsString() == "true"))
5492 return false;
5493
5494 // FIXME: If the switch is too sparse for a lookup table, perhaps we could
5495 // split off a dense part and build a lookup table for that.
5496
5497 // FIXME: This creates arrays of GEPs to constant strings, which means each
5498 // GEP needs a runtime relocation in PIC code. We should just build one big
5499 // string and lookup indices into that.
5500
5501 // Ignore switches with less than three cases. Lookup tables will not make
5502 // them faster, so we don't analyze them.
5503 if (SI->getNumCases() < 3)
5504 return false;
5505
5506 // Figure out the corresponding result for each case value and phi node in the
5507 // common destination, as well as the min and max case values.
5508 assert(!SI->cases().empty())((!SI->cases().empty()) ? static_cast<void> (0) : __assert_fail
("!SI->cases().empty()", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5508, __PRETTY_FUNCTION__))
;
5509 SwitchInst::CaseIt CI = SI->case_begin();
5510 ConstantInt *MinCaseVal = CI->getCaseValue();
5511 ConstantInt *MaxCaseVal = CI->getCaseValue();
5512
5513 BasicBlock *CommonDest = nullptr;
5514
5515 using ResultListTy = SmallVector<std::pair<ConstantInt *, Constant *>, 4>;
5516 SmallDenseMap<PHINode *, ResultListTy> ResultLists;
5517
5518 SmallDenseMap<PHINode *, Constant *> DefaultResults;
5519 SmallDenseMap<PHINode *, Type *> ResultTypes;
5520 SmallVector<PHINode *, 4> PHIs;
5521
5522 for (SwitchInst::CaseIt E = SI->case_end(); CI != E; ++CI) {
5523 ConstantInt *CaseVal = CI->getCaseValue();
5524 if (CaseVal->getValue().slt(MinCaseVal->getValue()))
5525 MinCaseVal = CaseVal;
5526 if (CaseVal->getValue().sgt(MaxCaseVal->getValue()))
5527 MaxCaseVal = CaseVal;
5528
5529 // Resulting value at phi nodes for this case value.
5530 using ResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
5531 ResultsTy Results;
5532 if (!GetCaseResults(SI, CaseVal, CI->getCaseSuccessor(), &CommonDest,
5533 Results, DL, TTI))
5534 return false;
5535
5536 // Append the result from this case to the list for each phi.
5537 for (const auto &I : Results) {
5538 PHINode *PHI = I.first;
5539 Constant *Value = I.second;
5540 if (!ResultLists.count(PHI))
5541 PHIs.push_back(PHI);
5542 ResultLists[PHI].push_back(std::make_pair(CaseVal, Value));
5543 }
5544 }
5545
5546 // Keep track of the result types.
5547 for (PHINode *PHI : PHIs) {
5548 ResultTypes[PHI] = ResultLists[PHI][0].second->getType();
5549 }
5550
5551 uint64_t NumResults = ResultLists[PHIs[0]].size();
5552 APInt RangeSpread = MaxCaseVal->getValue() - MinCaseVal->getValue();
5553 uint64_t TableSize = RangeSpread.getLimitedValue() + 1;
5554 bool TableHasHoles = (NumResults < TableSize);
5555
5556 // If the table has holes, we need a constant result for the default case
5557 // or a bitmask that fits in a register.
5558 SmallVector<std::pair<PHINode *, Constant *>, 4> DefaultResultsList;
5559 bool HasDefaultResults =
5560 GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest,
5561 DefaultResultsList, DL, TTI);
5562
5563 bool NeedMask = (TableHasHoles && !HasDefaultResults);
5564 if (NeedMask) {
5565 // As an extra penalty for the validity test we require more cases.
5566 if (SI->getNumCases() < 4) // FIXME: Find best threshold value (benchmark).
5567 return false;
5568 if (!DL.fitsInLegalInteger(TableSize))
5569 return false;
5570 }
5571
5572 for (const auto &I : DefaultResultsList) {
5573 PHINode *PHI = I.first;
5574 Constant *Result = I.second;
5575 DefaultResults[PHI] = Result;
5576 }
5577
5578 if (!ShouldBuildLookupTable(SI, TableSize, TTI, DL, ResultTypes))
5579 return false;
5580
5581 // Create the BB that does the lookups.
5582 Module &Mod = *CommonDest->getParent()->getParent();
5583 BasicBlock *LookupBB = BasicBlock::Create(
5584 Mod.getContext(), "switch.lookup", CommonDest->getParent(), CommonDest);
5585
5586 // Compute the table index value.
5587 Builder.SetInsertPoint(SI);
5588 Value *TableIndex;
5589 if (MinCaseVal->isNullValue())
5590 TableIndex = SI->getCondition();
5591 else
5592 TableIndex = Builder.CreateSub(SI->getCondition(), MinCaseVal,
5593 "switch.tableidx");
5594
5595 // Compute the maximum table size representable by the integer type we are
5596 // switching upon.
5597 unsigned CaseSize = MinCaseVal->getType()->getPrimitiveSizeInBits();
5598 uint64_t MaxTableSize = CaseSize > 63 ? UINT64_MAX(18446744073709551615UL) : 1ULL << CaseSize;
5599 assert(MaxTableSize >= TableSize &&((MaxTableSize >= TableSize && "It is impossible for a switch to have more entries than the max "
"representable value of its input integer type's size.") ? static_cast
<void> (0) : __assert_fail ("MaxTableSize >= TableSize && \"It is impossible for a switch to have more entries than the max \" \"representable value of its input integer type's size.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5601, __PRETTY_FUNCTION__))
5600 "It is impossible for a switch to have more entries than the max "((MaxTableSize >= TableSize && "It is impossible for a switch to have more entries than the max "
"representable value of its input integer type's size.") ? static_cast
<void> (0) : __assert_fail ("MaxTableSize >= TableSize && \"It is impossible for a switch to have more entries than the max \" \"representable value of its input integer type's size.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5601, __PRETTY_FUNCTION__))
5601 "representable value of its input integer type's size.")((MaxTableSize >= TableSize && "It is impossible for a switch to have more entries than the max "
"representable value of its input integer type's size.") ? static_cast
<void> (0) : __assert_fail ("MaxTableSize >= TableSize && \"It is impossible for a switch to have more entries than the max \" \"representable value of its input integer type's size.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5601, __PRETTY_FUNCTION__))
;
5602
5603 // If the default destination is unreachable, or if the lookup table covers
5604 // all values of the conditional variable, branch directly to the lookup table
5605 // BB. Otherwise, check that the condition is within the case range.
5606 const bool DefaultIsReachable =
5607 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
5608 const bool GeneratingCoveredLookupTable = (MaxTableSize == TableSize);
5609 BranchInst *RangeCheckBranch = nullptr;
5610
5611 if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
5612 Builder.CreateBr(LookupBB);
5613 // Note: We call removeProdecessor later since we need to be able to get the
5614 // PHI value for the default case in case we're using a bit mask.
5615 } else {
5616 Value *Cmp = Builder.CreateICmpULT(
5617 TableIndex, ConstantInt::get(MinCaseVal->getType(), TableSize));
5618 RangeCheckBranch =
5619 Builder.CreateCondBr(Cmp, LookupBB, SI->getDefaultDest());
5620 }
5621
5622 // Populate the BB that does the lookups.
5623 Builder.SetInsertPoint(LookupBB);
5624
5625 if (NeedMask) {
5626 // Before doing the lookup, we do the hole check. The LookupBB is therefore
5627 // re-purposed to do the hole check, and we create a new LookupBB.
5628 BasicBlock *MaskBB = LookupBB;
5629 MaskBB->setName("switch.hole_check");
5630 LookupBB = BasicBlock::Create(Mod.getContext(), "switch.lookup",
5631 CommonDest->getParent(), CommonDest);
5632
5633 // Make the mask's bitwidth at least 8-bit and a power-of-2 to avoid
5634 // unnecessary illegal types.
5635 uint64_t TableSizePowOf2 = NextPowerOf2(std::max(7ULL, TableSize - 1ULL));
5636 APInt MaskInt(TableSizePowOf2, 0);
5637 APInt One(TableSizePowOf2, 1);
5638 // Build bitmask; fill in a 1 bit for every case.
5639 const ResultListTy &ResultList = ResultLists[PHIs[0]];
5640 for (size_t I = 0, E = ResultList.size(); I != E; ++I) {
5641 uint64_t Idx = (ResultList[I].first->getValue() - MinCaseVal->getValue())
5642 .getLimitedValue();
5643 MaskInt |= One << Idx;
5644 }
5645 ConstantInt *TableMask = ConstantInt::get(Mod.getContext(), MaskInt);
5646
5647 // Get the TableIndex'th bit of the bitmask.
5648 // If this bit is 0 (meaning hole) jump to the default destination,
5649 // else continue with table lookup.
5650 IntegerType *MapTy = TableMask->getType();
5651 Value *MaskIndex =
5652 Builder.CreateZExtOrTrunc(TableIndex, MapTy, "switch.maskindex");
5653 Value *Shifted = Builder.CreateLShr(TableMask, MaskIndex, "switch.shifted");
5654 Value *LoBit = Builder.CreateTrunc(
5655 Shifted, Type::getInt1Ty(Mod.getContext()), "switch.lobit");
5656 Builder.CreateCondBr(LoBit, LookupBB, SI->getDefaultDest());
5657
5658 Builder.SetInsertPoint(LookupBB);
5659 AddPredecessorToBlock(SI->getDefaultDest(), MaskBB, SI->getParent());
5660 }
5661
5662 if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
5663 // We cached PHINodes in PHIs. To avoid accessing deleted PHINodes later,
5664 // do not delete PHINodes here.
5665 SI->getDefaultDest()->removePredecessor(SI->getParent(),
5666 /*KeepOneInputPHIs=*/true);
5667 }
5668
5669 bool ReturnedEarly = false;
5670 for (PHINode *PHI : PHIs) {
5671 const ResultListTy &ResultList = ResultLists[PHI];
5672
5673 // If using a bitmask, use any value to fill the lookup table holes.
5674 Constant *DV = NeedMask ? ResultLists[PHI][0].second : DefaultResults[PHI];
5675 StringRef FuncName = Fn->getName();
5676 SwitchLookupTable Table(Mod, TableSize, MinCaseVal, ResultList, DV, DL,
5677 FuncName);
5678
5679 Value *Result = Table.BuildLookup(TableIndex, Builder);
5680
5681 // If the result is used to return immediately from the function, we want to
5682 // do that right here.
5683 if (PHI->hasOneUse() && isa<ReturnInst>(*PHI->user_begin()) &&
5684 PHI->user_back() == CommonDest->getFirstNonPHIOrDbg()) {
5685 Builder.CreateRet(Result);
5686 ReturnedEarly = true;
5687 break;
5688 }
5689
5690 // Do a small peephole optimization: re-use the switch table compare if
5691 // possible.
5692 if (!TableHasHoles && HasDefaultResults && RangeCheckBranch) {
5693 BasicBlock *PhiBlock = PHI->getParent();
5694 // Search for compare instructions which use the phi.
5695 for (auto *User : PHI->users()) {
5696 reuseTableCompare(User, PhiBlock, RangeCheckBranch, DV, ResultList);
5697 }
5698 }
5699
5700 PHI->addIncoming(Result, LookupBB);
5701 }
5702
5703 if (!ReturnedEarly)
5704 Builder.CreateBr(CommonDest);
5705
5706 // Remove the switch.
5707 for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
5708 BasicBlock *Succ = SI->getSuccessor(i);
5709
5710 if (Succ == SI->getDefaultDest())
5711 continue;
5712 Succ->removePredecessor(SI->getParent());
5713 }
5714 SI->eraseFromParent();
5715
5716 ++NumLookupTables;
5717 if (NeedMask)
5718 ++NumLookupTablesHoles;
5719 return true;
5720}
5721
5722static bool isSwitchDense(ArrayRef<int64_t> Values) {
5723 // See also SelectionDAGBuilder::isDense(), which this function was based on.
5724 uint64_t Diff = (uint64_t)Values.back() - (uint64_t)Values.front();
5725 uint64_t Range = Diff + 1;
5726 uint64_t NumCases = Values.size();
5727 // 40% is the default density for building a jump table in optsize/minsize mode.
5728 uint64_t MinDensity = 40;
5729
5730 return NumCases * 100 >= Range * MinDensity;
5731}
5732
5733/// Try to transform a switch that has "holes" in it to a contiguous sequence
5734/// of cases.
5735///
5736/// A switch such as: switch(i) {case 5: case 9: case 13: case 17:} can be
5737/// range-reduced to: switch ((i-5) / 4) {case 0: case 1: case 2: case 3:}.
5738///
5739/// This converts a sparse switch into a dense switch which allows better
5740/// lowering and could also allow transforming into a lookup table.
5741static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder,
5742 const DataLayout &DL,
5743 const TargetTransformInfo &TTI) {
5744 auto *CondTy = cast<IntegerType>(SI->getCondition()->getType());
5745 if (CondTy->getIntegerBitWidth() > 64 ||
5746 !DL.fitsInLegalInteger(CondTy->getIntegerBitWidth()))
5747 return false;
5748 // Only bother with this optimization if there are more than 3 switch cases;
5749 // SDAG will only bother creating jump tables for 4 or more cases.
5750 if (SI->getNumCases() < 4)
5751 return false;
5752
5753 // This transform is agnostic to the signedness of the input or case values. We
5754 // can treat the case values as signed or unsigned. We can optimize more common
5755 // cases such as a sequence crossing zero {-4,0,4,8} if we interpret case values
5756 // as signed.
5757 SmallVector<int64_t,4> Values;
5758 for (auto &C : SI->cases())
5759 Values.push_back(C.getCaseValue()->getValue().getSExtValue());
5760 llvm::sort(Values);
5761
5762 // If the switch is already dense, there's nothing useful to do here.
5763 if (isSwitchDense(Values))
5764 return false;
5765
5766 // First, transform the values such that they start at zero and ascend.
5767 int64_t Base = Values[0];
5768 for (auto &V : Values)
5769 V -= (uint64_t)(Base);
5770
5771 // Now we have signed numbers that have been shifted so that, given enough
5772 // precision, there are no negative values. Since the rest of the transform
5773 // is bitwise only, we switch now to an unsigned representation.
5774
5775 // This transform can be done speculatively because it is so cheap - it
5776 // results in a single rotate operation being inserted.
5777 // FIXME: It's possible that optimizing a switch on powers of two might also
5778 // be beneficial - flag values are often powers of two and we could use a CLZ
5779 // as the key function.
5780
5781 // countTrailingZeros(0) returns 64. As Values is guaranteed to have more than
5782 // one element and LLVM disallows duplicate cases, Shift is guaranteed to be
5783 // less than 64.
5784 unsigned Shift = 64;
5785 for (auto &V : Values)
5786 Shift = std::min(Shift, countTrailingZeros((uint64_t)V));
5787 assert(Shift < 64)((Shift < 64) ? static_cast<void> (0) : __assert_fail
("Shift < 64", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5787, __PRETTY_FUNCTION__))
;
5788 if (Shift > 0)
5789 for (auto &V : Values)
5790 V = (int64_t)((uint64_t)V >> Shift);
5791
5792 if (!isSwitchDense(Values))
5793 // Transform didn't create a dense switch.
5794 return false;
5795
5796 // The obvious transform is to shift the switch condition right and emit a
5797 // check that the condition actually cleanly divided by GCD, i.e.
5798 // C & (1 << Shift - 1) == 0
5799 // inserting a new CFG edge to handle the case where it didn't divide cleanly.
5800 //
5801 // A cheaper way of doing this is a simple ROTR(C, Shift). This performs the
5802 // shift and puts the shifted-off bits in the uppermost bits. If any of these
5803 // are nonzero then the switch condition will be very large and will hit the
5804 // default case.
5805
5806 auto *Ty = cast<IntegerType>(SI->getCondition()->getType());
5807 Builder.SetInsertPoint(SI);
5808 auto *ShiftC = ConstantInt::get(Ty, Shift);
5809 auto *Sub = Builder.CreateSub(SI->getCondition(), ConstantInt::get(Ty, Base));
5810 auto *LShr = Builder.CreateLShr(Sub, ShiftC);
5811 auto *Shl = Builder.CreateShl(Sub, Ty->getBitWidth() - Shift);
5812 auto *Rot = Builder.CreateOr(LShr, Shl);
5813 SI->replaceUsesOfWith(SI->getCondition(), Rot);
5814
5815 for (auto Case : SI->cases()) {
5816 auto *Orig = Case.getCaseValue();
5817 auto Sub = Orig->getValue() - APInt(Ty->getBitWidth(), Base);
5818 Case.setValue(
5819 cast<ConstantInt>(ConstantInt::get(Ty, Sub.lshr(ShiftC->getValue()))));
5820 }
5821 return true;
5822}
5823
5824bool SimplifyCFGOpt::simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
5825 BasicBlock *BB = SI->getParent();
5826
5827 if (isValueEqualityComparison(SI)) {
5828 // If we only have one predecessor, and if it is a branch on this value,
5829 // see if that predecessor totally determines the outcome of this switch.
5830 if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
5831 if (SimplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder))
5832 return requestResimplify();
5833
5834 Value *Cond = SI->getCondition();
5835 if (SelectInst *Select = dyn_cast<SelectInst>(Cond))
5836 if (SimplifySwitchOnSelect(SI, Select))
5837 return requestResimplify();
5838
5839 // If the block only contains the switch, see if we can fold the block
5840 // away into any preds.
5841 if (SI == &*BB->instructionsWithoutDebug().begin())
5842 if (FoldValueComparisonIntoPredecessors(SI, Builder))
5843 return requestResimplify();
5844 }
5845
5846 // Try to transform the switch into an icmp and a branch.
5847 if (TurnSwitchRangeIntoICmp(SI, Builder))
5848 return requestResimplify();
5849
5850 // Remove unreachable cases.
5851 if (eliminateDeadSwitchCases(SI, Options.AC, DL))
5852 return requestResimplify();
5853