Bug Summary

File:llvm/lib/Transforms/Utils/SimplifyCFG.cpp
Warning:line 3106, column 11
3rd function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SimplifyCFG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/build-llvm/lib/Transforms/Utils -I /build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils -I /build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/build-llvm/lib/Transforms/Utils -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2021-01-22-054259-40355-1 -x c++ /build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp

/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp

1//===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Peephole optimize the CFG.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APInt.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/Optional.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/Sequence.h"
21#include "llvm/ADT/SetOperations.h"
22#include "llvm/ADT/SetVector.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/Analysis/AssumptionCache.h"
28#include "llvm/Analysis/ConstantFolding.h"
29#include "llvm/Analysis/EHPersonalities.h"
30#include "llvm/Analysis/GuardUtils.h"
31#include "llvm/Analysis/InstructionSimplify.h"
32#include "llvm/Analysis/MemorySSA.h"
33#include "llvm/Analysis/MemorySSAUpdater.h"
34#include "llvm/Analysis/TargetTransformInfo.h"
35#include "llvm/Analysis/ValueTracking.h"
36#include "llvm/IR/Attributes.h"
37#include "llvm/IR/BasicBlock.h"
38#include "llvm/IR/CFG.h"
39#include "llvm/IR/Constant.h"
40#include "llvm/IR/ConstantRange.h"
41#include "llvm/IR/Constants.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/DerivedTypes.h"
44#include "llvm/IR/Function.h"
45#include "llvm/IR/GlobalValue.h"
46#include "llvm/IR/GlobalVariable.h"
47#include "llvm/IR/IRBuilder.h"
48#include "llvm/IR/InstrTypes.h"
49#include "llvm/IR/Instruction.h"
50#include "llvm/IR/Instructions.h"
51#include "llvm/IR/IntrinsicInst.h"
52#include "llvm/IR/Intrinsics.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/MDBuilder.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/NoFolder.h"
58#include "llvm/IR/Operator.h"
59#include "llvm/IR/PatternMatch.h"
60#include "llvm/IR/Type.h"
61#include "llvm/IR/Use.h"
62#include "llvm/IR/User.h"
63#include "llvm/IR/Value.h"
64#include "llvm/Support/Casting.h"
65#include "llvm/Support/CommandLine.h"
66#include "llvm/Support/Debug.h"
67#include "llvm/Support/ErrorHandling.h"
68#include "llvm/Support/KnownBits.h"
69#include "llvm/Support/MathExtras.h"
70#include "llvm/Support/raw_ostream.h"
71#include "llvm/Transforms/Utils/BasicBlockUtils.h"
72#include "llvm/Transforms/Utils/Local.h"
73#include "llvm/Transforms/Utils/SSAUpdater.h"
74#include "llvm/Transforms/Utils/ValueMapper.h"
75#include <algorithm>
76#include <cassert>
77#include <climits>
78#include <cstddef>
79#include <cstdint>
80#include <iterator>
81#include <map>
82#include <set>
83#include <tuple>
84#include <utility>
85#include <vector>
86
87using namespace llvm;
88using namespace PatternMatch;
89
90#define DEBUG_TYPE"simplifycfg" "simplifycfg"
91
92cl::opt<bool> llvm::RequireAndPreserveDomTree(
93 "simplifycfg-require-and-preserve-domtree", cl::Hidden, cl::ZeroOrMore,
94 cl::init(false),
95 cl::desc("Temorary development switch used to gradually uplift SimplifyCFG "
96 "into preserving DomTree,"));
97
98// Chosen as 2 so as to be cheap, but still to have enough power to fold
99// a select, so the "clamp" idiom (of a min followed by a max) will be caught.
100// To catch this, we need to fold a compare and a select, hence '2' being the
101// minimum reasonable default.
102static cl::opt<unsigned> PHINodeFoldingThreshold(
103 "phi-node-folding-threshold", cl::Hidden, cl::init(2),
104 cl::desc(
105 "Control the amount of phi node folding to perform (default = 2)"));
106
107static cl::opt<unsigned> TwoEntryPHINodeFoldingThreshold(
108 "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4),
109 cl::desc("Control the maximal total instruction cost that we are willing "
110 "to speculatively execute to fold a 2-entry PHI node into a "
111 "select (default = 4)"));
112
113static cl::opt<bool> DupRet(
114 "simplifycfg-dup-ret", cl::Hidden, cl::init(false),
115 cl::desc("Duplicate return instructions into unconditional branches"));
116
117static cl::opt<bool>
118 HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true),
119 cl::desc("Hoist common instructions up to the parent block"));
120
121static cl::opt<bool>
122 SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
123 cl::desc("Sink common instructions down to the end block"));
124
125static cl::opt<bool> HoistCondStores(
126 "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true),
127 cl::desc("Hoist conditional stores if an unconditional store precedes"));
128
129static cl::opt<bool> MergeCondStores(
130 "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true),
131 cl::desc("Hoist conditional stores even if an unconditional store does not "
132 "precede - hoist multiple conditional stores into a single "
133 "predicated store"));
134
135static cl::opt<bool> MergeCondStoresAggressively(
136 "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false),
137 cl::desc("When merging conditional stores, do so even if the resultant "
138 "basic blocks are unlikely to be if-converted as a result"));
139
140static cl::opt<bool> SpeculateOneExpensiveInst(
141 "speculate-one-expensive-inst", cl::Hidden, cl::init(true),
142 cl::desc("Allow exactly one expensive instruction to be speculatively "
143 "executed"));
144
145static cl::opt<unsigned> MaxSpeculationDepth(
146 "max-speculation-depth", cl::Hidden, cl::init(10),
147 cl::desc("Limit maximum recursion depth when calculating costs of "
148 "speculatively executed instructions"));
149
150static cl::opt<int>
151MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden, cl::init(10),
152 cl::desc("Max size of a block which is still considered "
153 "small enough to thread through"));
154
155// Two is chosen to allow one negation and a logical combine.
156static cl::opt<unsigned>
157 BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden,
158 cl::init(2),
159 cl::desc("Maximum cost of combining conditions when "
160 "folding branches"));
161
162STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps")static llvm::Statistic NumBitMaps = {"simplifycfg", "NumBitMaps"
, "Number of switch instructions turned into bitmaps"}
;
163STATISTIC(NumLinearMaps,static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps"
, "Number of switch instructions turned into linear mapping"}
164 "Number of switch instructions turned into linear mapping")static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps"
, "Number of switch instructions turned into linear mapping"}
;
165STATISTIC(NumLookupTables,static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables"
, "Number of switch instructions turned into lookup tables"}
166 "Number of switch instructions turned into lookup tables")static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables"
, "Number of switch instructions turned into lookup tables"}
;
167STATISTIC(static llvm::Statistic NumLookupTablesHoles = {"simplifycfg",
"NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)"
}
168 NumLookupTablesHoles,static llvm::Statistic NumLookupTablesHoles = {"simplifycfg",
"NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)"
}
169 "Number of switch instructions turned into lookup tables (holes checked)")static llvm::Statistic NumLookupTablesHoles = {"simplifycfg",
"NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)"
}
;
170STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares")static llvm::Statistic NumTableCmpReuses = {"simplifycfg", "NumTableCmpReuses"
, "Number of reused switch table lookup compares"}
;
171STATISTIC(NumFoldValueComparisonIntoPredecessors,static llvm::Statistic NumFoldValueComparisonIntoPredecessors
= {"simplifycfg", "NumFoldValueComparisonIntoPredecessors", "Number of value comparisons folded into predecessor basic blocks"
}
172 "Number of value comparisons folded into predecessor basic blocks")static llvm::Statistic NumFoldValueComparisonIntoPredecessors
= {"simplifycfg", "NumFoldValueComparisonIntoPredecessors", "Number of value comparisons folded into predecessor basic blocks"
}
;
173STATISTIC(NumFoldBranchToCommonDest,static llvm::Statistic NumFoldBranchToCommonDest = {"simplifycfg"
, "NumFoldBranchToCommonDest", "Number of branches folded into predecessor basic block"
}
174 "Number of branches folded into predecessor basic block")static llvm::Statistic NumFoldBranchToCommonDest = {"simplifycfg"
, "NumFoldBranchToCommonDest", "Number of branches folded into predecessor basic block"
}
;
175STATISTIC(static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode"
, "Number of common instruction 'blocks' hoisted up to the begin block"
}
176 NumHoistCommonCode,static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode"
, "Number of common instruction 'blocks' hoisted up to the begin block"
}
177 "Number of common instruction 'blocks' hoisted up to the begin block")static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode"
, "Number of common instruction 'blocks' hoisted up to the begin block"
}
;
178STATISTIC(NumHoistCommonInstrs,static llvm::Statistic NumHoistCommonInstrs = {"simplifycfg",
"NumHoistCommonInstrs", "Number of common instructions hoisted up to the begin block"
}
179 "Number of common instructions hoisted up to the begin block")static llvm::Statistic NumHoistCommonInstrs = {"simplifycfg",
"NumHoistCommonInstrs", "Number of common instructions hoisted up to the begin block"
}
;
180STATISTIC(NumSinkCommonCode,static llvm::Statistic NumSinkCommonCode = {"simplifycfg", "NumSinkCommonCode"
, "Number of common instruction 'blocks' sunk down to the end block"
}
181 "Number of common instruction 'blocks' sunk down to the end block")static llvm::Statistic NumSinkCommonCode = {"simplifycfg", "NumSinkCommonCode"
, "Number of common instruction 'blocks' sunk down to the end block"
}
;
182STATISTIC(NumSinkCommonInstrs,static llvm::Statistic NumSinkCommonInstrs = {"simplifycfg", "NumSinkCommonInstrs"
, "Number of common instructions sunk down to the end block"}
183 "Number of common instructions sunk down to the end block")static llvm::Statistic NumSinkCommonInstrs = {"simplifycfg", "NumSinkCommonInstrs"
, "Number of common instructions sunk down to the end block"}
;
184STATISTIC(NumSpeculations, "Number of speculative executed instructions")static llvm::Statistic NumSpeculations = {"simplifycfg", "NumSpeculations"
, "Number of speculative executed instructions"}
;
185STATISTIC(NumInvokes,static llvm::Statistic NumInvokes = {"simplifycfg", "NumInvokes"
, "Number of invokes with empty resume blocks simplified into calls"
}
186 "Number of invokes with empty resume blocks simplified into calls")static llvm::Statistic NumInvokes = {"simplifycfg", "NumInvokes"
, "Number of invokes with empty resume blocks simplified into calls"
}
;
187
188namespace {
189
190// The first field contains the value that the switch produces when a certain
191// case group is selected, and the second field is a vector containing the
192// cases composing the case group.
193using SwitchCaseResultVectorTy =
194 SmallVector<std::pair<Constant *, SmallVector<ConstantInt *, 4>>, 2>;
195
196// The first field contains the phi node that generates a result of the switch
197// and the second field contains the value generated for a certain case in the
198// switch for that PHI.
199using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
200
201/// ValueEqualityComparisonCase - Represents a case of a switch.
202struct ValueEqualityComparisonCase {
203 ConstantInt *Value;
204 BasicBlock *Dest;
205
206 ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest)
207 : Value(Value), Dest(Dest) {}
208
209 bool operator<(ValueEqualityComparisonCase RHS) const {
210 // Comparing pointers is ok as we only rely on the order for uniquing.
211 return Value < RHS.Value;
212 }
213
214 bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; }
215};
216
217class SimplifyCFGOpt {
218 const TargetTransformInfo &TTI;
219 DomTreeUpdater *DTU;
220 const DataLayout &DL;
221 SmallPtrSetImpl<BasicBlock *> *LoopHeaders;
222 const SimplifyCFGOptions &Options;
223 bool Resimplify;
224
225 Value *isValueEqualityComparison(Instruction *TI);
226 BasicBlock *GetValueEqualityComparisonCases(
227 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases);
228 bool SimplifyEqualityComparisonWithOnlyPredecessor(Instruction *TI,
229 BasicBlock *Pred,
230 IRBuilder<> &Builder);
231 bool FoldValueComparisonIntoPredecessors(Instruction *TI,
232 IRBuilder<> &Builder);
233
234 bool simplifyReturn(ReturnInst *RI, IRBuilder<> &Builder);
235 bool simplifyResume(ResumeInst *RI, IRBuilder<> &Builder);
236 bool simplifySingleResume(ResumeInst *RI);
237 bool simplifyCommonResume(ResumeInst *RI);
238 bool simplifyCleanupReturn(CleanupReturnInst *RI);
239 bool simplifyUnreachable(UnreachableInst *UI);
240 bool simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder);
241 bool simplifyIndirectBr(IndirectBrInst *IBI);
242 bool simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder);
243 bool simplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder);
244 bool simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder);
245 bool SimplifyCondBranchToTwoReturns(BranchInst *BI, IRBuilder<> &Builder);
246
247 bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
248 IRBuilder<> &Builder);
249
250 bool HoistThenElseCodeToIf(BranchInst *BI, const TargetTransformInfo &TTI);
251 bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
252 const TargetTransformInfo &TTI);
253 bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
254 BasicBlock *TrueBB, BasicBlock *FalseBB,
255 uint32_t TrueWeight, uint32_t FalseWeight);
256 bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder,
257 const DataLayout &DL);
258 bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select);
259 bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI);
260 bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder);
261
262public:
263 SimplifyCFGOpt(const TargetTransformInfo &TTI, DomTreeUpdater *DTU,
264 const DataLayout &DL,
265 SmallPtrSetImpl<BasicBlock *> *LoopHeaders,
266 const SimplifyCFGOptions &Opts)
267 : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) {
268 assert((!DTU || !DTU->hasPostDomTree()) &&(((!DTU || !DTU->hasPostDomTree()) && "SimplifyCFG is not yet capable of maintaining validity of a "
"PostDomTree, so don't ask for it.") ? static_cast<void>
(0) : __assert_fail ("(!DTU || !DTU->hasPostDomTree()) && \"SimplifyCFG is not yet capable of maintaining validity of a \" \"PostDomTree, so don't ask for it.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 270, __PRETTY_FUNCTION__))
269 "SimplifyCFG is not yet capable of maintaining validity of a "(((!DTU || !DTU->hasPostDomTree()) && "SimplifyCFG is not yet capable of maintaining validity of a "
"PostDomTree, so don't ask for it.") ? static_cast<void>
(0) : __assert_fail ("(!DTU || !DTU->hasPostDomTree()) && \"SimplifyCFG is not yet capable of maintaining validity of a \" \"PostDomTree, so don't ask for it.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 270, __PRETTY_FUNCTION__))
270 "PostDomTree, so don't ask for it.")(((!DTU || !DTU->hasPostDomTree()) && "SimplifyCFG is not yet capable of maintaining validity of a "
"PostDomTree, so don't ask for it.") ? static_cast<void>
(0) : __assert_fail ("(!DTU || !DTU->hasPostDomTree()) && \"SimplifyCFG is not yet capable of maintaining validity of a \" \"PostDomTree, so don't ask for it.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 270, __PRETTY_FUNCTION__))
;
271 }
272
273 bool simplifyOnce(BasicBlock *BB);
274 bool simplifyOnceImpl(BasicBlock *BB);
275 bool run(BasicBlock *BB);
276
277 // Helper to set Resimplify and return change indication.
278 bool requestResimplify() {
279 Resimplify = true;
280 return true;
281 }
282};
283
284} // end anonymous namespace
285
286/// Return true if it is safe to merge these two
287/// terminator instructions together.
288static bool
289SafeToMergeTerminators(Instruction *SI1, Instruction *SI2,
290 SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) {
291 if (SI1 == SI2)
50
Assuming 'SI1' is not equal to 'SI2'
51
Taking false branch
292 return false; // Can't merge with self!
293
294 // It is not safe to merge these two switch instructions if they have a common
295 // successor, and if that successor has a PHI node, and if *that* PHI node has
296 // conflicting incoming values from the two switch blocks.
297 BasicBlock *SI1BB = SI1->getParent();
298 BasicBlock *SI2BB = SI2->getParent();
299
300 SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
301 bool Fail = false;
302 for (BasicBlock *Succ : successors(SI2BB))
303 if (SI1Succs.count(Succ))
304 for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) {
305 PHINode *PN = cast<PHINode>(BBI);
306 if (PN->getIncomingValueForBlock(SI1BB) !=
307 PN->getIncomingValueForBlock(SI2BB)) {
308 if (FailBlocks)
309 FailBlocks->insert(Succ);
310 Fail = true;
311 }
312 }
313
314 return !Fail;
52
Returning the value 1, which participates in a condition later
315}
316
317/// Return true if it is safe and profitable to merge these two terminator
318/// instructions together, where SI1 is an unconditional branch. PhiNodes will
319/// store all PHI nodes in common successors.
320static bool
321isProfitableToFoldUnconditional(BranchInst *SI1, BranchInst *SI2,
322 Instruction *Cond,
323 SmallVectorImpl<PHINode *> &PhiNodes) {
324 if (SI1 == SI2)
325 return false; // Can't merge with self!
326 assert(SI1->isUnconditional() && SI2->isConditional())((SI1->isUnconditional() && SI2->isConditional(
)) ? static_cast<void> (0) : __assert_fail ("SI1->isUnconditional() && SI2->isConditional()"
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 326, __PRETTY_FUNCTION__))
;
327
328 // We fold the unconditional branch if we can easily update all PHI nodes in
329 // common successors:
330 // 1> We have a constant incoming value for the conditional branch;
331 // 2> We have "Cond" as the incoming value for the unconditional branch;
332 // 3> SI2->getCondition() and Cond have same operands.
333 CmpInst *Ci2 = dyn_cast<CmpInst>(SI2->getCondition());
334 if (!Ci2)
335 return false;
336 if (!(Cond->getOperand(0) == Ci2->getOperand(0) &&
337 Cond->getOperand(1) == Ci2->getOperand(1)) &&
338 !(Cond->getOperand(0) == Ci2->getOperand(1) &&
339 Cond->getOperand(1) == Ci2->getOperand(0)))
340 return false;
341
342 BasicBlock *SI1BB = SI1->getParent();
343 BasicBlock *SI2BB = SI2->getParent();
344 SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
345 for (BasicBlock *Succ : successors(SI2BB))
346 if (SI1Succs.count(Succ))
347 for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) {
348 PHINode *PN = cast<PHINode>(BBI);
349 if (PN->getIncomingValueForBlock(SI1BB) != Cond ||
350 !isa<ConstantInt>(PN->getIncomingValueForBlock(SI2BB)))
351 return false;
352 PhiNodes.push_back(PN);
353 }
354 return true;
355}
356
357/// Update PHI nodes in Succ to indicate that there will now be entries in it
358/// from the 'NewPred' block. The values that will be flowing into the PHI nodes
359/// will be the same as those coming in from ExistPred, an existing predecessor
360/// of Succ.
361static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
362 BasicBlock *ExistPred,
363 MemorySSAUpdater *MSSAU = nullptr) {
364 for (PHINode &PN : Succ->phis())
365 PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred);
366 if (MSSAU)
367 if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ))
368 MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred);
369}
370
371/// Compute an abstract "cost" of speculating the given instruction,
372/// which is assumed to be safe to speculate. TCC_Free means cheap,
373/// TCC_Basic means less cheap, and TCC_Expensive means prohibitively
374/// expensive.
375static unsigned ComputeSpeculationCost(const User *I,
376 const TargetTransformInfo &TTI) {
377 assert(isSafeToSpeculativelyExecute(I) &&((isSafeToSpeculativelyExecute(I) && "Instruction is not safe to speculatively execute!"
) ? static_cast<void> (0) : __assert_fail ("isSafeToSpeculativelyExecute(I) && \"Instruction is not safe to speculatively execute!\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 378, __PRETTY_FUNCTION__))
378 "Instruction is not safe to speculatively execute!")((isSafeToSpeculativelyExecute(I) && "Instruction is not safe to speculatively execute!"
) ? static_cast<void> (0) : __assert_fail ("isSafeToSpeculativelyExecute(I) && \"Instruction is not safe to speculatively execute!\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 378, __PRETTY_FUNCTION__))
;
379 return TTI.getUserCost(I, TargetTransformInfo::TCK_SizeAndLatency);
380}
381
382/// If we have a merge point of an "if condition" as accepted above,
383/// return true if the specified value dominates the block. We
384/// don't handle the true generality of domination here, just a special case
385/// which works well enough for us.
386///
387/// If AggressiveInsts is non-null, and if V does not dominate BB, we check to
388/// see if V (which must be an instruction) and its recursive operands
389/// that do not dominate BB have a combined cost lower than CostRemaining and
390/// are non-trapping. If both are true, the instruction is inserted into the
391/// set and true is returned.
392///
393/// The cost for most non-trapping instructions is defined as 1 except for
394/// Select whose cost is 2.
395///
396/// After this function returns, CostRemaining is decreased by the cost of
397/// V plus its non-dominating operands. If that cost is greater than
398/// CostRemaining, false is returned and CostRemaining is undefined.
399static bool DominatesMergePoint(Value *V, BasicBlock *BB,
400 SmallPtrSetImpl<Instruction *> &AggressiveInsts,
401 int &BudgetRemaining,
402 const TargetTransformInfo &TTI,
403 unsigned Depth = 0) {
404 // It is possible to hit a zero-cost cycle (phi/gep instructions for example),
405 // so limit the recursion depth.
406 // TODO: While this recursion limit does prevent pathological behavior, it
407 // would be better to track visited instructions to avoid cycles.
408 if (Depth == MaxSpeculationDepth)
409 return false;
410
411 Instruction *I = dyn_cast<Instruction>(V);
412 if (!I) {
413 // Non-instructions all dominate instructions, but not all constantexprs
414 // can be executed unconditionally.
415 if (ConstantExpr *C = dyn_cast<ConstantExpr>(V))
416 if (C->canTrap())
417 return false;
418 return true;
419 }
420 BasicBlock *PBB = I->getParent();
421
422 // We don't want to allow weird loops that might have the "if condition" in
423 // the bottom of this block.
424 if (PBB == BB)
425 return false;
426
427 // If this instruction is defined in a block that contains an unconditional
428 // branch to BB, then it must be in the 'conditional' part of the "if
429 // statement". If not, it definitely dominates the region.
430 BranchInst *BI = dyn_cast<BranchInst>(PBB->getTerminator());
431 if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB)
432 return true;
433
434 // If we have seen this instruction before, don't count it again.
435 if (AggressiveInsts.count(I))
436 return true;
437
438 // Okay, it looks like the instruction IS in the "condition". Check to
439 // see if it's a cheap instruction to unconditionally compute, and if it
440 // only uses stuff defined outside of the condition. If so, hoist it out.
441 if (!isSafeToSpeculativelyExecute(I))
442 return false;
443
444 BudgetRemaining -= ComputeSpeculationCost(I, TTI);
445
446 // Allow exactly one instruction to be speculated regardless of its cost
447 // (as long as it is safe to do so).
448 // This is intended to flatten the CFG even if the instruction is a division
449 // or other expensive operation. The speculation of an expensive instruction
450 // is expected to be undone in CodeGenPrepare if the speculation has not
451 // enabled further IR optimizations.
452 if (BudgetRemaining < 0 &&
453 (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0))
454 return false;
455
456 // Okay, we can only really hoist these out if their operands do
457 // not take us over the cost threshold.
458 for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i)
459 if (!DominatesMergePoint(*i, BB, AggressiveInsts, BudgetRemaining, TTI,
460 Depth + 1))
461 return false;
462 // Okay, it's safe to do this! Remember this instruction.
463 AggressiveInsts.insert(I);
464 return true;
465}
466
467/// Extract ConstantInt from value, looking through IntToPtr
468/// and PointerNullValue. Return NULL if value is not a constant int.
469static ConstantInt *GetConstantInt(Value *V, const DataLayout &DL) {
470 // Normal constant int.
471 ConstantInt *CI = dyn_cast<ConstantInt>(V);
472 if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy())
473 return CI;
474
475 // This is some kind of pointer constant. Turn it into a pointer-sized
476 // ConstantInt if possible.
477 IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType()));
478
479 // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
480 if (isa<ConstantPointerNull>(V))
481 return ConstantInt::get(PtrTy, 0);
482
483 // IntToPtr const int.
484 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
485 if (CE->getOpcode() == Instruction::IntToPtr)
486 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) {
487 // The constant is very likely to have the right type already.
488 if (CI->getType() == PtrTy)
489 return CI;
490 else
491 return cast<ConstantInt>(
492 ConstantExpr::getIntegerCast(CI, PtrTy, /*isSigned=*/false));
493 }
494 return nullptr;
495}
496
497namespace {
498
499/// Given a chain of or (||) or and (&&) comparison of a value against a
500/// constant, this will try to recover the information required for a switch
501/// structure.
502/// It will depth-first traverse the chain of comparison, seeking for patterns
503/// like %a == 12 or %a < 4 and combine them to produce a set of integer
504/// representing the different cases for the switch.
505/// Note that if the chain is composed of '||' it will build the set of elements
506/// that matches the comparisons (i.e. any of this value validate the chain)
507/// while for a chain of '&&' it will build the set elements that make the test
508/// fail.
509struct ConstantComparesGatherer {
510 const DataLayout &DL;
511
512 /// Value found for the switch comparison
513 Value *CompValue = nullptr;
514
515 /// Extra clause to be checked before the switch
516 Value *Extra = nullptr;
517
518 /// Set of integers to match in switch
519 SmallVector<ConstantInt *, 8> Vals;
520
521 /// Number of comparisons matched in the and/or chain
522 unsigned UsedICmps = 0;
523
524 /// Construct and compute the result for the comparison instruction Cond
525 ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) {
526 gather(Cond);
527 }
528
529 ConstantComparesGatherer(const ConstantComparesGatherer &) = delete;
530 ConstantComparesGatherer &
531 operator=(const ConstantComparesGatherer &) = delete;
532
533private:
534 /// Try to set the current value used for the comparison, it succeeds only if
535 /// it wasn't set before or if the new value is the same as the old one
536 bool setValueOnce(Value *NewVal) {
537 if (CompValue && CompValue != NewVal)
538 return false;
539 CompValue = NewVal;
540 return (CompValue != nullptr);
541 }
542
543 /// Try to match Instruction "I" as a comparison against a constant and
544 /// populates the array Vals with the set of values that match (or do not
545 /// match depending on isEQ).
546 /// Return false on failure. On success, the Value the comparison matched
547 /// against is placed in CompValue.
548 /// If CompValue is already set, the function is expected to fail if a match
549 /// is found but the value compared to is different.
550 bool matchInstruction(Instruction *I, bool isEQ) {
551 // If this is an icmp against a constant, handle this as one of the cases.
552 ICmpInst *ICI;
553 ConstantInt *C;
554 if (!((ICI = dyn_cast<ICmpInst>(I)) &&
555 (C = GetConstantInt(I->getOperand(1), DL)))) {
556 return false;
557 }
558
559 Value *RHSVal;
560 const APInt *RHSC;
561
562 // Pattern match a special case
563 // (x & ~2^z) == y --> x == y || x == y|2^z
564 // This undoes a transformation done by instcombine to fuse 2 compares.
565 if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
566 // It's a little bit hard to see why the following transformations are
567 // correct. Here is a CVC3 program to verify them for 64-bit values:
568
569 /*
570 ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63);
571 x : BITVECTOR(64);
572 y : BITVECTOR(64);
573 z : BITVECTOR(64);
574 mask : BITVECTOR(64) = BVSHL(ONE, z);
575 QUERY( (y & ~mask = y) =>
576 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
577 );
578 QUERY( (y | mask = y) =>
579 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
580 );
581 */
582
583 // Please note that each pattern must be a dual implication (<--> or
584 // iff). One directional implication can create spurious matches. If the
585 // implication is only one-way, an unsatisfiable condition on the left
586 // side can imply a satisfiable condition on the right side. Dual
587 // implication ensures that satisfiable conditions are transformed to
588 // other satisfiable conditions and unsatisfiable conditions are
589 // transformed to other unsatisfiable conditions.
590
591 // Here is a concrete example of a unsatisfiable condition on the left
592 // implying a satisfiable condition on the right:
593 //
594 // mask = (1 << z)
595 // (x & ~mask) == y --> (x == y || x == (y | mask))
596 //
597 // Substituting y = 3, z = 0 yields:
598 // (x & -2) == 3 --> (x == 3 || x == 2)
599
600 // Pattern match a special case:
601 /*
602 QUERY( (y & ~mask = y) =>
603 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
604 );
605 */
606 if (match(ICI->getOperand(0),
607 m_And(m_Value(RHSVal), m_APInt(RHSC)))) {
608 APInt Mask = ~*RHSC;
609 if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) {
610 // If we already have a value for the switch, it has to match!
611 if (!setValueOnce(RHSVal))
612 return false;
613
614 Vals.push_back(C);
615 Vals.push_back(
616 ConstantInt::get(C->getContext(),
617 C->getValue() | Mask));
618 UsedICmps++;
619 return true;
620 }
621 }
622
623 // Pattern match a special case:
624 /*
625 QUERY( (y | mask = y) =>
626 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
627 );
628 */
629 if (match(ICI->getOperand(0),
630 m_Or(m_Value(RHSVal), m_APInt(RHSC)))) {
631 APInt Mask = *RHSC;
632 if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) {
633 // If we already have a value for the switch, it has to match!
634 if (!setValueOnce(RHSVal))
635 return false;
636
637 Vals.push_back(C);
638 Vals.push_back(ConstantInt::get(C->getContext(),
639 C->getValue() & ~Mask));
640 UsedICmps++;
641 return true;
642 }
643 }
644
645 // If we already have a value for the switch, it has to match!
646 if (!setValueOnce(ICI->getOperand(0)))
647 return false;
648
649 UsedICmps++;
650 Vals.push_back(C);
651 return ICI->getOperand(0);
652 }
653
654 // If we have "x ult 3", for example, then we can add 0,1,2 to the set.
655 ConstantRange Span = ConstantRange::makeAllowedICmpRegion(
656 ICI->getPredicate(), C->getValue());
657
658 // Shift the range if the compare is fed by an add. This is the range
659 // compare idiom as emitted by instcombine.
660 Value *CandidateVal = I->getOperand(0);
661 if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) {
662 Span = Span.subtract(*RHSC);
663 CandidateVal = RHSVal;
664 }
665
666 // If this is an and/!= check, then we are looking to build the set of
667 // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into
668 // x != 0 && x != 1.
669 if (!isEQ)
670 Span = Span.inverse();
671
672 // If there are a ton of values, we don't want to make a ginormous switch.
673 if (Span.isSizeLargerThan(8) || Span.isEmptySet()) {
674 return false;
675 }
676
677 // If we already have a value for the switch, it has to match!
678 if (!setValueOnce(CandidateVal))
679 return false;
680
681 // Add all values from the range to the set
682 for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp)
683 Vals.push_back(ConstantInt::get(I->getContext(), Tmp));
684
685 UsedICmps++;
686 return true;
687 }
688
689 /// Given a potentially 'or'd or 'and'd together collection of icmp
690 /// eq/ne/lt/gt instructions that compare a value against a constant, extract
691 /// the value being compared, and stick the list constants into the Vals
692 /// vector.
693 /// One "Extra" case is allowed to differ from the other.
694 void gather(Value *V) {
695 bool isEQ = match(V, m_LogicalOr(m_Value(), m_Value()));
696
697 // Keep a stack (SmallVector for efficiency) for depth-first traversal
698 SmallVector<Value *, 8> DFT;
699 SmallPtrSet<Value *, 8> Visited;
700
701 // Initialize
702 Visited.insert(V);
703 DFT.push_back(V);
704
705 while (!DFT.empty()) {
706 V = DFT.pop_back_val();
707
708 if (Instruction *I = dyn_cast<Instruction>(V)) {
709 // If it is a || (or && depending on isEQ), process the operands.
710 Value *Op0, *Op1;
711 if (isEQ ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1)))
712 : match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
713 if (Visited.insert(Op1).second)
714 DFT.push_back(Op1);
715 if (Visited.insert(Op0).second)
716 DFT.push_back(Op0);
717
718 continue;
719 }
720
721 // Try to match the current instruction
722 if (matchInstruction(I, isEQ))
723 // Match succeed, continue the loop
724 continue;
725 }
726
727 // One element of the sequence of || (or &&) could not be match as a
728 // comparison against the same value as the others.
729 // We allow only one "Extra" case to be checked before the switch
730 if (!Extra) {
731 Extra = V;
732 continue;
733 }
734 // Failed to parse a proper sequence, abort now
735 CompValue = nullptr;
736 break;
737 }
738 }
739};
740
741} // end anonymous namespace
742
743static void EraseTerminatorAndDCECond(Instruction *TI,
744 MemorySSAUpdater *MSSAU = nullptr) {
745 Instruction *Cond = nullptr;
746 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
747 Cond = dyn_cast<Instruction>(SI->getCondition());
748 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
749 if (BI->isConditional())
750 Cond = dyn_cast<Instruction>(BI->getCondition());
751 } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) {
752 Cond = dyn_cast<Instruction>(IBI->getAddress());
753 }
754
755 TI->eraseFromParent();
756 if (Cond)
757 RecursivelyDeleteTriviallyDeadInstructions(Cond, nullptr, MSSAU);
758}
759
760/// Return true if the specified terminator checks
761/// to see if a value is equal to constant integer value.
762Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) {
763 Value *CV = nullptr;
764 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
765 // Do not permit merging of large switch instructions into their
766 // predecessors unless there is only one predecessor.
767 if (!SI->getParent()->hasNPredecessorsOrMore(128 / SI->getNumSuccessors()))
768 CV = SI->getCondition();
769 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI))
770 if (BI->isConditional() && BI->getCondition()->hasOneUse())
771 if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) {
772 if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL))
773 CV = ICI->getOperand(0);
774 }
775
776 // Unwrap any lossless ptrtoint cast.
777 if (CV) {
778 if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) {
779 Value *Ptr = PTII->getPointerOperand();
780 if (PTII->getType() == DL.getIntPtrType(Ptr->getType()))
781 CV = Ptr;
782 }
783 }
784 return CV;
785}
786
787/// Given a value comparison instruction,
788/// decode all of the 'cases' that it represents and return the 'default' block.
789BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases(
790 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases) {
791 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
792 Cases.reserve(SI->getNumCases());
793 for (auto Case : SI->cases())
794 Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(),
795 Case.getCaseSuccessor()));
796 return SI->getDefaultDest();
797 }
798
799 BranchInst *BI = cast<BranchInst>(TI);
800 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
801 BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE);
802 Cases.push_back(ValueEqualityComparisonCase(
803 GetConstantInt(ICI->getOperand(1), DL), Succ));
804 return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ);
805}
806
807/// Given a vector of bb/value pairs, remove any entries
808/// in the list that match the specified block.
809static void
810EliminateBlockCases(BasicBlock *BB,
811 std::vector<ValueEqualityComparisonCase> &Cases) {
812 llvm::erase_value(Cases, BB);
813}
814
815/// Return true if there are any keys in C1 that exist in C2 as well.
816static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
817 std::vector<ValueEqualityComparisonCase> &C2) {
818 std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2;
819
820 // Make V1 be smaller than V2.
821 if (V1->size() > V2->size())
822 std::swap(V1, V2);
823
824 if (V1->empty())
825 return false;
826 if (V1->size() == 1) {
827 // Just scan V2.
828 ConstantInt *TheVal = (*V1)[0].Value;
829 for (unsigned i = 0, e = V2->size(); i != e; ++i)
830 if (TheVal == (*V2)[i].Value)
831 return true;
832 }
833
834 // Otherwise, just sort both lists and compare element by element.
835 array_pod_sort(V1->begin(), V1->end());
836 array_pod_sort(V2->begin(), V2->end());
837 unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size();
838 while (i1 != e1 && i2 != e2) {
839 if ((*V1)[i1].Value == (*V2)[i2].Value)
840 return true;
841 if ((*V1)[i1].Value < (*V2)[i2].Value)
842 ++i1;
843 else
844 ++i2;
845 }
846 return false;
847}
848
849// Set branch weights on SwitchInst. This sets the metadata if there is at
850// least one non-zero weight.
851static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights) {
852 // Check that there is at least one non-zero weight. Otherwise, pass
853 // nullptr to setMetadata which will erase the existing metadata.
854 MDNode *N = nullptr;
855 if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; }))
856 N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights);
857 SI->setMetadata(LLVMContext::MD_prof, N);
858}
859
860// Similar to the above, but for branch and select instructions that take
861// exactly 2 weights.
862static void setBranchWeights(Instruction *I, uint32_t TrueWeight,
863 uint32_t FalseWeight) {
864 assert(isa<BranchInst>(I) || isa<SelectInst>(I))((isa<BranchInst>(I) || isa<SelectInst>(I)) ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(I) || isa<SelectInst>(I)"
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 864, __PRETTY_FUNCTION__))
;
865 // Check that there is at least one non-zero weight. Otherwise, pass
866 // nullptr to setMetadata which will erase the existing metadata.
867 MDNode *N = nullptr;
868 if (TrueWeight || FalseWeight)
869 N = MDBuilder(I->getParent()->getContext())
870 .createBranchWeights(TrueWeight, FalseWeight);
871 I->setMetadata(LLVMContext::MD_prof, N);
872}
873
874/// If TI is known to be a terminator instruction and its block is known to
875/// only have a single predecessor block, check to see if that predecessor is
876/// also a value comparison with the same value, and if that comparison
877/// determines the outcome of this comparison. If so, simplify TI. This does a
878/// very limited form of jump threading.
879bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
880 Instruction *TI, BasicBlock *Pred, IRBuilder<> &Builder) {
881 Value *PredVal = isValueEqualityComparison(Pred->getTerminator());
882 if (!PredVal)
883 return false; // Not a value comparison in predecessor.
884
885 Value *ThisVal = isValueEqualityComparison(TI);
886 assert(ThisVal && "This isn't a value comparison!!")((ThisVal && "This isn't a value comparison!!") ? static_cast
<void> (0) : __assert_fail ("ThisVal && \"This isn't a value comparison!!\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 886, __PRETTY_FUNCTION__))
;
887 if (ThisVal != PredVal)
888 return false; // Different predicates.
889
890 // TODO: Preserve branch weight metadata, similarly to how
891 // FoldValueComparisonIntoPredecessors preserves it.
892
893 // Find out information about when control will move from Pred to TI's block.
894 std::vector<ValueEqualityComparisonCase> PredCases;
895 BasicBlock *PredDef =
896 GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases);
897 EliminateBlockCases(PredDef, PredCases); // Remove default from cases.
898
899 // Find information about how control leaves this block.
900 std::vector<ValueEqualityComparisonCase> ThisCases;
901 BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases);
902 EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases.
903
904 // If TI's block is the default block from Pred's comparison, potentially
905 // simplify TI based on this knowledge.
906 if (PredDef == TI->getParent()) {
907 // If we are here, we know that the value is none of those cases listed in
908 // PredCases. If there are any cases in ThisCases that are in PredCases, we
909 // can simplify TI.
910 if (!ValuesOverlap(PredCases, ThisCases))
911 return false;
912
913 if (isa<BranchInst>(TI)) {
914 // Okay, one of the successors of this condbr is dead. Convert it to a
915 // uncond br.
916 assert(ThisCases.size() == 1 && "Branch can only have one case!")((ThisCases.size() == 1 && "Branch can only have one case!"
) ? static_cast<void> (0) : __assert_fail ("ThisCases.size() == 1 && \"Branch can only have one case!\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 916, __PRETTY_FUNCTION__))
;
917 // Insert the new branch.
918 Instruction *NI = Builder.CreateBr(ThisDef);
919 (void)NI;
920
921 // Remove PHI node entries for the dead edge.
922 ThisCases[0].Dest->removePredecessor(PredDef);
923
924 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
925 << "Through successor TI: " << *TI << "Leaving: " << *NIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
926 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
;
927
928 EraseTerminatorAndDCECond(TI);
929
930 if (DTU)
931 DTU->applyUpdates(
932 {{DominatorTree::Delete, PredDef, ThisCases[0].Dest}});
933
934 return true;
935 }
936
937 SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI);
938 // Okay, TI has cases that are statically dead, prune them away.
939 SmallPtrSet<Constant *, 16> DeadCases;
940 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
941 DeadCases.insert(PredCases[i].Value);
942
943 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI; } } while (false)
944 << "Through successor TI: " << *TI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI; } } while (false)
;
945
946 SmallMapVector<BasicBlock *, int, 8> NumPerSuccessorCases;
947 for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
948 --i;
949 auto *Successor = i->getCaseSuccessor();
950 ++NumPerSuccessorCases[Successor];
951 if (DeadCases.count(i->getCaseValue())) {
952 Successor->removePredecessor(PredDef);
953 SI.removeCase(i);
954 --NumPerSuccessorCases[Successor];
955 }
956 }
957
958 std::vector<DominatorTree::UpdateType> Updates;
959 for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
960 if (I.second == 0)
961 Updates.push_back({DominatorTree::Delete, PredDef, I.first});
962 if (DTU)
963 DTU->applyUpdates(Updates);
964
965 LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Leaving: " << *TI <<
"\n"; } } while (false)
;
966 return true;
967 }
968
969 // Otherwise, TI's block must correspond to some matched value. Find out
970 // which value (or set of values) this is.
971 ConstantInt *TIV = nullptr;
972 BasicBlock *TIBB = TI->getParent();
973 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
974 if (PredCases[i].Dest == TIBB) {
975 if (TIV)
976 return false; // Cannot handle multiple values coming to this block.
977 TIV = PredCases[i].Value;
978 }
979 assert(TIV && "No edge from pred to succ?")((TIV && "No edge from pred to succ?") ? static_cast<
void> (0) : __assert_fail ("TIV && \"No edge from pred to succ?\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 979, __PRETTY_FUNCTION__))
;
980
981 // Okay, we found the one constant that our value can be if we get into TI's
982 // BB. Find out which successor will unconditionally be branched to.
983 BasicBlock *TheRealDest = nullptr;
984 for (unsigned i = 0, e = ThisCases.size(); i != e; ++i)
985 if (ThisCases[i].Value == TIV) {
986 TheRealDest = ThisCases[i].Dest;
987 break;
988 }
989
990 // If not handled by any explicit cases, it is handled by the default case.
991 if (!TheRealDest)
992 TheRealDest = ThisDef;
993
994 SmallSetVector<BasicBlock *, 2> RemovedSuccs;
995
996 // Remove PHI node entries for dead edges.
997 BasicBlock *CheckEdge = TheRealDest;
998 for (BasicBlock *Succ : successors(TIBB))
999 if (Succ != CheckEdge) {
1000 if (Succ != TheRealDest)
1001 RemovedSuccs.insert(Succ);
1002 Succ->removePredecessor(TIBB);
1003 } else
1004 CheckEdge = nullptr;
1005
1006 // Insert the new branch.
1007 Instruction *NI = Builder.CreateBr(TheRealDest);
1008 (void)NI;
1009
1010 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
1011 << "Through successor TI: " << *TI << "Leaving: " << *NIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
1012 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
;
1013
1014 EraseTerminatorAndDCECond(TI);
1015 if (DTU) {
1016 SmallVector<DominatorTree::UpdateType, 2> Updates;
1017 Updates.reserve(RemovedSuccs.size());
1018 for (auto *RemovedSucc : RemovedSuccs)
1019 Updates.push_back({DominatorTree::Delete, TIBB, RemovedSucc});
1020 DTU->applyUpdates(Updates);
1021 }
1022 return true;
1023}
1024
1025namespace {
1026
1027/// This class implements a stable ordering of constant
1028/// integers that does not depend on their address. This is important for
1029/// applications that sort ConstantInt's to ensure uniqueness.
1030struct ConstantIntOrdering {
1031 bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const {
1032 return LHS->getValue().ult(RHS->getValue());
1033 }
1034};
1035
1036} // end anonymous namespace
1037
1038static int ConstantIntSortPredicate(ConstantInt *const *P1,
1039 ConstantInt *const *P2) {
1040 const ConstantInt *LHS = *P1;
1041 const ConstantInt *RHS = *P2;
1042 if (LHS == RHS)
1043 return 0;
1044 return LHS->getValue().ult(RHS->getValue()) ? 1 : -1;
1045}
1046
1047static inline bool HasBranchWeights(const Instruction *I) {
1048 MDNode *ProfMD = I->getMetadata(LLVMContext::MD_prof);
1049 if (ProfMD && ProfMD->getOperand(0))
1050 if (MDString *MDS = dyn_cast<MDString>(ProfMD->getOperand(0)))
1051 return MDS->getString().equals("branch_weights");
1052
1053 return false;
1054}
1055
1056/// Get Weights of a given terminator, the default weight is at the front
1057/// of the vector. If TI is a conditional eq, we need to swap the branch-weight
1058/// metadata.
1059static void GetBranchWeights(Instruction *TI,
1060 SmallVectorImpl<uint64_t> &Weights) {
1061 MDNode *MD = TI->getMetadata(LLVMContext::MD_prof);
1062 assert(MD)((MD) ? static_cast<void> (0) : __assert_fail ("MD", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1062, __PRETTY_FUNCTION__))
;
1063 for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) {
1064 ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(i));
1065 Weights.push_back(CI->getValue().getZExtValue());
1066 }
1067
1068 // If TI is a conditional eq, the default case is the false case,
1069 // and the corresponding branch-weight data is at index 2. We swap the
1070 // default weight to be the first entry.
1071 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
1072 assert(Weights.size() == 2)((Weights.size() == 2) ? static_cast<void> (0) : __assert_fail
("Weights.size() == 2", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1072, __PRETTY_FUNCTION__))
;
1073 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
1074 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
1075 std::swap(Weights.front(), Weights.back());
1076 }
1077}
1078
1079/// Keep halving the weights until all can fit in uint32_t.
1080static void FitWeights(MutableArrayRef<uint64_t> Weights) {
1081 uint64_t Max = *std::max_element(Weights.begin(), Weights.end());
1082 if (Max > UINT_MAX(2147483647 *2U +1U)) {
1083 unsigned Offset = 32 - countLeadingZeros(Max);
1084 for (uint64_t &I : Weights)
1085 I >>= Offset;
1086 }
1087}
1088
1089/// The specified terminator is a value equality comparison instruction
1090/// (either a switch or a branch on "X == c").
1091/// See if any of the predecessors of the terminator block are value comparisons
1092/// on the same value. If so, and if safe to do so, fold them together.
1093bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(Instruction *TI,
1094 IRBuilder<> &Builder) {
1095 BasicBlock *BB = TI->getParent();
1096 Value *CV = isValueEqualityComparison(TI); // CondVal
1097 assert(CV && "Not a comparison?")((CV && "Not a comparison?") ? static_cast<void>
(0) : __assert_fail ("CV && \"Not a comparison?\"", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1097, __PRETTY_FUNCTION__))
;
1098
1099 bool Changed = false;
1100
1101 auto _ = make_scope_exit([&]() {
1102 if (Changed)
1103 ++NumFoldValueComparisonIntoPredecessors;
1104 });
1105
1106 SmallSetVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB));
1107 while (!Preds.empty()) {
1108 BasicBlock *Pred = Preds.pop_back_val();
1109
1110 // See if the predecessor is a comparison with the same value.
1111 Instruction *PTI = Pred->getTerminator();
1112 Value *PCV = isValueEqualityComparison(PTI); // PredCondVal
1113
1114 if (PCV == CV && TI != PTI) {
1115 SmallSetVector<BasicBlock*, 4> FailBlocks;
1116 if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) {
1117 for (auto *Succ : FailBlocks) {
1118 if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split",
1119 DTU))
1120 return false;
1121 }
1122 }
1123
1124 std::vector<DominatorTree::UpdateType> Updates;
1125
1126 // Figure out which 'cases' to copy from SI to PSI.
1127 std::vector<ValueEqualityComparisonCase> BBCases;
1128 BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases);
1129
1130 std::vector<ValueEqualityComparisonCase> PredCases;
1131 BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases);
1132
1133 // Based on whether the default edge from PTI goes to BB or not, fill in
1134 // PredCases and PredDefault with the new switch cases we would like to
1135 // build.
1136 SmallMapVector<BasicBlock *, int, 8> NewSuccessors;
1137
1138 // Update the branch weight metadata along the way
1139 SmallVector<uint64_t, 8> Weights;
1140 bool PredHasWeights = HasBranchWeights(PTI);
1141 bool SuccHasWeights = HasBranchWeights(TI);
1142
1143 if (PredHasWeights) {
1144 GetBranchWeights(PTI, Weights);
1145 // branch-weight metadata is inconsistent here.
1146 if (Weights.size() != 1 + PredCases.size())
1147 PredHasWeights = SuccHasWeights = false;
1148 } else if (SuccHasWeights)
1149 // If there are no predecessor weights but there are successor weights,
1150 // populate Weights with 1, which will later be scaled to the sum of
1151 // successor's weights
1152 Weights.assign(1 + PredCases.size(), 1);
1153
1154 SmallVector<uint64_t, 8> SuccWeights;
1155 if (SuccHasWeights) {
1156 GetBranchWeights(TI, SuccWeights);
1157 // branch-weight metadata is inconsistent here.
1158 if (SuccWeights.size() != 1 + BBCases.size())
1159 PredHasWeights = SuccHasWeights = false;
1160 } else if (PredHasWeights)
1161 SuccWeights.assign(1 + BBCases.size(), 1);
1162
1163 if (PredDefault == BB) {
1164 // If this is the default destination from PTI, only the edges in TI
1165 // that don't occur in PTI, or that branch to BB will be activated.
1166 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1167 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1168 if (PredCases[i].Dest != BB)
1169 PTIHandled.insert(PredCases[i].Value);
1170 else {
1171 // The default destination is BB, we don't need explicit targets.
1172 std::swap(PredCases[i], PredCases.back());
1173
1174 if (PredHasWeights || SuccHasWeights) {
1175 // Increase weight for the default case.
1176 Weights[0] += Weights[i + 1];
1177 std::swap(Weights[i + 1], Weights.back());
1178 Weights.pop_back();
1179 }
1180
1181 PredCases.pop_back();
1182 --i;
1183 --e;
1184 }
1185
1186 // Reconstruct the new switch statement we will be building.
1187 if (PredDefault != BBDefault) {
1188 PredDefault->removePredecessor(Pred);
1189 if (PredDefault != BB)
1190 Updates.push_back({DominatorTree::Delete, Pred, PredDefault});
1191 PredDefault = BBDefault;
1192 ++NewSuccessors[BBDefault];
1193 }
1194
1195 unsigned CasesFromPred = Weights.size();
1196 uint64_t ValidTotalSuccWeight = 0;
1197 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1198 if (!PTIHandled.count(BBCases[i].Value) &&
1199 BBCases[i].Dest != BBDefault) {
1200 PredCases.push_back(BBCases[i]);
1201 ++NewSuccessors[BBCases[i].Dest];
1202 if (SuccHasWeights || PredHasWeights) {
1203 // The default weight is at index 0, so weight for the ith case
1204 // should be at index i+1. Scale the cases from successor by
1205 // PredDefaultWeight (Weights[0]).
1206 Weights.push_back(Weights[0] * SuccWeights[i + 1]);
1207 ValidTotalSuccWeight += SuccWeights[i + 1];
1208 }
1209 }
1210
1211 if (SuccHasWeights || PredHasWeights) {
1212 ValidTotalSuccWeight += SuccWeights[0];
1213 // Scale the cases from predecessor by ValidTotalSuccWeight.
1214 for (unsigned i = 1; i < CasesFromPred; ++i)
1215 Weights[i] *= ValidTotalSuccWeight;
1216 // Scale the default weight by SuccDefaultWeight (SuccWeights[0]).
1217 Weights[0] *= SuccWeights[0];
1218 }
1219 } else {
1220 // If this is not the default destination from PSI, only the edges
1221 // in SI that occur in PSI with a destination of BB will be
1222 // activated.
1223 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1224 std::map<ConstantInt *, uint64_t> WeightsForHandled;
1225 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1226 if (PredCases[i].Dest == BB) {
1227 PTIHandled.insert(PredCases[i].Value);
1228
1229 if (PredHasWeights || SuccHasWeights) {
1230 WeightsForHandled[PredCases[i].Value] = Weights[i + 1];
1231 std::swap(Weights[i + 1], Weights.back());
1232 Weights.pop_back();
1233 }
1234
1235 std::swap(PredCases[i], PredCases.back());
1236 PredCases.pop_back();
1237 --i;
1238 --e;
1239 }
1240
1241 // Okay, now we know which constants were sent to BB from the
1242 // predecessor. Figure out where they will all go now.
1243 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1244 if (PTIHandled.count(BBCases[i].Value)) {
1245 // If this is one we are capable of getting...
1246 if (PredHasWeights || SuccHasWeights)
1247 Weights.push_back(WeightsForHandled[BBCases[i].Value]);
1248 PredCases.push_back(BBCases[i]);
1249 ++NewSuccessors[BBCases[i].Dest];
1250 PTIHandled.erase(
1251 BBCases[i].Value); // This constant is taken care of
1252 }
1253
1254 // If there are any constants vectored to BB that TI doesn't handle,
1255 // they must go to the default destination of TI.
1256 for (ConstantInt *I : PTIHandled) {
1257 if (PredHasWeights || SuccHasWeights)
1258 Weights.push_back(WeightsForHandled[I]);
1259 PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault));
1260 ++NewSuccessors[BBDefault];
1261 }
1262 }
1263
1264 // Okay, at this point, we know which new successor Pred will get. Make
1265 // sure we update the number of entries in the PHI nodes for these
1266 // successors.
1267 for (const std::pair<BasicBlock *, int /*Num*/> &NewSuccessor :
1268 NewSuccessors) {
1269 for (auto I : seq(0, NewSuccessor.second)) {
1270 (void)I;
1271 AddPredecessorToBlock(NewSuccessor.first, Pred, BB);
1272 }
1273 if (!is_contained(successors(Pred), NewSuccessor.first))
1274 Updates.push_back({DominatorTree::Insert, Pred, NewSuccessor.first});
1275 }
1276
1277 Builder.SetInsertPoint(PTI);
1278 // Convert pointer to int before we switch.
1279 if (CV->getType()->isPointerTy()) {
1280 CV = Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()),
1281 "magicptr");
1282 }
1283
1284 // Now that the successors are updated, create the new Switch instruction.
1285 SwitchInst *NewSI =
1286 Builder.CreateSwitch(CV, PredDefault, PredCases.size());
1287 NewSI->setDebugLoc(PTI->getDebugLoc());
1288 for (ValueEqualityComparisonCase &V : PredCases)
1289 NewSI->addCase(V.Value, V.Dest);
1290
1291 if (PredHasWeights || SuccHasWeights) {
1292 // Halve the weights if any of them cannot fit in an uint32_t
1293 FitWeights(Weights);
1294
1295 SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
1296
1297 setBranchWeights(NewSI, MDWeights);
1298 }
1299
1300 EraseTerminatorAndDCECond(PTI);
1301
1302 // Okay, last check. If BB is still a successor of PSI, then we must
1303 // have an infinite loop case. If so, add an infinitely looping block
1304 // to handle the case to preserve the behavior of the code.
1305 BasicBlock *InfLoopBlock = nullptr;
1306 for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i)
1307 if (NewSI->getSuccessor(i) == BB) {
1308 if (!InfLoopBlock) {
1309 // Insert it at the end of the function, because it's either code,
1310 // or it won't matter if it's hot. :)
1311 InfLoopBlock = BasicBlock::Create(BB->getContext(), "infloop",
1312 BB->getParent());
1313 BranchInst::Create(InfLoopBlock, InfLoopBlock);
1314 Updates.push_back(
1315 {DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
1316 }
1317 NewSI->setSuccessor(i, InfLoopBlock);
1318 }
1319
1320 if (InfLoopBlock)
1321 Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock});
1322
1323 Updates.push_back({DominatorTree::Delete, Pred, BB});
1324
1325 if (DTU)
1326 DTU->applyUpdates(Updates);
1327
1328 Changed = true;
1329 }
1330 }
1331 return Changed;
1332}
1333
1334// If we would need to insert a select that uses the value of this invoke
1335// (comments in HoistThenElseCodeToIf explain why we would need to do this), we
1336// can't hoist the invoke, as there is nowhere to put the select in this case.
1337static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2,
1338 Instruction *I1, Instruction *I2) {
1339 for (BasicBlock *Succ : successors(BB1)) {
1340 for (const PHINode &PN : Succ->phis()) {
1341 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1342 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1343 if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) {
1344 return false;
1345 }
1346 }
1347 }
1348 return true;
1349}
1350
1351static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified = false);
1352
1353/// Given a conditional branch that goes to BB1 and BB2, hoist any common code
1354/// in the two blocks up into the branch block. The caller of this function
1355/// guarantees that BI's block dominates BB1 and BB2.
1356bool SimplifyCFGOpt::HoistThenElseCodeToIf(BranchInst *BI,
1357 const TargetTransformInfo &TTI) {
1358 // This does very trivial matching, with limited scanning, to find identical
1359 // instructions in the two blocks. In particular, we don't want to get into
1360 // O(M*N) situations here where M and N are the sizes of BB1 and BB2. As
1361 // such, we currently just scan for obviously identical instructions in an
1362 // identical order.
1363 BasicBlock *BB1 = BI->getSuccessor(0); // The true destination.
1364 BasicBlock *BB2 = BI->getSuccessor(1); // The false destination
1365
1366 BasicBlock::iterator BB1_Itr = BB1->begin();
1367 BasicBlock::iterator BB2_Itr = BB2->begin();
1368
1369 Instruction *I1 = &*BB1_Itr++, *I2 = &*BB2_Itr++;
1370 // Skip debug info if it is not identical.
1371 DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1);
1372 DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2);
1373 if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1374 while (isa<DbgInfoIntrinsic>(I1))
1375 I1 = &*BB1_Itr++;
1376 while (isa<DbgInfoIntrinsic>(I2))
1377 I2 = &*BB2_Itr++;
1378 }
1379 // FIXME: Can we define a safety predicate for CallBr?
1380 if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) ||
1381 (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) ||
1382 isa<CallBrInst>(I1))
1383 return false;
1384
1385 BasicBlock *BIParent = BI->getParent();
1386
1387 bool Changed = false;
1388
1389 auto _ = make_scope_exit([&]() {
1390 if (Changed)
1391 ++NumHoistCommonCode;
1392 });
1393
1394 do {
1395 // If we are hoisting the terminator instruction, don't move one (making a
1396 // broken BB), instead clone it, and remove BI.
1397 if (I1->isTerminator())
1398 goto HoistTerminator;
1399
1400 // If we're going to hoist a call, make sure that the two instructions we're
1401 // commoning/hoisting are both marked with musttail, or neither of them is
1402 // marked as such. Otherwise, we might end up in a situation where we hoist
1403 // from a block where the terminator is a `ret` to a block where the terminator
1404 // is a `br`, and `musttail` calls expect to be followed by a return.
1405 auto *C1 = dyn_cast<CallInst>(I1);
1406 auto *C2 = dyn_cast<CallInst>(I2);
1407 if (C1 && C2)
1408 if (C1->isMustTailCall() != C2->isMustTailCall())
1409 return Changed;
1410
1411 if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2))
1412 return Changed;
1413
1414 // If any of the two call sites has nomerge attribute, stop hoisting.
1415 if (const auto *CB1 = dyn_cast<CallBase>(I1))
1416 if (CB1->cannotMerge())
1417 return Changed;
1418 if (const auto *CB2 = dyn_cast<CallBase>(I2))
1419 if (CB2->cannotMerge())
1420 return Changed;
1421
1422 if (isa<DbgInfoIntrinsic>(I1) || isa<DbgInfoIntrinsic>(I2)) {
1423 assert (isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2))((isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic
>(I2)) ? static_cast<void> (0) : __assert_fail ("isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2)"
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1423, __PRETTY_FUNCTION__))
;
1424 // The debug location is an integral part of a debug info intrinsic
1425 // and can't be separated from it or replaced. Instead of attempting
1426 // to merge locations, simply hoist both copies of the intrinsic.
1427 BIParent->getInstList().splice(BI->getIterator(),
1428 BB1->getInstList(), I1);
1429 BIParent->getInstList().splice(BI->getIterator(),
1430 BB2->getInstList(), I2);
1431 Changed = true;
1432 } else {
1433 // For a normal instruction, we just move one to right before the branch,
1434 // then replace all uses of the other with the first. Finally, we remove
1435 // the now redundant second instruction.
1436 BIParent->getInstList().splice(BI->getIterator(),
1437 BB1->getInstList(), I1);
1438 if (!I2->use_empty())
1439 I2->replaceAllUsesWith(I1);
1440 I1->andIRFlags(I2);
1441 unsigned KnownIDs[] = {LLVMContext::MD_tbaa,
1442 LLVMContext::MD_range,
1443 LLVMContext::MD_fpmath,
1444 LLVMContext::MD_invariant_load,
1445 LLVMContext::MD_nonnull,
1446 LLVMContext::MD_invariant_group,
1447 LLVMContext::MD_align,
1448 LLVMContext::MD_dereferenceable,
1449 LLVMContext::MD_dereferenceable_or_null,
1450 LLVMContext::MD_mem_parallel_loop_access,
1451 LLVMContext::MD_access_group,
1452 LLVMContext::MD_preserve_access_index};
1453 combineMetadata(I1, I2, KnownIDs, true);
1454
1455 // I1 and I2 are being combined into a single instruction. Its debug
1456 // location is the merged locations of the original instructions.
1457 I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1458
1459 I2->eraseFromParent();
1460 Changed = true;
1461 }
1462 ++NumHoistCommonInstrs;
1463
1464 I1 = &*BB1_Itr++;
1465 I2 = &*BB2_Itr++;
1466 // Skip debug info if it is not identical.
1467 DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1);
1468 DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2);
1469 if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1470 while (isa<DbgInfoIntrinsic>(I1))
1471 I1 = &*BB1_Itr++;
1472 while (isa<DbgInfoIntrinsic>(I2))
1473 I2 = &*BB2_Itr++;
1474 }
1475 } while (I1->isIdenticalToWhenDefined(I2));
1476
1477 return true;
1478
1479HoistTerminator:
1480 // It may not be possible to hoist an invoke.
1481 // FIXME: Can we define a safety predicate for CallBr?
1482 if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))
1483 return Changed;
1484
1485 // TODO: callbr hoisting currently disabled pending further study.
1486 if (isa<CallBrInst>(I1))
1487 return Changed;
1488
1489 for (BasicBlock *Succ : successors(BB1)) {
1490 for (PHINode &PN : Succ->phis()) {
1491 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1492 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1493 if (BB1V == BB2V)
1494 continue;
1495
1496 // Check for passingValueIsAlwaysUndefined here because we would rather
1497 // eliminate undefined control flow then converting it to a select.
1498 if (passingValueIsAlwaysUndefined(BB1V, &PN) ||
1499 passingValueIsAlwaysUndefined(BB2V, &PN))
1500 return Changed;
1501
1502 if (isa<ConstantExpr>(BB1V) && !isSafeToSpeculativelyExecute(BB1V))
1503 return Changed;
1504 if (isa<ConstantExpr>(BB2V) && !isSafeToSpeculativelyExecute(BB2V))
1505 return Changed;
1506 }
1507 }
1508
1509 // Okay, it is safe to hoist the terminator.
1510 Instruction *NT = I1->clone();
1511 BIParent->getInstList().insert(BI->getIterator(), NT);
1512 if (!NT->getType()->isVoidTy()) {
1513 I1->replaceAllUsesWith(NT);
1514 I2->replaceAllUsesWith(NT);
1515 NT->takeName(I1);
1516 }
1517 Changed = true;
1518 ++NumHoistCommonInstrs;
1519
1520 // Ensure terminator gets a debug location, even an unknown one, in case
1521 // it involves inlinable calls.
1522 NT->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1523
1524 // PHIs created below will adopt NT's merged DebugLoc.
1525 IRBuilder<NoFolder> Builder(NT);
1526
1527 // Hoisting one of the terminators from our successor is a great thing.
1528 // Unfortunately, the successors of the if/else blocks may have PHI nodes in
1529 // them. If they do, all PHI entries for BB1/BB2 must agree for all PHI
1530 // nodes, so we insert select instruction to compute the final result.
1531 std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects;
1532 for (BasicBlock *Succ : successors(BB1)) {
1533 for (PHINode &PN : Succ->phis()) {
1534 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1535 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1536 if (BB1V == BB2V)
1537 continue;
1538
1539 // These values do not agree. Insert a select instruction before NT
1540 // that determines the right value.
1541 SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)];
1542 if (!SI) {
1543 // Propagate fast-math-flags from phi node to its replacement select.
1544 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
1545 if (isa<FPMathOperator>(PN))
1546 Builder.setFastMathFlags(PN.getFastMathFlags());
1547
1548 SI = cast<SelectInst>(
1549 Builder.CreateSelect(BI->getCondition(), BB1V, BB2V,
1550 BB1V->getName() + "." + BB2V->getName(), BI));
1551 }
1552
1553 // Make the PHI node use the select for all incoming values for BB1/BB2
1554 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
1555 if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2)
1556 PN.setIncomingValue(i, SI);
1557 }
1558 }
1559
1560 SmallVector<DominatorTree::UpdateType, 4> Updates;
1561
1562 // Update any PHI nodes in our new successors.
1563 for (BasicBlock *Succ : successors(BB1)) {
1564 AddPredecessorToBlock(Succ, BIParent, BB1);
1565 Updates.push_back({DominatorTree::Insert, BIParent, Succ});
1566 }
1567 for (BasicBlock *Succ : successors(BI))
1568 Updates.push_back({DominatorTree::Delete, BIParent, Succ});
1569
1570 EraseTerminatorAndDCECond(BI);
1571 if (DTU)
1572 DTU->applyUpdates(Updates);
1573 return Changed;
1574}
1575
1576// Check lifetime markers.
1577static bool isLifeTimeMarker(const Instruction *I) {
1578 if (auto II = dyn_cast<IntrinsicInst>(I)) {
1579 switch (II->getIntrinsicID()) {
1580 default:
1581 break;
1582 case Intrinsic::lifetime_start:
1583 case Intrinsic::lifetime_end:
1584 return true;
1585 }
1586 }
1587 return false;
1588}
1589
1590// TODO: Refine this. This should avoid cases like turning constant memcpy sizes
1591// into variables.
1592static bool replacingOperandWithVariableIsCheap(const Instruction *I,
1593 int OpIdx) {
1594 return !isa<IntrinsicInst>(I);
1595}
1596
1597// All instructions in Insts belong to different blocks that all unconditionally
1598// branch to a common successor. Analyze each instruction and return true if it
1599// would be possible to sink them into their successor, creating one common
1600// instruction instead. For every value that would be required to be provided by
1601// PHI node (because an operand varies in each input block), add to PHIOperands.
1602static bool canSinkInstructions(
1603 ArrayRef<Instruction *> Insts,
1604 DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) {
1605 // Prune out obviously bad instructions to move. Each instruction must have
1606 // exactly zero or one use, and we check later that use is by a single, common
1607 // PHI instruction in the successor.
1608 bool HasUse = !Insts.front()->user_empty();
1609 for (auto *I : Insts) {
1610 // These instructions may change or break semantics if moved.
1611 if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) ||
1612 I->getType()->isTokenTy())
1613 return false;
1614
1615 // Conservatively return false if I is an inline-asm instruction. Sinking
1616 // and merging inline-asm instructions can potentially create arguments
1617 // that cannot satisfy the inline-asm constraints.
1618 // If the instruction has nomerge attribute, return false.
1619 if (const auto *C = dyn_cast<CallBase>(I))
1620 if (C->isInlineAsm() || C->cannotMerge())
1621 return false;
1622
1623 // Each instruction must have zero or one use.
1624 if (HasUse && !I->hasOneUse())
1625 return false;
1626 if (!HasUse && !I->user_empty())
1627 return false;
1628 }
1629
1630 const Instruction *I0 = Insts.front();
1631 for (auto *I : Insts)
1632 if (!I->isSameOperationAs(I0))
1633 return false;
1634
1635 // All instructions in Insts are known to be the same opcode. If they have a
1636 // use, check that the only user is a PHI or in the same block as the
1637 // instruction, because if a user is in the same block as an instruction we're
1638 // contemplating sinking, it must already be determined to be sinkable.
1639 if (HasUse) {
1640 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1641 auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0);
1642 if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool {
1643 auto *U = cast<Instruction>(*I->user_begin());
1644 return (PNUse &&
1645 PNUse->getParent() == Succ &&
1646 PNUse->getIncomingValueForBlock(I->getParent()) == I) ||
1647 U->getParent() == I->getParent();
1648 }))
1649 return false;
1650 }
1651
1652 // Because SROA can't handle speculating stores of selects, try not to sink
1653 // loads, stores or lifetime markers of allocas when we'd have to create a
1654 // PHI for the address operand. Also, because it is likely that loads or
1655 // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink
1656 // them.
1657 // This can cause code churn which can have unintended consequences down
1658 // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244.
1659 // FIXME: This is a workaround for a deficiency in SROA - see
1660 // https://llvm.org/bugs/show_bug.cgi?id=30188
1661 if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) {
1662 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
1663 }))
1664 return false;
1665 if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) {
1666 return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
1667 }))
1668 return false;
1669 if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) {
1670 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
1671 }))
1672 return false;
1673
1674 for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) {
1675 Value *Op = I0->getOperand(OI);
1676 if (Op->getType()->isTokenTy())
1677 // Don't touch any operand of token type.
1678 return false;
1679
1680 auto SameAsI0 = [&I0, OI](const Instruction *I) {
1681 assert(I->getNumOperands() == I0->getNumOperands())((I->getNumOperands() == I0->getNumOperands()) ? static_cast
<void> (0) : __assert_fail ("I->getNumOperands() == I0->getNumOperands()"
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1681, __PRETTY_FUNCTION__))
;
1682 return I->getOperand(OI) == I0->getOperand(OI);
1683 };
1684 if (!all_of(Insts, SameAsI0)) {
1685 if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) ||
1686 !canReplaceOperandWithVariable(I0, OI))
1687 // We can't create a PHI from this GEP.
1688 return false;
1689 // Don't create indirect calls! The called value is the final operand.
1690 if (isa<CallBase>(I0) && OI == OE - 1) {
1691 // FIXME: if the call was *already* indirect, we should do this.
1692 return false;
1693 }
1694 for (auto *I : Insts)
1695 PHIOperands[I].push_back(I->getOperand(OI));
1696 }
1697 }
1698 return true;
1699}
1700
1701// Assuming canSinkLastInstruction(Blocks) has returned true, sink the last
1702// instruction of every block in Blocks to their common successor, commoning
1703// into one instruction.
1704static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
1705 auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0);
1706
1707 // canSinkLastInstruction returning true guarantees that every block has at
1708 // least one non-terminator instruction.
1709 SmallVector<Instruction*,4> Insts;
1710 for (auto *BB : Blocks) {
1711 Instruction *I = BB->getTerminator();
1712 do {
1713 I = I->getPrevNode();
1714 } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front());
1715 if (!isa<DbgInfoIntrinsic>(I))
1716 Insts.push_back(I);
1717 }
1718
1719 // The only checking we need to do now is that all users of all instructions
1720 // are the same PHI node. canSinkLastInstruction should have checked this but
1721 // it is slightly over-aggressive - it gets confused by commutative instructions
1722 // so double-check it here.
1723 Instruction *I0 = Insts.front();
1724 if (!I0->user_empty()) {
1725 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1726 if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool {
1727 auto *U = cast<Instruction>(*I->user_begin());
1728 return U == PNUse;
1729 }))
1730 return false;
1731 }
1732
1733 // We don't need to do any more checking here; canSinkLastInstruction should
1734 // have done it all for us.
1735 SmallVector<Value*, 4> NewOperands;
1736 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) {
1737 // This check is different to that in canSinkLastInstruction. There, we
1738 // cared about the global view once simplifycfg (and instcombine) have
1739 // completed - it takes into account PHIs that become trivially
1740 // simplifiable. However here we need a more local view; if an operand
1741 // differs we create a PHI and rely on instcombine to clean up the very
1742 // small mess we may make.
1743 bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) {
1744 return I->getOperand(O) != I0->getOperand(O);
1745 });
1746 if (!NeedPHI) {
1747 NewOperands.push_back(I0->getOperand(O));
1748 continue;
1749 }
1750
1751 // Create a new PHI in the successor block and populate it.
1752 auto *Op = I0->getOperand(O);
1753 assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!")((!Op->getType()->isTokenTy() && "Can't PHI tokens!"
) ? static_cast<void> (0) : __assert_fail ("!Op->getType()->isTokenTy() && \"Can't PHI tokens!\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1753, __PRETTY_FUNCTION__))
;
1754 auto *PN = PHINode::Create(Op->getType(), Insts.size(),
1755 Op->getName() + ".sink", &BBEnd->front());
1756 for (auto *I : Insts)
1757 PN->addIncoming(I->getOperand(O), I->getParent());
1758 NewOperands.push_back(PN);
1759 }
1760
1761 // Arbitrarily use I0 as the new "common" instruction; remap its operands
1762 // and move it to the start of the successor block.
1763 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O)
1764 I0->getOperandUse(O).set(NewOperands[O]);
1765 I0->moveBefore(&*BBEnd->getFirstInsertionPt());
1766
1767 // Update metadata and IR flags, and merge debug locations.
1768 for (auto *I : Insts)
1769 if (I != I0) {
1770 // The debug location for the "common" instruction is the merged locations
1771 // of all the commoned instructions. We start with the original location
1772 // of the "common" instruction and iteratively merge each location in the
1773 // loop below.
1774 // This is an N-way merge, which will be inefficient if I0 is a CallInst.
1775 // However, as N-way merge for CallInst is rare, so we use simplified API
1776 // instead of using complex API for N-way merge.
1777 I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc());
1778 combineMetadataForCSE(I0, I, true);
1779 I0->andIRFlags(I);
1780 }
1781
1782 if (!I0->user_empty()) {
1783 // canSinkLastInstruction checked that all instructions were used by
1784 // one and only one PHI node. Find that now, RAUW it to our common
1785 // instruction and nuke it.
1786 auto *PN = cast<PHINode>(*I0->user_begin());
1787 PN->replaceAllUsesWith(I0);
1788 PN->eraseFromParent();
1789 }
1790
1791 // Finally nuke all instructions apart from the common instruction.
1792 for (auto *I : Insts)
1793 if (I != I0)
1794 I->eraseFromParent();
1795
1796 return true;
1797}
1798
1799namespace {
1800
1801 // LockstepReverseIterator - Iterates through instructions
1802 // in a set of blocks in reverse order from the first non-terminator.
1803 // For example (assume all blocks have size n):
1804 // LockstepReverseIterator I([B1, B2, B3]);
1805 // *I-- = [B1[n], B2[n], B3[n]];
1806 // *I-- = [B1[n-1], B2[n-1], B3[n-1]];
1807 // *I-- = [B1[n-2], B2[n-2], B3[n-2]];
1808 // ...
1809 class LockstepReverseIterator {
1810 ArrayRef<BasicBlock*> Blocks;
1811 SmallVector<Instruction*,4> Insts;
1812 bool Fail;
1813
1814 public:
1815 LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) : Blocks(Blocks) {
1816 reset();
1817 }
1818
1819 void reset() {
1820 Fail = false;
1821 Insts.clear();
1822 for (auto *BB : Blocks) {
1823 Instruction *Inst = BB->getTerminator();
1824 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1825 Inst = Inst->getPrevNode();
1826 if (!Inst) {
1827 // Block wasn't big enough.
1828 Fail = true;
1829 return;
1830 }
1831 Insts.push_back(Inst);
1832 }
1833 }
1834
1835 bool isValid() const {
1836 return !Fail;
1837 }
1838
1839 void operator--() {
1840 if (Fail)
1841 return;
1842 for (auto *&Inst : Insts) {
1843 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1844 Inst = Inst->getPrevNode();
1845 // Already at beginning of block.
1846 if (!Inst) {
1847 Fail = true;
1848 return;
1849 }
1850 }
1851 }
1852
1853 ArrayRef<Instruction*> operator * () const {
1854 return Insts;
1855 }
1856 };
1857
1858} // end anonymous namespace
1859
1860/// Check whether BB's predecessors end with unconditional branches. If it is
1861/// true, sink any common code from the predecessors to BB.
1862/// We also allow one predecessor to end with conditional branch (but no more
1863/// than one).
1864static bool SinkCommonCodeFromPredecessors(BasicBlock *BB,
1865 DomTreeUpdater *DTU) {
1866 // We support two situations:
1867 // (1) all incoming arcs are unconditional
1868 // (2) one incoming arc is conditional
1869 //
1870 // (2) is very common in switch defaults and
1871 // else-if patterns;
1872 //
1873 // if (a) f(1);
1874 // else if (b) f(2);
1875 //
1876 // produces:
1877 //
1878 // [if]
1879 // / \
1880 // [f(1)] [if]
1881 // | | \
1882 // | | |
1883 // | [f(2)]|
1884 // \ | /
1885 // [ end ]
1886 //
1887 // [end] has two unconditional predecessor arcs and one conditional. The
1888 // conditional refers to the implicit empty 'else' arc. This conditional
1889 // arc can also be caused by an empty default block in a switch.
1890 //
1891 // In this case, we attempt to sink code from all *unconditional* arcs.
1892 // If we can sink instructions from these arcs (determined during the scan
1893 // phase below) we insert a common successor for all unconditional arcs and
1894 // connect that to [end], to enable sinking:
1895 //
1896 // [if]
1897 // / \
1898 // [x(1)] [if]
1899 // | | \
1900 // | | \
1901 // | [x(2)] |
1902 // \ / |
1903 // [sink.split] |
1904 // \ /
1905 // [ end ]
1906 //
1907 SmallVector<BasicBlock*,4> UnconditionalPreds;
1908 Instruction *Cond = nullptr;
1909 for (auto *B : predecessors(BB)) {
1910 auto *T = B->getTerminator();
1911 if (isa<BranchInst>(T) && cast<BranchInst>(T)->isUnconditional())
1912 UnconditionalPreds.push_back(B);
1913 else if ((isa<BranchInst>(T) || isa<SwitchInst>(T)) && !Cond)
1914 Cond = T;
1915 else
1916 return false;
1917 }
1918 if (UnconditionalPreds.size() < 2)
1919 return false;
1920
1921 // We take a two-step approach to tail sinking. First we scan from the end of
1922 // each block upwards in lockstep. If the n'th instruction from the end of each
1923 // block can be sunk, those instructions are added to ValuesToSink and we
1924 // carry on. If we can sink an instruction but need to PHI-merge some operands
1925 // (because they're not identical in each instruction) we add these to
1926 // PHIOperands.
1927 unsigned ScanIdx = 0;
1928 SmallPtrSet<Value*,4> InstructionsToSink;
1929 DenseMap<Instruction*, SmallVector<Value*,4>> PHIOperands;
1930 LockstepReverseIterator LRI(UnconditionalPreds);
1931 while (LRI.isValid() &&
1932 canSinkInstructions(*LRI, PHIOperands)) {
1933 LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: instruction can be sunk: "
<< *(*LRI)[0] << "\n"; } } while (false)
1934 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: instruction can be sunk: "
<< *(*LRI)[0] << "\n"; } } while (false)
;
1935 InstructionsToSink.insert((*LRI).begin(), (*LRI).end());
1936 ++ScanIdx;
1937 --LRI;
1938 }
1939
1940 // If no instructions can be sunk, early-return.
1941 if (ScanIdx == 0)
1942 return false;
1943
1944 bool Changed = false;
1945
1946 auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) {
1947 unsigned NumPHIdValues = 0;
1948 for (auto *I : *LRI)
1949 for (auto *V : PHIOperands[I])
1950 if (InstructionsToSink.count(V) == 0)
1951 ++NumPHIdValues;
1952 LLVM_DEBUG(dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: #phid values: " <<
NumPHIdValues << "\n"; } } while (false)
;
1953 unsigned NumPHIInsts = NumPHIdValues / UnconditionalPreds.size();
1954 if ((NumPHIdValues % UnconditionalPreds.size()) != 0)
1955 NumPHIInsts++;
1956
1957 return NumPHIInsts <= 1;
1958 };
1959
1960 if (Cond) {
1961 // Check if we would actually sink anything first! This mutates the CFG and
1962 // adds an extra block. The goal in doing this is to allow instructions that
1963 // couldn't be sunk before to be sunk - obviously, speculatable instructions
1964 // (such as trunc, add) can be sunk and predicated already. So we check that
1965 // we're going to sink at least one non-speculatable instruction.
1966 LRI.reset();
1967 unsigned Idx = 0;
1968 bool Profitable = false;
1969 while (ProfitableToSinkInstruction(LRI) && Idx < ScanIdx) {
1970 if (!isSafeToSpeculativelyExecute((*LRI)[0])) {
1971 Profitable = true;
1972 break;
1973 }
1974 --LRI;
1975 ++Idx;
1976 }
1977 if (!Profitable)
1978 return false;
1979
1980 LLVM_DEBUG(dbgs() << "SINK: Splitting edge\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Splitting edge\n"; }
} while (false)
;
1981 // We have a conditional edge and we're going to sink some instructions.
1982 // Insert a new block postdominating all blocks we're going to sink from.
1983 if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split", DTU))
1984 // Edges couldn't be split.
1985 return false;
1986 Changed = true;
1987 }
1988
1989 // Now that we've analyzed all potential sinking candidates, perform the
1990 // actual sink. We iteratively sink the last non-terminator of the source
1991 // blocks into their common successor unless doing so would require too
1992 // many PHI instructions to be generated (currently only one PHI is allowed
1993 // per sunk instruction).
1994 //
1995 // We can use InstructionsToSink to discount values needing PHI-merging that will
1996 // actually be sunk in a later iteration. This allows us to be more
1997 // aggressive in what we sink. This does allow a false positive where we
1998 // sink presuming a later value will also be sunk, but stop half way through
1999 // and never actually sink it which means we produce more PHIs than intended.
2000 // This is unlikely in practice though.
2001 unsigned SinkIdx = 0;
2002 for (; SinkIdx != ScanIdx; ++SinkIdx) {
2003 LLVM_DEBUG(dbgs() << "SINK: Sink: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds
[0]->getTerminator()->getPrevNode() << "\n"; } } while
(false)
2004 << *UnconditionalPreds[0]->getTerminator()->getPrevNode()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds
[0]->getTerminator()->getPrevNode() << "\n"; } } while
(false)
2005 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds
[0]->getTerminator()->getPrevNode() << "\n"; } } while
(false)
;
2006
2007 // Because we've sunk every instruction in turn, the current instruction to
2008 // sink is always at index 0.
2009 LRI.reset();
2010 if (!ProfitableToSinkInstruction(LRI)) {
2011 // Too many PHIs would be created.
2012 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, too many PHIs would be created!\n"
; } } while (false)
2013 dbgs() << "SINK: stopping here, too many PHIs would be created!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, too many PHIs would be created!\n"
; } } while (false)
;
2014 break;
2015 }
2016
2017 if (!sinkLastInstruction(UnconditionalPreds)) {
2018 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, failed to actually sink instruction!\n"
; } } while (false)
2019 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, failed to actually sink instruction!\n"
; } } while (false)
2020 << "SINK: stopping here, failed to actually sink instruction!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, failed to actually sink instruction!\n"
; } } while (false)
;
2021 break;
2022 }
2023
2024 NumSinkCommonInstrs++;
2025 Changed = true;
2026 }
2027 if (SinkIdx != 0)
2028 ++NumSinkCommonCode;
2029 return Changed;
2030}
2031
2032/// Determine if we can hoist sink a sole store instruction out of a
2033/// conditional block.
2034///
2035/// We are looking for code like the following:
2036/// BrBB:
2037/// store i32 %add, i32* %arrayidx2
2038/// ... // No other stores or function calls (we could be calling a memory
2039/// ... // function).
2040/// %cmp = icmp ult %x, %y
2041/// br i1 %cmp, label %EndBB, label %ThenBB
2042/// ThenBB:
2043/// store i32 %add5, i32* %arrayidx2
2044/// br label EndBB
2045/// EndBB:
2046/// ...
2047/// We are going to transform this into:
2048/// BrBB:
2049/// store i32 %add, i32* %arrayidx2
2050/// ... //
2051/// %cmp = icmp ult %x, %y
2052/// %add.add5 = select i1 %cmp, i32 %add, %add5
2053/// store i32 %add.add5, i32* %arrayidx2
2054/// ...
2055///
2056/// \return The pointer to the value of the previous store if the store can be
2057/// hoisted into the predecessor block. 0 otherwise.
2058static Value *isSafeToSpeculateStore(Instruction *I, BasicBlock *BrBB,
2059 BasicBlock *StoreBB, BasicBlock *EndBB) {
2060 StoreInst *StoreToHoist = dyn_cast<StoreInst>(I);
2061 if (!StoreToHoist)
2062 return nullptr;
2063
2064 // Volatile or atomic.
2065 if (!StoreToHoist->isSimple())
2066 return nullptr;
2067
2068 Value *StorePtr = StoreToHoist->getPointerOperand();
2069
2070 // Look for a store to the same pointer in BrBB.
2071 unsigned MaxNumInstToLookAt = 9;
2072 // Skip pseudo probe intrinsic calls which are not really killing any memory
2073 // accesses.
2074 for (Instruction &CurI : reverse(BrBB->instructionsWithoutDebug(true))) {
2075 if (!MaxNumInstToLookAt)
2076 break;
2077 --MaxNumInstToLookAt;
2078
2079 // Could be calling an instruction that affects memory like free().
2080 if (CurI.mayHaveSideEffects() && !isa<StoreInst>(CurI))
2081 return nullptr;
2082
2083 if (auto *SI = dyn_cast<StoreInst>(&CurI)) {
2084 // Found the previous store make sure it stores to the same location.
2085 if (SI->getPointerOperand() == StorePtr)
2086 // Found the previous store, return its value operand.
2087 return SI->getValueOperand();
2088 return nullptr; // Unknown store.
2089 }
2090 }
2091
2092 return nullptr;
2093}
2094
2095/// Estimate the cost of the insertion(s) and check that the PHI nodes can be
2096/// converted to selects.
2097static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB,
2098 BasicBlock *EndBB,
2099 unsigned &SpeculatedInstructions,
2100 int &BudgetRemaining,
2101 const TargetTransformInfo &TTI) {
2102 TargetTransformInfo::TargetCostKind CostKind =
2103 BB->getParent()->hasMinSize()
2104 ? TargetTransformInfo::TCK_CodeSize
2105 : TargetTransformInfo::TCK_SizeAndLatency;
2106
2107 bool HaveRewritablePHIs = false;
2108 for (PHINode &PN : EndBB->phis()) {
2109 Value *OrigV = PN.getIncomingValueForBlock(BB);
2110 Value *ThenV = PN.getIncomingValueForBlock(ThenBB);
2111
2112 // FIXME: Try to remove some of the duplication with HoistThenElseCodeToIf.
2113 // Skip PHIs which are trivial.
2114 if (ThenV == OrigV)
2115 continue;
2116
2117 BudgetRemaining -=
2118 TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr,
2119 CmpInst::BAD_ICMP_PREDICATE, CostKind);
2120
2121 // Don't convert to selects if we could remove undefined behavior instead.
2122 if (passingValueIsAlwaysUndefined(OrigV, &PN) ||
2123 passingValueIsAlwaysUndefined(ThenV, &PN))
2124 return false;
2125
2126 HaveRewritablePHIs = true;
2127 ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV);
2128 ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV);
2129 if (!OrigCE && !ThenCE)
2130 continue; // Known safe and cheap.
2131
2132 if ((ThenCE && !isSafeToSpeculativelyExecute(ThenCE)) ||
2133 (OrigCE && !isSafeToSpeculativelyExecute(OrigCE)))
2134 return false;
2135 unsigned OrigCost = OrigCE ? ComputeSpeculationCost(OrigCE, TTI) : 0;
2136 unsigned ThenCost = ThenCE ? ComputeSpeculationCost(ThenCE, TTI) : 0;
2137 unsigned MaxCost =
2138 2 * PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2139 if (OrigCost + ThenCost > MaxCost)
2140 return false;
2141
2142 // Account for the cost of an unfolded ConstantExpr which could end up
2143 // getting expanded into Instructions.
2144 // FIXME: This doesn't account for how many operations are combined in the
2145 // constant expression.
2146 ++SpeculatedInstructions;
2147 if (SpeculatedInstructions > 1)
2148 return false;
2149 }
2150
2151 return HaveRewritablePHIs;
2152}
2153
2154/// Speculate a conditional basic block flattening the CFG.
2155///
2156/// Note that this is a very risky transform currently. Speculating
2157/// instructions like this is most often not desirable. Instead, there is an MI
2158/// pass which can do it with full awareness of the resource constraints.
2159/// However, some cases are "obvious" and we should do directly. An example of
2160/// this is speculating a single, reasonably cheap instruction.
2161///
2162/// There is only one distinct advantage to flattening the CFG at the IR level:
2163/// it makes very common but simplistic optimizations such as are common in
2164/// instcombine and the DAG combiner more powerful by removing CFG edges and
2165/// modeling their effects with easier to reason about SSA value graphs.
2166///
2167///
2168/// An illustration of this transform is turning this IR:
2169/// \code
2170/// BB:
2171/// %cmp = icmp ult %x, %y
2172/// br i1 %cmp, label %EndBB, label %ThenBB
2173/// ThenBB:
2174/// %sub = sub %x, %y
2175/// br label BB2
2176/// EndBB:
2177/// %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ]
2178/// ...
2179/// \endcode
2180///
2181/// Into this IR:
2182/// \code
2183/// BB:
2184/// %cmp = icmp ult %x, %y
2185/// %sub = sub %x, %y
2186/// %cond = select i1 %cmp, 0, %sub
2187/// ...
2188/// \endcode
2189///
2190/// \returns true if the conditional block is removed.
2191bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
2192 const TargetTransformInfo &TTI) {
2193 // Be conservative for now. FP select instruction can often be expensive.
2194 Value *BrCond = BI->getCondition();
2195 if (isa<FCmpInst>(BrCond))
2196 return false;
2197
2198 BasicBlock *BB = BI->getParent();
2199 BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0);
2200 int BudgetRemaining =
2201 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2202
2203 // If ThenBB is actually on the false edge of the conditional branch, remember
2204 // to swap the select operands later.
2205 bool Invert = false;
2206 if (ThenBB != BI->getSuccessor(0)) {
2207 assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?")((ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?"
) ? static_cast<void> (0) : __assert_fail ("ThenBB == BI->getSuccessor(1) && \"No edge from 'if' block?\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2207, __PRETTY_FUNCTION__))
;
2208 Invert = true;
2209 }
2210 assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block")((EndBB == BI->getSuccessor(!Invert) && "No edge from to end block"
) ? static_cast<void> (0) : __assert_fail ("EndBB == BI->getSuccessor(!Invert) && \"No edge from to end block\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2210, __PRETTY_FUNCTION__))
;
2211
2212 // Keep a count of how many times instructions are used within ThenBB when
2213 // they are candidates for sinking into ThenBB. Specifically:
2214 // - They are defined in BB, and
2215 // - They have no side effects, and
2216 // - All of their uses are in ThenBB.
2217 SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts;
2218
2219 SmallVector<Instruction *, 4> SpeculatedDbgIntrinsics;
2220
2221 unsigned SpeculatedInstructions = 0;
2222 Value *SpeculatedStoreValue = nullptr;
2223 StoreInst *SpeculatedStore = nullptr;
2224 for (BasicBlock::iterator BBI = ThenBB->begin(),
2225 BBE = std::prev(ThenBB->end());
2226 BBI != BBE; ++BBI) {
2227 Instruction *I = &*BBI;
2228 // Skip debug info.
2229 if (isa<DbgInfoIntrinsic>(I)) {
2230 SpeculatedDbgIntrinsics.push_back(I);
2231 continue;
2232 }
2233
2234 // Skip pseudo probes. The consequence is we lose track of the branch
2235 // probability for ThenBB, which is fine since the optimization here takes
2236 // place regardless of the branch probability.
2237 if (isa<PseudoProbeInst>(I)) {
2238 SpeculatedDbgIntrinsics.push_back(I);
2239 continue;
2240 }
2241
2242 // Only speculatively execute a single instruction (not counting the
2243 // terminator) for now.
2244 ++SpeculatedInstructions;
2245 if (SpeculatedInstructions > 1)
2246 return false;
2247
2248 // Don't hoist the instruction if it's unsafe or expensive.
2249 if (!isSafeToSpeculativelyExecute(I) &&
2250 !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore(
2251 I, BB, ThenBB, EndBB))))
2252 return false;
2253 if (!SpeculatedStoreValue &&
2254 ComputeSpeculationCost(I, TTI) >
2255 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic)
2256 return false;
2257
2258 // Store the store speculation candidate.
2259 if (SpeculatedStoreValue)
2260 SpeculatedStore = cast<StoreInst>(I);
2261
2262 // Do not hoist the instruction if any of its operands are defined but not
2263 // used in BB. The transformation will prevent the operand from
2264 // being sunk into the use block.
2265 for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i) {
2266 Instruction *OpI = dyn_cast<Instruction>(*i);
2267 if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects())
2268 continue; // Not a candidate for sinking.
2269
2270 ++SinkCandidateUseCounts[OpI];
2271 }
2272 }
2273
2274 // Consider any sink candidates which are only used in ThenBB as costs for
2275 // speculation. Note, while we iterate over a DenseMap here, we are summing
2276 // and so iteration order isn't significant.
2277 for (SmallDenseMap<Instruction *, unsigned, 4>::iterator
2278 I = SinkCandidateUseCounts.begin(),
2279 E = SinkCandidateUseCounts.end();
2280 I != E; ++I)
2281 if (I->first->hasNUses(I->second)) {
2282 ++SpeculatedInstructions;
2283 if (SpeculatedInstructions > 1)
2284 return false;
2285 }
2286
2287 // Check that we can insert the selects and that it's not too expensive to do
2288 // so.
2289 bool Convert = SpeculatedStore != nullptr;
2290 Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB,
2291 SpeculatedInstructions,
2292 BudgetRemaining, TTI);
2293 if (!Convert || BudgetRemaining < 0)
2294 return false;
2295
2296 // If we get here, we can hoist the instruction and if-convert.
2297 LLVM_DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SPECULATIVELY EXECUTING BB"
<< *ThenBB << "\n";; } } while (false)
;
2298
2299 // Insert a select of the value of the speculated store.
2300 if (SpeculatedStoreValue) {
2301 IRBuilder<NoFolder> Builder(BI);
2302 Value *TrueV = SpeculatedStore->getValueOperand();
2303 Value *FalseV = SpeculatedStoreValue;
2304 if (Invert)
2305 std::swap(TrueV, FalseV);
2306 Value *S = Builder.CreateSelect(
2307 BrCond, TrueV, FalseV, "spec.store.select", BI);
2308 SpeculatedStore->setOperand(0, S);
2309 SpeculatedStore->applyMergedLocation(BI->getDebugLoc(),
2310 SpeculatedStore->getDebugLoc());
2311 }
2312
2313 // Metadata can be dependent on the condition we are hoisting above.
2314 // Conservatively strip all metadata on the instruction. Drop the debug loc
2315 // to avoid making it appear as if the condition is a constant, which would
2316 // be misleading while debugging.
2317 for (auto &I : *ThenBB) {
2318 if (!SpeculatedStoreValue || &I != SpeculatedStore)
2319 I.setDebugLoc(DebugLoc());
2320 I.dropUnknownNonDebugMetadata();
2321 }
2322
2323 // Hoist the instructions.
2324 BB->getInstList().splice(BI->getIterator(), ThenBB->getInstList(),
2325 ThenBB->begin(), std::prev(ThenBB->end()));
2326
2327 // Insert selects and rewrite the PHI operands.
2328 IRBuilder<NoFolder> Builder(BI);
2329 for (PHINode &PN : EndBB->phis()) {
2330 unsigned OrigI = PN.getBasicBlockIndex(BB);
2331 unsigned ThenI = PN.getBasicBlockIndex(ThenBB);
2332 Value *OrigV = PN.getIncomingValue(OrigI);
2333 Value *ThenV = PN.getIncomingValue(ThenI);
2334
2335 // Skip PHIs which are trivial.
2336 if (OrigV == ThenV)
2337 continue;
2338
2339 // Create a select whose true value is the speculatively executed value and
2340 // false value is the pre-existing value. Swap them if the branch
2341 // destinations were inverted.
2342 Value *TrueV = ThenV, *FalseV = OrigV;
2343 if (Invert)
2344 std::swap(TrueV, FalseV);
2345 Value *V = Builder.CreateSelect(BrCond, TrueV, FalseV, "spec.select", BI);
2346 PN.setIncomingValue(OrigI, V);
2347 PN.setIncomingValue(ThenI, V);
2348 }
2349
2350 // Remove speculated dbg intrinsics.
2351 // FIXME: Is it possible to do this in a more elegant way? Moving/merging the
2352 // dbg value for the different flows and inserting it after the select.
2353 for (Instruction *I : SpeculatedDbgIntrinsics)
2354 I->eraseFromParent();
2355
2356 ++NumSpeculations;
2357 return true;
2358}
2359
2360/// Return true if we can thread a branch across this block.
2361static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
2362 int Size = 0;
2363
2364 for (Instruction &I : BB->instructionsWithoutDebug()) {
2365 if (Size > MaxSmallBlockSize)
2366 return false; // Don't clone large BB's.
2367
2368 // Can't fold blocks that contain noduplicate or convergent calls.
2369 if (CallInst *CI = dyn_cast<CallInst>(&I))
2370 if (CI->cannotDuplicate() || CI->isConvergent())
2371 return false;
2372
2373 // We will delete Phis while threading, so Phis should not be accounted in
2374 // block's size
2375 if (!isa<PHINode>(I))
2376 ++Size;
2377
2378 // We can only support instructions that do not define values that are
2379 // live outside of the current basic block.
2380 for (User *U : I.users()) {
2381 Instruction *UI = cast<Instruction>(U);
2382 if (UI->getParent() != BB || isa<PHINode>(UI))
2383 return false;
2384 }
2385
2386 // Looks ok, continue checking.
2387 }
2388
2389 return true;
2390}
2391
2392/// If we have a conditional branch on a PHI node value that is defined in the
2393/// same block as the branch and if any PHI entries are constants, thread edges
2394/// corresponding to that entry to be branches to their ultimate destination.
2395static bool FoldCondBranchOnPHI(BranchInst *BI, DomTreeUpdater *DTU,
2396 const DataLayout &DL, AssumptionCache *AC) {
2397 BasicBlock *BB = BI->getParent();
2398 PHINode *PN = dyn_cast<PHINode>(BI->getCondition());
2399 // NOTE: we currently cannot transform this case if the PHI node is used
2400 // outside of the block.
2401 if (!PN || PN->getParent() != BB || !PN->hasOneUse())
2402 return false;
2403
2404 // Degenerate case of a single entry PHI.
2405 if (PN->getNumIncomingValues() == 1) {
2406 FoldSingleEntryPHINodes(PN->getParent());
2407 return true;
2408 }
2409
2410 // Now we know that this block has multiple preds and two succs.
2411 if (!BlockIsSimpleEnoughToThreadThrough(BB))
2412 return false;
2413
2414 // Okay, this is a simple enough basic block. See if any phi values are
2415 // constants.
2416 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
2417 ConstantInt *CB = dyn_cast<ConstantInt>(PN->getIncomingValue(i));
2418 if (!CB || !CB->getType()->isIntegerTy(1))
2419 continue;
2420
2421 // Okay, we now know that all edges from PredBB should be revectored to
2422 // branch to RealDest.
2423 BasicBlock *PredBB = PN->getIncomingBlock(i);
2424 BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue());
2425
2426 if (RealDest == BB)
2427 continue; // Skip self loops.
2428 // Skip if the predecessor's terminator is an indirect branch.
2429 if (isa<IndirectBrInst>(PredBB->getTerminator()))
2430 continue;
2431
2432 SmallVector<DominatorTree::UpdateType, 3> Updates;
2433
2434 // The dest block might have PHI nodes, other predecessors and other
2435 // difficult cases. Instead of being smart about this, just insert a new
2436 // block that jumps to the destination block, effectively splitting
2437 // the edge we are about to create.
2438 BasicBlock *EdgeBB =
2439 BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge",
2440 RealDest->getParent(), RealDest);
2441 BranchInst *CritEdgeBranch = BranchInst::Create(RealDest, EdgeBB);
2442 Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest});
2443 CritEdgeBranch->setDebugLoc(BI->getDebugLoc());
2444
2445 // Update PHI nodes.
2446 AddPredecessorToBlock(RealDest, EdgeBB, BB);
2447
2448 // BB may have instructions that are being threaded over. Clone these
2449 // instructions into EdgeBB. We know that there will be no uses of the
2450 // cloned instructions outside of EdgeBB.
2451 BasicBlock::iterator InsertPt = EdgeBB->begin();
2452 DenseMap<Value *, Value *> TranslateMap; // Track translated values.
2453 for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
2454 if (PHINode *PN = dyn_cast<PHINode>(BBI)) {
2455 TranslateMap[PN] = PN->getIncomingValueForBlock(PredBB);
2456 continue;
2457 }
2458 // Clone the instruction.
2459 Instruction *N = BBI->clone();
2460 if (BBI->hasName())
2461 N->setName(BBI->getName() + ".c");
2462
2463 // Update operands due to translation.
2464 for (User::op_iterator i = N->op_begin(), e = N->op_end(); i != e; ++i) {
2465 DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(*i);
2466 if (PI != TranslateMap.end())
2467 *i = PI->second;
2468 }
2469
2470 // Check for trivial simplification.
2471 if (Value *V = SimplifyInstruction(N, {DL, nullptr, nullptr, AC})) {
2472 if (!BBI->use_empty())
2473 TranslateMap[&*BBI] = V;
2474 if (!N->mayHaveSideEffects()) {
2475 N->deleteValue(); // Instruction folded away, don't need actual inst
2476 N = nullptr;
2477 }
2478 } else {
2479 if (!BBI->use_empty())
2480 TranslateMap[&*BBI] = N;
2481 }
2482 if (N) {
2483 // Insert the new instruction into its new home.
2484 EdgeBB->getInstList().insert(InsertPt, N);
2485
2486 // Register the new instruction with the assumption cache if necessary.
2487 if (AC && match(N, m_Intrinsic<Intrinsic::assume>()))
2488 AC->registerAssumption(cast<IntrinsicInst>(N));
2489 }
2490 }
2491
2492 // Loop over all of the edges from PredBB to BB, changing them to branch
2493 // to EdgeBB instead.
2494 Instruction *PredBBTI = PredBB->getTerminator();
2495 for (unsigned i = 0, e = PredBBTI->getNumSuccessors(); i != e; ++i)
2496 if (PredBBTI->getSuccessor(i) == BB) {
2497 BB->removePredecessor(PredBB);
2498 PredBBTI->setSuccessor(i, EdgeBB);
2499 }
2500
2501 Updates.push_back({DominatorTree::Insert, PredBB, EdgeBB});
2502 Updates.push_back({DominatorTree::Delete, PredBB, BB});
2503
2504 if (DTU)
2505 DTU->applyUpdates(Updates);
2506
2507 // Recurse, simplifying any other constants.
2508 return FoldCondBranchOnPHI(BI, DTU, DL, AC) || true;
2509 }
2510
2511 return false;
2512}
2513
2514/// Given a BB that starts with the specified two-entry PHI node,
2515/// see if we can eliminate it.
2516static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
2517 DomTreeUpdater *DTU, const DataLayout &DL) {
2518 // Ok, this is a two entry PHI node. Check to see if this is a simple "if
2519 // statement", which has a very simple dominance structure. Basically, we
2520 // are trying to find the condition that is being branched on, which
2521 // subsequently causes this merge to happen. We really want control
2522 // dependence information for this check, but simplifycfg can't keep it up
2523 // to date, and this catches most of the cases we care about anyway.
2524 BasicBlock *BB = PN->getParent();
2525
2526 BasicBlock *IfTrue, *IfFalse;
2527 Value *IfCond = GetIfCondition(BB, IfTrue, IfFalse);
2528 if (!IfCond ||
2529 // Don't bother if the branch will be constant folded trivially.
2530 isa<ConstantInt>(IfCond))
2531 return false;
2532
2533 // Okay, we found that we can merge this two-entry phi node into a select.
2534 // Doing so would require us to fold *all* two entry phi nodes in this block.
2535 // At some point this becomes non-profitable (particularly if the target
2536 // doesn't support cmov's). Only do this transformation if there are two or
2537 // fewer PHI nodes in this block.
2538 unsigned NumPhis = 0;
2539 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I)
2540 if (NumPhis > 2)
2541 return false;
2542
2543 // Loop over the PHI's seeing if we can promote them all to select
2544 // instructions. While we are at it, keep track of the instructions
2545 // that need to be moved to the dominating block.
2546 SmallPtrSet<Instruction *, 4> AggressiveInsts;
2547 int BudgetRemaining =
2548 TwoEntryPHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2549
2550 bool Changed = false;
2551 for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
2552 PHINode *PN = cast<PHINode>(II++);
2553 if (Value *V = SimplifyInstruction(PN, {DL, PN})) {
2554 PN->replaceAllUsesWith(V);
2555 PN->eraseFromParent();
2556 Changed = true;
2557 continue;
2558 }
2559
2560 if (!DominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts,
2561 BudgetRemaining, TTI) ||
2562 !DominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts,
2563 BudgetRemaining, TTI))
2564 return Changed;
2565 }
2566
2567 // If we folded the first phi, PN dangles at this point. Refresh it. If
2568 // we ran out of PHIs then we simplified them all.
2569 PN = dyn_cast<PHINode>(BB->begin());
2570 if (!PN)
2571 return true;
2572
2573 // Return true if at least one of these is a 'not', and another is either
2574 // a 'not' too, or a constant.
2575 auto CanHoistNotFromBothValues = [](Value *V0, Value *V1) {
2576 if (!match(V0, m_Not(m_Value())))
2577 std::swap(V0, V1);
2578 auto Invertible = m_CombineOr(m_Not(m_Value()), m_AnyIntegralConstant());
2579 return match(V0, m_Not(m_Value())) && match(V1, Invertible);
2580 };
2581
2582 // Don't fold i1 branches on PHIs which contain binary operators, unless one
2583 // of the incoming values is an 'not' and another one is freely invertible.
2584 // These can often be turned into switches and other things.
2585 if (PN->getType()->isIntegerTy(1) &&
2586 (isa<BinaryOperator>(PN->getIncomingValue(0)) ||
2587 isa<BinaryOperator>(PN->getIncomingValue(1)) ||
2588 isa<BinaryOperator>(IfCond)) &&
2589 !CanHoistNotFromBothValues(PN->getIncomingValue(0),
2590 PN->getIncomingValue(1)))
2591 return Changed;
2592
2593 // If all PHI nodes are promotable, check to make sure that all instructions
2594 // in the predecessor blocks can be promoted as well. If not, we won't be able
2595 // to get rid of the control flow, so it's not worth promoting to select
2596 // instructions.
2597 BasicBlock *DomBlock = nullptr;
2598 BasicBlock *IfBlock1 = PN->getIncomingBlock(0);
2599 BasicBlock *IfBlock2 = PN->getIncomingBlock(1);
2600 if (cast<BranchInst>(IfBlock1->getTerminator())->isConditional()) {
2601 IfBlock1 = nullptr;
2602 } else {
2603 DomBlock = *pred_begin(IfBlock1);
2604 for (BasicBlock::iterator I = IfBlock1->begin(); !I->isTerminator(); ++I)
2605 if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I) &&
2606 !isa<PseudoProbeInst>(I)) {
2607 // This is not an aggressive instruction that we can promote.
2608 // Because of this, we won't be able to get rid of the control flow, so
2609 // the xform is not worth it.
2610 return Changed;
2611 }
2612 }
2613
2614 if (cast<BranchInst>(IfBlock2->getTerminator())->isConditional()) {
2615 IfBlock2 = nullptr;
2616 } else {
2617 DomBlock = *pred_begin(IfBlock2);
2618 for (BasicBlock::iterator I = IfBlock2->begin(); !I->isTerminator(); ++I)
2619 if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I) &&
2620 !isa<PseudoProbeInst>(I)) {
2621 // This is not an aggressive instruction that we can promote.
2622 // Because of this, we won't be able to get rid of the control flow, so
2623 // the xform is not worth it.
2624 return Changed;
2625 }
2626 }
2627 assert(DomBlock && "Failed to find root DomBlock")((DomBlock && "Failed to find root DomBlock") ? static_cast
<void> (0) : __assert_fail ("DomBlock && \"Failed to find root DomBlock\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2627, __PRETTY_FUNCTION__))
;
2628
2629 LLVM_DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfConddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " <<
*IfCond << " T: " << IfTrue->getName() <<
" F: " << IfFalse->getName() << "\n"; } } while
(false)
2630 << " T: " << IfTrue->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " <<
*IfCond << " T: " << IfTrue->getName() <<
" F: " << IfFalse->getName() << "\n"; } } while
(false)
2631 << " F: " << IfFalse->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " <<
*IfCond << " T: " << IfTrue->getName() <<
" F: " << IfFalse->getName() << "\n"; } } while
(false)
;
2632
2633 // If we can still promote the PHI nodes after this gauntlet of tests,
2634 // do all of the PHI's now.
2635 Instruction *InsertPt = DomBlock->getTerminator();
2636 IRBuilder<NoFolder> Builder(InsertPt);
2637
2638 // Move all 'aggressive' instructions, which are defined in the
2639 // conditional parts of the if's up to the dominating block.
2640 if (IfBlock1)
2641 hoistAllInstructionsInto(DomBlock, InsertPt, IfBlock1);
2642 if (IfBlock2)
2643 hoistAllInstructionsInto(DomBlock, InsertPt, IfBlock2);
2644
2645 // Propagate fast-math-flags from phi nodes to replacement selects.
2646 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2647 while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
2648 if (isa<FPMathOperator>(PN))
2649 Builder.setFastMathFlags(PN->getFastMathFlags());
2650
2651 // Change the PHI node into a select instruction.
2652 Value *TrueVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfFalse);
2653 Value *FalseVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfTrue);
2654
2655 Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", InsertPt);
2656 PN->replaceAllUsesWith(Sel);
2657 Sel->takeName(PN);
2658 PN->eraseFromParent();
2659 }
2660
2661 // At this point, IfBlock1 and IfBlock2 are both empty, so our if statement
2662 // has been flattened. Change DomBlock to jump directly to our new block to
2663 // avoid other simplifycfg's kicking in on the diamond.
2664 Instruction *OldTI = DomBlock->getTerminator();
2665 Builder.SetInsertPoint(OldTI);
2666 Builder.CreateBr(BB);
2667
2668 SmallVector<DominatorTree::UpdateType, 3> Updates;
2669 if (DTU) {
2670 Updates.push_back({DominatorTree::Insert, DomBlock, BB});
2671 for (auto *Successor : successors(DomBlock))
2672 Updates.push_back({DominatorTree::Delete, DomBlock, Successor});
2673 }
2674
2675 OldTI->eraseFromParent();
2676 if (DTU)
2677 DTU->applyUpdates(Updates);
2678
2679 return true;
2680}
2681
2682/// If we found a conditional branch that goes to two returning blocks,
2683/// try to merge them together into one return,
2684/// introducing a select if the return values disagree.
2685bool SimplifyCFGOpt::SimplifyCondBranchToTwoReturns(BranchInst *BI,
2686 IRBuilder<> &Builder) {
2687 auto *BB = BI->getParent();
2688 assert(BI->isConditional() && "Must be a conditional branch")((BI->isConditional() && "Must be a conditional branch"
) ? static_cast<void> (0) : __assert_fail ("BI->isConditional() && \"Must be a conditional branch\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2688, __PRETTY_FUNCTION__))
;
2689 BasicBlock *TrueSucc = BI->getSuccessor(0);
2690 BasicBlock *FalseSucc = BI->getSuccessor(1);
2691 // NOTE: destinations may match, this could be degenerate uncond branch.
2692 ReturnInst *TrueRet = cast<ReturnInst>(TrueSucc->getTerminator());
2693 ReturnInst *FalseRet = cast<ReturnInst>(FalseSucc->getTerminator());
2694
2695 // Check to ensure both blocks are empty (just a return) or optionally empty
2696 // with PHI nodes. If there are other instructions, merging would cause extra
2697 // computation on one path or the other.
2698 if (!TrueSucc->getFirstNonPHIOrDbg()->isTerminator())
2699 return false;
2700 if (!FalseSucc->getFirstNonPHIOrDbg()->isTerminator())
2701 return false;
2702
2703 Builder.SetInsertPoint(BI);
2704 // Okay, we found a branch that is going to two return nodes. If
2705 // there is no return value for this function, just change the
2706 // branch into a return.
2707 if (FalseRet->getNumOperands() == 0) {
2708 TrueSucc->removePredecessor(BB);
2709 FalseSucc->removePredecessor(BB);
2710 Builder.CreateRetVoid();
2711 EraseTerminatorAndDCECond(BI);
2712 if (DTU) {
2713 SmallVector<DominatorTree::UpdateType, 2> Updates;
2714 Updates.push_back({DominatorTree::Delete, BB, TrueSucc});
2715 if (TrueSucc != FalseSucc)
2716 Updates.push_back({DominatorTree::Delete, BB, FalseSucc});
2717 DTU->applyUpdates(Updates);
2718 }
2719 return true;
2720 }
2721
2722 // Otherwise, figure out what the true and false return values are
2723 // so we can insert a new select instruction.
2724 Value *TrueValue = TrueRet->getReturnValue();
2725 Value *FalseValue = FalseRet->getReturnValue();
2726
2727 // Unwrap any PHI nodes in the return blocks.
2728 if (PHINode *TVPN = dyn_cast_or_null<PHINode>(TrueValue))
2729 if (TVPN->getParent() == TrueSucc)
2730 TrueValue = TVPN->getIncomingValueForBlock(BB);
2731 if (PHINode *FVPN = dyn_cast_or_null<PHINode>(FalseValue))
2732 if (FVPN->getParent() == FalseSucc)
2733 FalseValue = FVPN->getIncomingValueForBlock(BB);
2734
2735 // In order for this transformation to be safe, we must be able to
2736 // unconditionally execute both operands to the return. This is
2737 // normally the case, but we could have a potentially-trapping
2738 // constant expression that prevents this transformation from being
2739 // safe.
2740 if (ConstantExpr *TCV = dyn_cast_or_null<ConstantExpr>(TrueValue))
2741 if (TCV->canTrap())
2742 return false;
2743 if (ConstantExpr *FCV = dyn_cast_or_null<ConstantExpr>(FalseValue))
2744 if (FCV->canTrap())
2745 return false;
2746
2747 // Okay, we collected all the mapped values and checked them for sanity, and
2748 // defined to really do this transformation. First, update the CFG.
2749 TrueSucc->removePredecessor(BB);
2750 FalseSucc->removePredecessor(BB);
2751
2752 // Insert select instructions where needed.
2753 Value *BrCond = BI->getCondition();
2754 if (TrueValue) {
2755 // Insert a select if the results differ.
2756 if (TrueValue == FalseValue || isa<UndefValue>(FalseValue)) {
2757 } else if (isa<UndefValue>(TrueValue)) {
2758 TrueValue = FalseValue;
2759 } else {
2760 TrueValue =
2761 Builder.CreateSelect(BrCond, TrueValue, FalseValue, "retval", BI);
2762 }
2763 }
2764
2765 Value *RI =
2766 !TrueValue ? Builder.CreateRetVoid() : Builder.CreateRet(TrueValue);
2767
2768 (void)RI;
2769
2770 LLVM_DEBUG(dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
<< "\n " << *BI << "\nNewRet = " <<
*RI << "\nTRUEBLOCK: " << *TrueSucc << "\nFALSEBLOCK: "
<< *FalseSucc; } } while (false)
2771 << "\n " << *BI << "\nNewRet = " << *RI << "\nTRUEBLOCK: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
<< "\n " << *BI << "\nNewRet = " <<
*RI << "\nTRUEBLOCK: " << *TrueSucc << "\nFALSEBLOCK: "
<< *FalseSucc; } } while (false)
2772 << *TrueSucc << "\nFALSEBLOCK: " << *FalseSucc)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
<< "\n " << *BI << "\nNewRet = " <<
*RI << "\nTRUEBLOCK: " << *TrueSucc << "\nFALSEBLOCK: "
<< *FalseSucc; } } while (false)
;
2773
2774 EraseTerminatorAndDCECond(BI);
2775 if (DTU) {
2776 SmallVector<DominatorTree::UpdateType, 2> Updates;
2777 Updates.push_back({DominatorTree::Delete, BB, TrueSucc});
2778 if (TrueSucc != FalseSucc)
2779 Updates.push_back({DominatorTree::Delete, BB, FalseSucc});
2780 DTU->applyUpdates(Updates);
2781 }
2782
2783 return true;
2784}
2785
2786/// Return true if the given instruction is available
2787/// in its predecessor block. If yes, the instruction will be removed.
2788static bool tryCSEWithPredecessor(Instruction *Inst, BasicBlock *PB) {
2789 if (!isa<BinaryOperator>(Inst) && !isa<CmpInst>(Inst))
2790 return false;
2791 for (Instruction &I : *PB) {
2792 Instruction *PBI = &I;
2793 // Check whether Inst and PBI generate the same value.
2794 if (Inst->isIdenticalTo(PBI)) {
2795 Inst->replaceAllUsesWith(PBI);
2796 Inst->eraseFromParent();
2797 return true;
2798 }
2799 }
2800 return false;
2801}
2802
2803/// Return true if either PBI or BI has branch weight available, and store
2804/// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does
2805/// not have branch weight, use 1:1 as its weight.
2806static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI,
2807 uint64_t &PredTrueWeight,
2808 uint64_t &PredFalseWeight,
2809 uint64_t &SuccTrueWeight,
2810 uint64_t &SuccFalseWeight) {
2811 bool PredHasWeights =
2812 PBI->extractProfMetadata(PredTrueWeight, PredFalseWeight);
2813 bool SuccHasWeights =
2814 BI->extractProfMetadata(SuccTrueWeight, SuccFalseWeight);
2815 if (PredHasWeights || SuccHasWeights) {
2816 if (!PredHasWeights)
2817 PredTrueWeight = PredFalseWeight = 1;
2818 if (!SuccHasWeights)
2819 SuccTrueWeight = SuccFalseWeight = 1;
2820 return true;
2821 } else {
2822 return false;
2823 }
2824}
2825
2826/// If this basic block is simple enough, and if a predecessor branches to us
2827/// and one of our successors, fold the block into the predecessor and use
2828/// logical operations to pick the right destination.
2829bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
2830 MemorySSAUpdater *MSSAU,
2831 const TargetTransformInfo *TTI,
2832 unsigned BonusInstThreshold) {
2833 BasicBlock *BB = BI->getParent();
2834
2835 const unsigned PredCount = pred_size(BB);
2836
2837 bool Changed = false;
2838
2839 auto _ = make_scope_exit([&]() {
2840 if (Changed)
2841 ++NumFoldBranchToCommonDest;
2842 });
2843
2844 TargetTransformInfo::TargetCostKind CostKind =
2845 BB->getParent()->hasMinSize() ? TargetTransformInfo::TCK_CodeSize
1
Assuming the condition is false
2
'?' condition is false
2846 : TargetTransformInfo::TCK_SizeAndLatency;
2847
2848 Instruction *Cond = nullptr;
2849 if (BI->isConditional())
3
Calling 'BranchInst::isConditional'
6
Returning from 'BranchInst::isConditional'
7
Taking true branch
2850 Cond = dyn_cast<Instruction>(BI->getCondition());
8
Assuming the object is a 'Instruction'
2851 else {
2852 // For unconditional branch, check for a simple CFG pattern, where
2853 // BB has a single predecessor and BB's successor is also its predecessor's
2854 // successor. If such pattern exists, check for CSE between BB and its
2855 // predecessor.
2856 if (BasicBlock *PB = BB->getSinglePredecessor())
2857 if (BranchInst *PBI = dyn_cast<BranchInst>(PB->getTerminator()))
2858 if (PBI->isConditional() &&
2859 (BI->getSuccessor(0) == PBI->getSuccessor(0) ||
2860 BI->getSuccessor(0) == PBI->getSuccessor(1))) {
2861 for (auto I = BB->instructionsWithoutDebug().begin(),
2862 E = BB->instructionsWithoutDebug().end();
2863 I != E;) {
2864 Instruction *Curr = &*I++;
2865 if (isa<CmpInst>(Curr)) {
2866 Cond = Curr;
2867 break;
2868 }
2869 // Quit if we can't remove this instruction.
2870 if (!tryCSEWithPredecessor(Curr, PB))
2871 return Changed;
2872 Changed = true;
2873 }
2874 }
2875
2876 if (!Cond)
2877 return Changed;
2878 }
2879
2880 if (!Cond
8.1
'Cond' is non-null
8.1
'Cond' is non-null
8.1
'Cond' is non-null
8.1
'Cond' is non-null
8.1
'Cond' is non-null
8.1
'Cond' is non-null
|| (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond)) ||
9
Assuming 'Cond' is not a 'CmpInst'
10
Assuming 'Cond' is a 'BinaryOperator'
20
Taking false branch
2881 Cond->getParent() != BB || !Cond->hasOneUse())
11
Assuming the condition is false
12
Calling 'Value::hasOneUse'
18
Returning from 'Value::hasOneUse'
19
Assuming the condition is false
2882 return Changed;
2883
2884 // Only allow this transformation if computing the condition doesn't involve
2885 // too many instructions and these involved instructions can be executed
2886 // unconditionally. We denote all involved instructions except the condition
2887 // as "bonus instructions", and only allow this transformation when the
2888 // number of the bonus instructions we'll need to create when cloning into
2889 // each predecessor does not exceed a certain threshold.
2890 unsigned NumBonusInsts = 0;
2891 for (Instruction &I : *BB) {
2892 // Don't check the branch condition comparison itself.
2893 if (&I == Cond)
2894 continue;
2895 // Ignore dbg intrinsics, and the terminator.
2896 if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I))
2897 continue;
2898 // I must be safe to execute unconditionally.
2899 if (!isSafeToSpeculativelyExecute(&I))
2900 return Changed;
2901
2902 // Account for the cost of duplicating this instruction into each
2903 // predecessor.
2904 NumBonusInsts += PredCount;
2905 // Early exits once we reach the limit.
2906 if (NumBonusInsts > BonusInstThreshold)
2907 return Changed;
2908 }
2909
2910 // Also, for now, all liveout uses of bonus instructions must be in PHI nodes
2911 // in successor blocks as incoming values from the bonus instructions's block,
2912 // otherwise we'll fail to update them.
2913 // FIXME: We could lift this restriction, but we need to form PHI nodes and
2914 // rewrite offending uses, but we can't do that without having a domtree.
2915 if (any_of(*BB, [BB](Instruction &I) {
21
Calling 'any_of<llvm::BasicBlock &, (lambda at /build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp:2915:19)>'
32
Returning from 'any_of<llvm::BasicBlock &, (lambda at /build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp:2915:19)>'
33
Taking false branch
2916 return any_of(I.uses(), [BB](Use &U) {
2917 auto *User = cast<Instruction>(U.getUser());
2918 if (User->getParent() == BB)
2919 return false; // Not an external use.
2920 auto *PN = dyn_cast<PHINode>(User);
2921 return !PN || PN->getIncomingBlock(U) != BB;
2922 });
2923 }))
2924 return Changed;
2925
2926 // Cond is known to be a compare or binary operator. Check to make sure that
2927 // neither operand is a potentially-trapping constant expression.
2928 if (ConstantExpr *CE
34.1
'CE' is null
34.1
'CE' is null
34.1
'CE' is null
34.1
'CE' is null
34.1
'CE' is null
34.1
'CE' is null
= dyn_cast<ConstantExpr>(Cond->getOperand(0)))
34
Assuming the object is not a 'ConstantExpr'
35
Taking false branch
2929 if (CE->canTrap())
2930 return Changed;
2931 if (ConstantExpr *CE
36.1
'CE' is null
36.1
'CE' is null
36.1
'CE' is null
36.1
'CE' is null
36.1
'CE' is null
36.1
'CE' is null
= dyn_cast<ConstantExpr>(Cond->getOperand(1)))
36
Assuming the object is not a 'ConstantExpr'
37
Taking false branch
2932 if (CE->canTrap())
2933 return Changed;
2934
2935 // Finally, don't infinitely unroll conditional loops.
2936 BasicBlock *TrueDest = BI->getSuccessor(0);
2937 BasicBlock *FalseDest = (BI->isConditional()) ? BI->getSuccessor(1) : nullptr;
38
'?' condition is true
2938 if (TrueDest
38.1
'TrueDest' is not equal to 'BB'
38.1
'TrueDest' is not equal to 'BB'
38.1
'TrueDest' is not equal to 'BB'
38.1
'TrueDest' is not equal to 'BB'
38.1
'TrueDest' is not equal to 'BB'
38.1
'TrueDest' is not equal to 'BB'
== BB || FalseDest
38.2
'FalseDest' is not equal to 'BB'
38.2
'FalseDest' is not equal to 'BB'
38.2
'FalseDest' is not equal to 'BB'
38.2
'FalseDest' is not equal to 'BB'
38.2
'FalseDest' is not equal to 'BB'
38.2
'FalseDest' is not equal to 'BB'
== BB)
39
Taking false branch
2939 return Changed;
2940
2941 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
40
Loop condition is true. Entering loop body
2942 BasicBlock *PredBlock = *PI;
2943 BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
41
Assuming the object is a 'BranchInst'
2944
2945 // Check that we have two conditional branches. If there is a PHI node in
2946 // the common successor, verify that the same value flows in from both
2947 // blocks.
2948 SmallVector<PHINode *, 4> PHIs;
2949 if (!PBI
41.1
'PBI' is non-null
41.1
'PBI' is non-null
41.1
'PBI' is non-null
41.1
'PBI' is non-null
41.1
'PBI' is non-null
41.1
'PBI' is non-null
|| PBI->isUnconditional() ||
42
Calling 'BranchInst::isUnconditional'
45
Returning from 'BranchInst::isUnconditional'
2950 (BI->isConditional() && !SafeToMergeTerminators(BI, PBI)) ||
46
Calling 'BranchInst::isConditional'
48
Returning from 'BranchInst::isConditional'
49
Calling 'SafeToMergeTerminators'
53
Returning from 'SafeToMergeTerminators'
2951 (!BI->isConditional() &&
54
Calling 'BranchInst::isConditional'
56
Returning from 'BranchInst::isConditional'
2952 !isProfitableToFoldUnconditional(BI, PBI, Cond, PHIs)))
2953 continue;
2954
2955 // Determine if the two branches share a common destination.
2956 Instruction::BinaryOps Opc = Instruction::BinaryOpsEnd;
2957 bool InvertPredCond = false;
2958
2959 if (BI->isConditional()) {
57
Calling 'BranchInst::isConditional'
59
Returning from 'BranchInst::isConditional'
60
Taking true branch
2960 if (PBI->getSuccessor(0) == TrueDest) {
61
Taking true branch
2961 Opc = Instruction::Or;
2962 } else if (PBI->getSuccessor(1) == FalseDest) {
2963 Opc = Instruction::And;
2964 } else if (PBI->getSuccessor(0) == FalseDest) {
2965 Opc = Instruction::And;
2966 InvertPredCond = true;
2967 } else if (PBI->getSuccessor(1) == TrueDest) {
2968 Opc = Instruction::Or;
2969 InvertPredCond = true;
2970 } else {
2971 continue;
2972 }
2973 } else {
2974 if (PBI->getSuccessor(0) != TrueDest && PBI->getSuccessor(1) != TrueDest)
2975 continue;
2976 }
2977
2978 // Check the cost of inserting the necessary logic before performing the
2979 // transformation.
2980 if (TTI && Opc != Instruction::BinaryOpsEnd) {
62
Assuming 'TTI' is null
2981 Type *Ty = BI->getCondition()->getType();
2982 unsigned Cost = TTI->getArithmeticInstrCost(Opc, Ty, CostKind);
2983 if (InvertPredCond && (!PBI->getCondition()->hasOneUse() ||
2984 !isa<CmpInst>(PBI->getCondition())))
2985 Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind);
2986
2987 if (Cost > BranchFoldThreshold)
2988 continue;
2989 }
2990
2991 LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING BRANCH TO COMMON DEST:\n"
<< *PBI << *BB; } } while (false)
;
63
Assuming 'DebugFlag' is false
64
Loop condition is false. Exiting loop
2992 Changed = true;
2993
2994 SmallVector<DominatorTree::UpdateType, 3> Updates;
2995
2996 IRBuilder<> Builder(PBI);
2997 // The builder is used to create instructions to eliminate the branch in BB.
2998 // If BB's terminator has !annotation metadata, add it to the new
2999 // instructions.
3000 Builder.CollectMetadataToCopy(BB->getTerminator(),
3001 {LLVMContext::MD_annotation});
3002
3003 // If we need to invert the condition in the pred block to match, do so now.
3004 if (InvertPredCond
64.1
'InvertPredCond' is false
64.1
'InvertPredCond' is false
64.1
'InvertPredCond' is false
64.1
'InvertPredCond' is false
64.1
'InvertPredCond' is false
64.1
'InvertPredCond' is false
) {
65
Taking false branch
3005 Value *NewCond = PBI->getCondition();
3006 if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) {
3007 CmpInst *CI = cast<CmpInst>(NewCond);
3008 CI->setPredicate(CI->getInversePredicate());
3009 } else {
3010 NewCond =
3011 Builder.CreateNot(NewCond, PBI->getCondition()->getName() + ".not");
3012 }
3013
3014 PBI->setCondition(NewCond);
3015 PBI->swapSuccessors();
3016 }
3017
3018 BasicBlock *UniqueSucc =
3019 BI->isConditional()
66
'?' condition is true
3020 ? (PBI->getSuccessor(0) == BB ? TrueDest : FalseDest)
67
'?' condition is false
3021 : TrueDest;
3022
3023 // Before cloning instructions, notify the successor basic block that it
3024 // is about to have a new predecessor. This will update PHI nodes,
3025 // which will allow us to update live-out uses of bonus instructions.
3026 if (BI->isConditional())
68
Taking true branch
3027 AddPredecessorToBlock(UniqueSucc, PredBlock, BB, MSSAU);
3028
3029 // If we have bonus instructions, clone them into the predecessor block.
3030 // Note that there may be multiple predecessor blocks, so we cannot move
3031 // bonus instructions to a predecessor block.
3032 ValueToValueMapTy VMap; // maps original values to cloned values
3033 Instruction *CondInPred;
69
'CondInPred' declared without an initial value
3034 for (Instruction &BonusInst : *BB) {
3035 if (isa<DbgInfoIntrinsic>(BonusInst) || isa<BranchInst>(BonusInst))
3036 continue;
3037
3038 Instruction *NewBonusInst = BonusInst.clone();
3039
3040 if (&BonusInst == Cond)
3041 CondInPred = NewBonusInst;
3042
3043 if (PBI->getDebugLoc() != NewBonusInst->getDebugLoc()) {
3044 // Unless the instruction has the same !dbg location as the original
3045 // branch, drop it. When we fold the bonus instructions we want to make
3046 // sure we reset their debug locations in order to avoid stepping on
3047 // dead code caused by folding dead branches.
3048 NewBonusInst->setDebugLoc(DebugLoc());
3049 }
3050
3051 RemapInstruction(NewBonusInst, VMap,
3052 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
3053 VMap[&BonusInst] = NewBonusInst;
3054
3055 // If we moved a load, we cannot any longer claim any knowledge about
3056 // its potential value. The previous information might have been valid
3057 // only given the branch precondition.
3058 // For an analogous reason, we must also drop all the metadata whose
3059 // semantics we don't understand. We *can* preserve !annotation, because
3060 // it is tied to the instruction itself, not the value or position.
3061 NewBonusInst->dropUnknownNonDebugMetadata(LLVMContext::MD_annotation);
3062
3063 PredBlock->getInstList().insert(PBI->getIterator(), NewBonusInst);
3064 NewBonusInst->takeName(&BonusInst);
3065 BonusInst.setName(BonusInst.getName() + ".old");
3066 BonusInst.replaceUsesWithIf(
3067 NewBonusInst, [BB, BI, UniqueSucc, PredBlock](Use &U) {
3068 auto *User = cast<Instruction>(U.getUser());
3069 // Ignore non-external uses of bonus instructions.
3070 if (User->getParent() == BB) {
3071 assert(!isa<PHINode>(User) &&((!isa<PHINode>(User) && "Non-external users are never PHI instructions."
) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(User) && \"Non-external users are never PHI instructions.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3072, __PRETTY_FUNCTION__))
3072 "Non-external users are never PHI instructions.")((!isa<PHINode>(User) && "Non-external users are never PHI instructions."
) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(User) && \"Non-external users are never PHI instructions.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3072, __PRETTY_FUNCTION__))
;
3073 return false;
3074 }
3075 if (User->getParent() == PredBlock) {
3076 // The "exteral" use is in the block into which we just cloned the
3077 // bonus instruction. This means two things: 1. we are in an
3078 // unreachable block 2. the instruction is self-referencing.
3079 // So let's just rewrite it...
3080 return true;
3081 }
3082 (void)BI;
3083 assert(isa<PHINode>(User) && "All external users must be PHI's.")((isa<PHINode>(User) && "All external users must be PHI's."
) ? static_cast<void> (0) : __assert_fail ("isa<PHINode>(User) && \"All external users must be PHI's.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3083, __PRETTY_FUNCTION__))
;
3084 auto *PN = cast<PHINode>(User);
3085 assert(is_contained(successors(BB), User->getParent()) &&((is_contained(successors(BB), User->getParent()) &&
"All external users must be in successors of BB.") ? static_cast
<void> (0) : __assert_fail ("is_contained(successors(BB), User->getParent()) && \"All external users must be in successors of BB.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3086, __PRETTY_FUNCTION__))
3086 "All external users must be in successors of BB.")((is_contained(successors(BB), User->getParent()) &&
"All external users must be in successors of BB.") ? static_cast
<void> (0) : __assert_fail ("is_contained(successors(BB), User->getParent()) && \"All external users must be in successors of BB.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3086, __PRETTY_FUNCTION__))
;
3087 assert((PN->getIncomingBlock(U) == BB ||(((PN->getIncomingBlock(U) == BB || PN->getIncomingBlock
(U) == PredBlock) && "The incoming block for that incoming value external use "
"must be either the original block with bonus instructions, "
"or the new predecessor block.") ? static_cast<void> (
0) : __assert_fail ("(PN->getIncomingBlock(U) == BB || PN->getIncomingBlock(U) == PredBlock) && \"The incoming block for that incoming value external use \" \"must be either the original block with bonus instructions, \" \"or the new predecessor block.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3091, __PRETTY_FUNCTION__))
3088 PN->getIncomingBlock(U) == PredBlock) &&(((PN->getIncomingBlock(U) == BB || PN->getIncomingBlock
(U) == PredBlock) && "The incoming block for that incoming value external use "
"must be either the original block with bonus instructions, "
"or the new predecessor block.") ? static_cast<void> (
0) : __assert_fail ("(PN->getIncomingBlock(U) == BB || PN->getIncomingBlock(U) == PredBlock) && \"The incoming block for that incoming value external use \" \"must be either the original block with bonus instructions, \" \"or the new predecessor block.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3091, __PRETTY_FUNCTION__))
3089 "The incoming block for that incoming value external use "(((PN->getIncomingBlock(U) == BB || PN->getIncomingBlock
(U) == PredBlock) && "The incoming block for that incoming value external use "
"must be either the original block with bonus instructions, "
"or the new predecessor block.") ? static_cast<void> (
0) : __assert_fail ("(PN->getIncomingBlock(U) == BB || PN->getIncomingBlock(U) == PredBlock) && \"The incoming block for that incoming value external use \" \"must be either the original block with bonus instructions, \" \"or the new predecessor block.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3091, __PRETTY_FUNCTION__))
3090 "must be either the original block with bonus instructions, "(((PN->getIncomingBlock(U) == BB || PN->getIncomingBlock
(U) == PredBlock) && "The incoming block for that incoming value external use "
"must be either the original block with bonus instructions, "
"or the new predecessor block.") ? static_cast<void> (
0) : __assert_fail ("(PN->getIncomingBlock(U) == BB || PN->getIncomingBlock(U) == PredBlock) && \"The incoming block for that incoming value external use \" \"must be either the original block with bonus instructions, \" \"or the new predecessor block.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3091, __PRETTY_FUNCTION__))
3091 "or the new predecessor block.")(((PN->getIncomingBlock(U) == BB || PN->getIncomingBlock
(U) == PredBlock) && "The incoming block for that incoming value external use "
"must be either the original block with bonus instructions, "
"or the new predecessor block.") ? static_cast<void> (
0) : __assert_fail ("(PN->getIncomingBlock(U) == BB || PN->getIncomingBlock(U) == PredBlock) && \"The incoming block for that incoming value external use \" \"must be either the original block with bonus instructions, \" \"or the new predecessor block.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3091, __PRETTY_FUNCTION__))
;
3092 // UniqueSucc is the block for which we change it's predecessors,
3093 // so it is the only block in which we'll need to update PHI nodes.
3094 if (User->getParent() != UniqueSucc)
3095 return false;
3096 // Update the incoming value for the new predecessor.
3097 return PN->getIncomingBlock(U) ==
3098 (BI->isConditional() ? PredBlock : BB);
3099 });
3100 }
3101
3102 // Now that the Cond was cloned into the predecessor basic block,
3103 // or/and the two conditions together.
3104 if (BI->isConditional()) {
70
Calling 'BranchInst::isConditional'
72
Returning from 'BranchInst::isConditional'
73
Taking true branch
3105 Instruction *NewCond = cast<Instruction>(
3106 Builder.CreateBinOp(Opc, PBI->getCondition(), CondInPred, "or.cond"));
74
3rd function call argument is an uninitialized value
3107 PBI->setCondition(NewCond);
3108
3109 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
3110 bool HasWeights =
3111 extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
3112 SuccTrueWeight, SuccFalseWeight);
3113 SmallVector<uint64_t, 8> NewWeights;
3114
3115 if (PBI->getSuccessor(0) == BB) {
3116 if (HasWeights) {
3117 // PBI: br i1 %x, BB, FalseDest
3118 // BI: br i1 %y, UniqueSucc, FalseDest
3119 // TrueWeight is TrueWeight for PBI * TrueWeight for BI.
3120 NewWeights.push_back(PredTrueWeight * SuccTrueWeight);
3121 // FalseWeight is FalseWeight for PBI * TotalWeight for BI +
3122 // TrueWeight for PBI * FalseWeight for BI.
3123 // We assume that total weights of a BranchInst can fit into 32 bits.
3124 // Therefore, we will not have overflow using 64-bit arithmetic.
3125 NewWeights.push_back(PredFalseWeight *
3126 (SuccFalseWeight + SuccTrueWeight) +
3127 PredTrueWeight * SuccFalseWeight);
3128 }
3129 PBI->setSuccessor(0, UniqueSucc);
3130 }
3131 if (PBI->getSuccessor(1) == BB) {
3132 if (HasWeights) {
3133 // PBI: br i1 %x, TrueDest, BB
3134 // BI: br i1 %y, TrueDest, UniqueSucc
3135 // TrueWeight is TrueWeight for PBI * TotalWeight for BI +
3136 // FalseWeight for PBI * TrueWeight for BI.
3137 NewWeights.push_back(PredTrueWeight *
3138 (SuccFalseWeight + SuccTrueWeight) +
3139 PredFalseWeight * SuccTrueWeight);
3140 // FalseWeight is FalseWeight for PBI * FalseWeight for BI.
3141 NewWeights.push_back(PredFalseWeight * SuccFalseWeight);
3142 }
3143 PBI->setSuccessor(1, UniqueSucc);
3144 }
3145 if (NewWeights.size() == 2) {
3146 // Halve the weights if any of them cannot fit in an uint32_t
3147 FitWeights(NewWeights);
3148
3149 SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(),
3150 NewWeights.end());
3151 setBranchWeights(PBI, MDWeights[0], MDWeights[1]);
3152 } else
3153 PBI->setMetadata(LLVMContext::MD_prof, nullptr);
3154
3155 Updates.push_back({DominatorTree::Insert, PredBlock, UniqueSucc});
3156 Updates.push_back({DominatorTree::Delete, PredBlock, BB});
3157 } else {
3158 // Update PHI nodes in the common successors.
3159 for (unsigned i = 0, e = PHIs.size(); i != e; ++i) {
3160 ConstantInt *PBI_C = cast<ConstantInt>(
3161 PHIs[i]->getIncomingValueForBlock(PBI->getParent()));
3162 assert(PBI_C->getType()->isIntegerTy(1))((PBI_C->getType()->isIntegerTy(1)) ? static_cast<void
> (0) : __assert_fail ("PBI_C->getType()->isIntegerTy(1)"
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3162, __PRETTY_FUNCTION__))
;
3163 Instruction *MergedCond = nullptr;
3164 if (PBI->getSuccessor(0) == UniqueSucc) {
3165 Updates.push_back(
3166 {DominatorTree::Delete, PredBlock, PBI->getSuccessor(1)});
3167 // Create (PBI_Cond and PBI_C) or (!PBI_Cond and BI_Value)
3168 // PBI_C is true: PBI_Cond or (!PBI_Cond and BI_Value)
3169 // is false: !PBI_Cond and BI_Value
3170 Instruction *NotCond = cast<Instruction>(
3171 Builder.CreateNot(PBI->getCondition(), "not.cond"));
3172 MergedCond = cast<Instruction>(
3173 Builder.CreateBinOp(Instruction::And, NotCond, CondInPred,
3174 "and.cond"));
3175 if (PBI_C->isOne())
3176 MergedCond = cast<Instruction>(Builder.CreateBinOp(
3177 Instruction::Or, PBI->getCondition(), MergedCond, "or.cond"));
3178 } else {
3179 assert(PBI->getSuccessor(1) == UniqueSucc && "Unexpected branch")((PBI->getSuccessor(1) == UniqueSucc && "Unexpected branch"
) ? static_cast<void> (0) : __assert_fail ("PBI->getSuccessor(1) == UniqueSucc && \"Unexpected branch\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3179, __PRETTY_FUNCTION__))
;
3180 Updates.push_back(
3181 {DominatorTree::Delete, PredBlock, PBI->getSuccessor(0)});
3182 // Create (PBI_Cond and BI_Value) or (!PBI_Cond and PBI_C)
3183 // PBI_C is true: (PBI_Cond and BI_Value) or (!PBI_Cond)
3184 // is false: PBI_Cond and BI_Value
3185 MergedCond = cast<Instruction>(Builder.CreateBinOp(
3186 Instruction::And, PBI->getCondition(), CondInPred, "and.cond"));
3187 if (PBI_C->isOne()) {
3188 Instruction *NotCond = cast<Instruction>(
3189 Builder.CreateNot(PBI->getCondition(), "not.cond"));
3190 MergedCond = cast<Instruction>(Builder.CreateBinOp(
3191 Instruction::Or, NotCond, MergedCond, "or.cond"));
3192 }
3193 }
3194 // Update PHI Node.
3195 PHIs[i]->setIncomingValueForBlock(PBI->getParent(), MergedCond);
3196 }
3197
3198 // PBI is changed to branch to UniqueSucc below. Remove itself from
3199 // potential phis from all other successors.
3200 if (MSSAU)
3201 MSSAU->changeCondBranchToUnconditionalTo(PBI, UniqueSucc);
3202
3203 // Change PBI from Conditional to Unconditional.
3204 BranchInst *New_PBI = BranchInst::Create(UniqueSucc, PBI);
3205 EraseTerminatorAndDCECond(PBI, MSSAU);
3206 PBI = New_PBI;
3207 }
3208
3209 if (DTU)
3210 DTU->applyUpdates(Updates);
3211
3212 // If BI was a loop latch, it may have had associated loop metadata.
3213 // We need to copy it to the new latch, that is, PBI.
3214 if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop))
3215 PBI->setMetadata(LLVMContext::MD_loop, LoopMD);
3216
3217 // TODO: If BB is reachable from all paths through PredBlock, then we
3218 // could replace PBI's branch probabilities with BI's.
3219
3220 // Copy any debug value intrinsics into the end of PredBlock.
3221 for (Instruction &I : *BB) {
3222 if (isa<DbgInfoIntrinsic>(I)) {
3223 Instruction *NewI = I.clone();
3224 RemapInstruction(NewI, VMap,
3225 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
3226 NewI->insertBefore(PBI);
3227 }
3228 }
3229
3230 return Changed;
3231 }
3232 return Changed;
3233}
3234
3235// If there is only one store in BB1 and BB2, return it, otherwise return
3236// nullptr.
3237static StoreInst *findUniqueStoreInBlocks(BasicBlock *BB1, BasicBlock *BB2) {
3238 StoreInst *S = nullptr;
3239 for (auto *BB : {BB1, BB2}) {
3240 if (!BB)
3241 continue;
3242 for (auto &I : *BB)
3243 if (auto *SI = dyn_cast<StoreInst>(&I)) {
3244 if (S)
3245 // Multiple stores seen.
3246 return nullptr;
3247 else
3248 S = SI;
3249 }
3250 }
3251 return S;
3252}
3253
3254static Value *ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB,
3255 Value *AlternativeV = nullptr) {
3256 // PHI is going to be a PHI node that allows the value V that is defined in
3257 // BB to be referenced in BB's only successor.
3258 //
3259 // If AlternativeV is nullptr, the only value we care about in PHI is V. It
3260 // doesn't matter to us what the other operand is (it'll never get used). We
3261 // could just create a new PHI with an undef incoming value, but that could
3262 // increase register pressure if EarlyCSE/InstCombine can't fold it with some
3263 // other PHI. So here we directly look for some PHI in BB's successor with V
3264 // as an incoming operand. If we find one, we use it, else we create a new
3265 // one.
3266 //
3267 // If AlternativeV is not nullptr, we care about both incoming values in PHI.
3268 // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV]
3269 // where OtherBB is the single other predecessor of BB's only successor.
3270 PHINode *PHI = nullptr;
3271 BasicBlock *Succ = BB->getSingleSuccessor();
3272
3273 for (auto I = Succ->begin(); isa<PHINode>(I); ++I)
3274 if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) {
3275 PHI = cast<PHINode>(I);
3276 if (!AlternativeV)
3277 break;
3278
3279 assert(Succ->hasNPredecessors(2))((Succ->hasNPredecessors(2)) ? static_cast<void> (0)
: __assert_fail ("Succ->hasNPredecessors(2)", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3279, __PRETTY_FUNCTION__))
;
3280 auto PredI = pred_begin(Succ);
3281 BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI;
3282 if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV)
3283 break;
3284 PHI = nullptr;
3285 }
3286 if (PHI)
3287 return PHI;
3288
3289 // If V is not an instruction defined in BB, just return it.
3290 if (!AlternativeV &&
3291 (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB))
3292 return V;
3293
3294 PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge", &Succ->front());
3295 PHI->addIncoming(V, BB);
3296 for (BasicBlock *PredBB : predecessors(Succ))
3297 if (PredBB != BB)
3298 PHI->addIncoming(
3299 AlternativeV ? AlternativeV : UndefValue::get(V->getType()), PredBB);
3300 return PHI;
3301}
3302
3303static bool mergeConditionalStoreToAddress(
3304 BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB,
3305 BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond,
3306 DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI) {
3307 // For every pointer, there must be exactly two stores, one coming from
3308 // PTB or PFB, and the other from QTB or QFB. We don't support more than one
3309 // store (to any address) in PTB,PFB or QTB,QFB.
3310 // FIXME: We could relax this restriction with a bit more work and performance
3311 // testing.
3312 StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB);
3313 StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB);
3314 if (!PStore || !QStore)
3315 return false;
3316
3317 // Now check the stores are compatible.
3318 if (!QStore->isUnordered() || !PStore->isUnordered())
3319 return false;
3320
3321 // Check that sinking the store won't cause program behavior changes. Sinking
3322 // the store out of the Q blocks won't change any behavior as we're sinking
3323 // from a block to its unconditional successor. But we're moving a store from
3324 // the P blocks down through the middle block (QBI) and past both QFB and QTB.
3325 // So we need to check that there are no aliasing loads or stores in
3326 // QBI, QTB and QFB. We also need to check there are no conflicting memory
3327 // operations between PStore and the end of its parent block.
3328 //
3329 // The ideal way to do this is to query AliasAnalysis, but we don't
3330 // preserve AA currently so that is dangerous. Be super safe and just
3331 // check there are no other memory operations at all.
3332 for (auto &I : *QFB->getSinglePredecessor())
3333 if (I.mayReadOrWriteMemory())
3334 return false;
3335 for (auto &I : *QFB)
3336 if (&I != QStore && I.mayReadOrWriteMemory())
3337 return false;
3338 if (QTB)
3339 for (auto &I : *QTB)
3340 if (&I != QStore && I.mayReadOrWriteMemory())
3341 return false;
3342 for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end();
3343 I != E; ++I)
3344 if (&*I != PStore && I->mayReadOrWriteMemory())
3345 return false;
3346
3347 // If we're not in aggressive mode, we only optimize if we have some
3348 // confidence that by optimizing we'll allow P and/or Q to be if-converted.
3349 auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) {
3350 if (!BB)
3351 return true;
3352 // Heuristic: if the block can be if-converted/phi-folded and the
3353 // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
3354 // thread this store.
3355 int BudgetRemaining =
3356 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
3357 for (auto &I : BB->instructionsWithoutDebug()) {
3358 // Consider terminator instruction to be free.
3359 if (I.isTerminator())
3360 continue;
3361 // If this is one the stores that we want to speculate out of this BB,
3362 // then don't count it's cost, consider it to be free.
3363 if (auto *S = dyn_cast<StoreInst>(&I))
3364 if (llvm::find(FreeStores, S))
3365 continue;
3366 // Else, we have a white-list of instructions that we are ak speculating.
3367 if (!isa<BinaryOperator>(I) && !isa<GetElementPtrInst>(I))
3368 return false; // Not in white-list - not worthwhile folding.
3369 // And finally, if this is a non-free instruction that we are okay
3370 // speculating, ensure that we consider the speculation budget.
3371 BudgetRemaining -= TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency);
3372 if (BudgetRemaining < 0)
3373 return false; // Eagerly refuse to fold as soon as we're out of budget.
3374 }
3375 assert(BudgetRemaining >= 0 &&((BudgetRemaining >= 0 && "When we run out of budget we will eagerly return from within the "
"per-instruction loop.") ? static_cast<void> (0) : __assert_fail
("BudgetRemaining >= 0 && \"When we run out of budget we will eagerly return from within the \" \"per-instruction loop.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3377, __PRETTY_FUNCTION__))
3376 "When we run out of budget we will eagerly return from within the "((BudgetRemaining >= 0 && "When we run out of budget we will eagerly return from within the "
"per-instruction loop.") ? static_cast<void> (0) : __assert_fail
("BudgetRemaining >= 0 && \"When we run out of budget we will eagerly return from within the \" \"per-instruction loop.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3377, __PRETTY_FUNCTION__))
3377 "per-instruction loop.")((BudgetRemaining >= 0 && "When we run out of budget we will eagerly return from within the "
"per-instruction loop.") ? static_cast<void> (0) : __assert_fail
("BudgetRemaining >= 0 && \"When we run out of budget we will eagerly return from within the \" \"per-instruction loop.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3377, __PRETTY_FUNCTION__))
;
3378 return true;
3379 };
3380
3381 const std::array<StoreInst *, 2> FreeStores = {PStore, QStore};
3382 if (!MergeCondStoresAggressively &&
3383 (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) ||
3384 !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores)))
3385 return false;
3386
3387 // If PostBB has more than two predecessors, we need to split it so we can
3388 // sink the store.
3389 if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) {
3390 // We know that QFB's only successor is PostBB. And QFB has a single
3391 // predecessor. If QTB exists, then its only successor is also PostBB.
3392 // If QTB does not exist, then QFB's only predecessor has a conditional
3393 // branch to QFB and PostBB.
3394 BasicBlock *TruePred = QTB ? QTB : QFB->getSinglePredecessor();
3395 BasicBlock *NewBB =
3396 SplitBlockPredecessors(PostBB, {QFB, TruePred}, "condstore.split", DTU);
3397 if (!NewBB)
3398 return false;
3399 PostBB = NewBB;
3400 }
3401
3402 // OK, we're going to sink the stores to PostBB. The store has to be
3403 // conditional though, so first create the predicate.
3404 Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator())
3405 ->getCondition();
3406 Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator())
3407 ->getCondition();
3408
3409 Value *PPHI = ensureValueAvailableInSuccessor(PStore->getValueOperand(),
3410 PStore->getParent());
3411 Value *QPHI = ensureValueAvailableInSuccessor(QStore->getValueOperand(),
3412 QStore->getParent(), PPHI);
3413
3414 IRBuilder<> QB(&*PostBB->getFirstInsertionPt());
3415
3416 Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond);
3417 Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond);
3418
3419 if (InvertPCond)
3420 PPred = QB.CreateNot(PPred);
3421 if (InvertQCond)
3422 QPred = QB.CreateNot(QPred);
3423 Value *CombinedPred = QB.CreateOr(PPred, QPred);
3424
3425 auto *T = SplitBlockAndInsertIfThen(CombinedPred, &*QB.GetInsertPoint(),
3426 /*Unreachable=*/false,
3427 /*BranchWeights=*/nullptr, DTU);
3428 QB.SetInsertPoint(T);
3429 StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address));
3430 AAMDNodes AAMD;
3431 PStore->getAAMetadata(AAMD, /*Merge=*/false);
3432 PStore->getAAMetadata(AAMD, /*Merge=*/true);
3433 SI->setAAMetadata(AAMD);
3434 // Choose the minimum alignment. If we could prove both stores execute, we
3435 // could use biggest one. In this case, though, we only know that one of the
3436 // stores executes. And we don't know it's safe to take the alignment from a
3437 // store that doesn't execute.
3438 SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign()));
3439
3440 QStore->eraseFromParent();
3441 PStore->eraseFromParent();
3442
3443 return true;
3444}
3445
3446static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI,
3447 DomTreeUpdater *DTU, const DataLayout &DL,
3448 const TargetTransformInfo &TTI) {
3449 // The intention here is to find diamonds or triangles (see below) where each
3450 // conditional block contains a store to the same address. Both of these
3451 // stores are conditional, so they can't be unconditionally sunk. But it may
3452 // be profitable to speculatively sink the stores into one merged store at the
3453 // end, and predicate the merged store on the union of the two conditions of
3454 // PBI and QBI.
3455 //
3456 // This can reduce the number of stores executed if both of the conditions are
3457 // true, and can allow the blocks to become small enough to be if-converted.
3458 // This optimization will also chain, so that ladders of test-and-set
3459 // sequences can be if-converted away.
3460 //
3461 // We only deal with simple diamonds or triangles:
3462 //
3463 // PBI or PBI or a combination of the two
3464 // / \ | \
3465 // PTB PFB | PFB
3466 // \ / | /
3467 // QBI QBI
3468 // / \ | \
3469 // QTB QFB | QFB
3470 // \ / | /
3471 // PostBB PostBB
3472 //
3473 // We model triangles as a type of diamond with a nullptr "true" block.
3474 // Triangles are canonicalized so that the fallthrough edge is represented by
3475 // a true condition, as in the diagram above.
3476 BasicBlock *PTB = PBI->getSuccessor(0);
3477 BasicBlock *PFB = PBI->getSuccessor(1);
3478 BasicBlock *QTB = QBI->getSuccessor(0);
3479 BasicBlock *QFB = QBI->getSuccessor(1);
3480 BasicBlock *PostBB = QFB->getSingleSuccessor();
3481
3482 // Make sure we have a good guess for PostBB. If QTB's only successor is
3483 // QFB, then QFB is a better PostBB.
3484 if (QTB->getSingleSuccessor() == QFB)
3485 PostBB = QFB;
3486
3487 // If we couldn't find a good PostBB, stop.
3488 if (!PostBB)
3489 return false;
3490
3491 bool InvertPCond = false, InvertQCond = false;
3492 // Canonicalize fallthroughs to the true branches.
3493 if (PFB == QBI->getParent()) {
3494 std::swap(PFB, PTB);
3495 InvertPCond = true;
3496 }
3497 if (QFB == PostBB) {
3498 std::swap(QFB, QTB);
3499 InvertQCond = true;
3500 }
3501
3502 // From this point on we can assume PTB or QTB may be fallthroughs but PFB
3503 // and QFB may not. Model fallthroughs as a nullptr block.
3504 if (PTB == QBI->getParent())
3505 PTB = nullptr;
3506 if (QTB == PostBB)
3507 QTB = nullptr;
3508
3509 // Legality bailouts. We must have at least the non-fallthrough blocks and
3510 // the post-dominating block, and the non-fallthroughs must only have one
3511 // predecessor.
3512 auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) {
3513 return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S;
3514 };
3515 if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) ||
3516 !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB))
3517 return false;
3518 if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) ||
3519 (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB)))
3520 return false;
3521 if (!QBI->getParent()->hasNUses(2))
3522 return false;
3523
3524 // OK, this is a sequence of two diamonds or triangles.
3525 // Check if there are stores in PTB or PFB that are repeated in QTB or QFB.
3526 SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses;
3527 for (auto *BB : {PTB, PFB}) {
3528 if (!BB)
3529 continue;
3530 for (auto &I : *BB)
3531 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3532 PStoreAddresses.insert(SI->getPointerOperand());
3533 }
3534 for (auto *BB : {QTB, QFB}) {
3535 if (!BB)
3536 continue;
3537 for (auto &I : *BB)
3538 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3539 QStoreAddresses.insert(SI->getPointerOperand());
3540 }
3541
3542 set_intersect(PStoreAddresses, QStoreAddresses);
3543 // set_intersect mutates PStoreAddresses in place. Rename it here to make it
3544 // clear what it contains.
3545 auto &CommonAddresses = PStoreAddresses;
3546
3547 bool Changed = false;
3548 for (auto *Address : CommonAddresses)
3549 Changed |=
3550 mergeConditionalStoreToAddress(PTB, PFB, QTB, QFB, PostBB, Address,
3551 InvertPCond, InvertQCond, DTU, DL, TTI);
3552 return Changed;
3553}
3554
3555/// If the previous block ended with a widenable branch, determine if reusing
3556/// the target block is profitable and legal. This will have the effect of
3557/// "widening" PBI, but doesn't require us to reason about hosting safety.
3558static bool tryWidenCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
3559 DomTreeUpdater *DTU) {
3560 // TODO: This can be generalized in two important ways:
3561 // 1) We can allow phi nodes in IfFalseBB and simply reuse all the input
3562 // values from the PBI edge.
3563 // 2) We can sink side effecting instructions into BI's fallthrough
3564 // successor provided they doesn't contribute to computation of
3565 // BI's condition.
3566 Value *CondWB, *WC;
3567 BasicBlock *IfTrueBB, *IfFalseBB;
3568 if (!parseWidenableBranch(PBI, CondWB, WC, IfTrueBB, IfFalseBB) ||
3569 IfTrueBB != BI->getParent() || !BI->getParent()->getSinglePredecessor())
3570 return false;
3571 if (!IfFalseBB->phis().empty())
3572 return false; // TODO
3573 // Use lambda to lazily compute expensive condition after cheap ones.
3574 auto NoSideEffects = [](BasicBlock &BB) {
3575 return !llvm::any_of(BB, [](const Instruction &I) {
3576 return I.mayWriteToMemory() || I.mayHaveSideEffects();
3577 });
3578 };
3579 if (BI->getSuccessor(1) != IfFalseBB && // no inf looping
3580 BI->getSuccessor(1)->getTerminatingDeoptimizeCall() && // profitability
3581 NoSideEffects(*BI->getParent())) {
3582 auto *OldSuccessor = BI->getSuccessor(1);
3583 OldSuccessor->removePredecessor(BI->getParent());
3584 BI->setSuccessor(1, IfFalseBB);
3585 if (DTU)
3586 DTU->applyUpdates(
3587 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
3588 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
3589 return true;
3590 }
3591 if (BI->getSuccessor(0) != IfFalseBB && // no inf looping
3592 BI->getSuccessor(0)->getTerminatingDeoptimizeCall() && // profitability
3593 NoSideEffects(*BI->getParent())) {
3594 auto *OldSuccessor = BI->getSuccessor(0);
3595 OldSuccessor->removePredecessor(BI->getParent());
3596 BI->setSuccessor(0, IfFalseBB);
3597 if (DTU)
3598 DTU->applyUpdates(
3599 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
3600 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
3601 return true;
3602 }
3603 return false;
3604}
3605
3606/// If we have a conditional branch as a predecessor of another block,
3607/// this function tries to simplify it. We know
3608/// that PBI and BI are both conditional branches, and BI is in one of the
3609/// successor blocks of PBI - PBI branches to BI.
3610static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
3611 DomTreeUpdater *DTU,
3612 const DataLayout &DL,
3613 const TargetTransformInfo &TTI) {
3614 assert(PBI->isConditional() && BI->isConditional())((PBI->isConditional() && BI->isConditional()) ?
static_cast<void> (0) : __assert_fail ("PBI->isConditional() && BI->isConditional()"
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3614, __PRETTY_FUNCTION__))
;
3615 BasicBlock *BB = BI->getParent();
3616
3617 // If this block ends with a branch instruction, and if there is a
3618 // predecessor that ends on a branch of the same condition, make
3619 // this conditional branch redundant.
3620 if (PBI->getCondition() == BI->getCondition() &&
3621 PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3622 // Okay, the outcome of this conditional branch is statically
3623 // knowable. If this block had a single pred, handle specially.
3624 if (BB->getSinglePredecessor()) {
3625 // Turn this into a branch on constant.
3626 bool CondIsTrue = PBI->getSuccessor(0) == BB;
3627 BI->setCondition(
3628 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue));
3629 return true; // Nuke the branch on constant.
3630 }
3631
3632 // Otherwise, if there are multiple predecessors, insert a PHI that merges
3633 // in the constant and simplify the block result. Subsequent passes of
3634 // simplifycfg will thread the block.
3635 if (BlockIsSimpleEnoughToThreadThrough(BB)) {
3636 pred_iterator PB = pred_begin(BB), PE = pred_end(BB);
3637 PHINode *NewPN = PHINode::Create(
3638 Type::getInt1Ty(BB->getContext()), std::distance(PB, PE),
3639 BI->getCondition()->getName() + ".pr", &BB->front());
3640 // Okay, we're going to insert the PHI node. Since PBI is not the only
3641 // predecessor, compute the PHI'd conditional value for all of the preds.
3642 // Any predecessor where the condition is not computable we keep symbolic.
3643 for (pred_iterator PI = PB; PI != PE; ++PI) {
3644 BasicBlock *P = *PI;
3645 if ((PBI = dyn_cast<BranchInst>(P->getTerminator())) && PBI != BI &&
3646 PBI->isConditional() && PBI->getCondition() == BI->getCondition() &&
3647 PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3648 bool CondIsTrue = PBI->getSuccessor(0) == BB;
3649 NewPN->addIncoming(
3650 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue),
3651 P);
3652 } else {
3653 NewPN->addIncoming(BI->getCondition(), P);
3654 }
3655 }
3656
3657 BI->setCondition(NewPN);
3658 return true;
3659 }
3660 }
3661
3662 // If the previous block ended with a widenable branch, determine if reusing
3663 // the target block is profitable and legal. This will have the effect of
3664 // "widening" PBI, but doesn't require us to reason about hosting safety.
3665 if (tryWidenCondBranchToCondBranch(PBI, BI, DTU))
3666 return true;
3667
3668 if (auto *CE = dyn_cast<ConstantExpr>(BI->getCondition()))
3669 if (CE->canTrap())
3670 return false;
3671
3672 // If both branches are conditional and both contain stores to the same
3673 // address, remove the stores from the conditionals and create a conditional
3674 // merged store at the end.
3675 if (MergeCondStores && mergeConditionalStores(PBI, BI, DTU, DL, TTI))
3676 return true;
3677
3678 // If this is a conditional branch in an empty block, and if any
3679 // predecessors are a conditional branch to one of our destinations,
3680 // fold the conditions into logical ops and one cond br.
3681
3682 // Ignore dbg intrinsics.
3683 if (&*BB->instructionsWithoutDebug().begin() != BI)
3684 return false;
3685
3686 int PBIOp, BIOp;
3687 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
3688 PBIOp = 0;
3689 BIOp = 0;
3690 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
3691 PBIOp = 0;
3692 BIOp = 1;
3693 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
3694 PBIOp = 1;
3695 BIOp = 0;
3696 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
3697 PBIOp = 1;
3698 BIOp = 1;
3699 } else {
3700 return false;
3701 }
3702
3703 // Check to make sure that the other destination of this branch
3704 // isn't BB itself. If so, this is an infinite loop that will
3705 // keep getting unwound.
3706 if (PBI->getSuccessor(PBIOp) == BB)
3707 return false;
3708
3709 // Do not perform this transformation if it would require
3710 // insertion of a large number of select instructions. For targets
3711 // without predication/cmovs, this is a big pessimization.
3712
3713 // Also do not perform this transformation if any phi node in the common
3714 // destination block can trap when reached by BB or PBB (PR17073). In that
3715 // case, it would be unsafe to hoist the operation into a select instruction.
3716
3717 BasicBlock *CommonDest = PBI->getSuccessor(PBIOp);
3718 BasicBlock *RemovedDest = PBI->getSuccessor(PBIOp ^ 1);
3719 unsigned NumPhis = 0;
3720 for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II);
3721 ++II, ++NumPhis) {
3722 if (NumPhis > 2) // Disable this xform.
3723 return false;
3724
3725 PHINode *PN = cast<PHINode>(II);
3726 Value *BIV = PN->getIncomingValueForBlock(BB);
3727 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(BIV))
3728 if (CE->canTrap())
3729 return false;
3730
3731 unsigned PBBIdx = PN->getBasicBlockIndex(PBI->getParent());
3732 Value *PBIV = PN->getIncomingValue(PBBIdx);
3733 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(PBIV))
3734 if (CE->canTrap())
3735 return false;
3736 }
3737
3738 // Finally, if everything is ok, fold the branches to logical ops.
3739 BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1);
3740
3741 LLVM_DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING BRs:" << *PBI
->getParent() << "AND: " << *BI->getParent(
); } } while (false)
3742 << "AND: " << *BI->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING BRs:" << *PBI
->getParent() << "AND: " << *BI->getParent(
); } } while (false)
;
3743
3744 SmallVector<DominatorTree::UpdateType, 5> Updates;
3745
3746 // If OtherDest *is* BB, then BB is a basic block with a single conditional
3747 // branch in it, where one edge (OtherDest) goes back to itself but the other
3748 // exits. We don't *know* that the program avoids the infinite loop
3749 // (even though that seems likely). If we do this xform naively, we'll end up
3750 // recursively unpeeling the loop. Since we know that (after the xform is
3751 // done) that the block *is* infinite if reached, we just make it an obviously
3752 // infinite loop with no cond branch.
3753 if (OtherDest == BB) {
3754 // Insert it at the end of the function, because it's either code,
3755 // or it won't matter if it's hot. :)
3756 BasicBlock *InfLoopBlock =
3757 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
3758 BranchInst::Create(InfLoopBlock, InfLoopBlock);
3759 Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
3760 OtherDest = InfLoopBlock;
3761 }
3762
3763 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << *PBI->getParent()->getParent
(); } } while (false)
;
3764
3765 // BI may have other predecessors. Because of this, we leave
3766 // it alone, but modify PBI.
3767
3768 // Make sure we get to CommonDest on True&True directions.
3769 Value *PBICond = PBI->getCondition();
3770 IRBuilder<NoFolder> Builder(PBI);
3771 if (PBIOp)
3772 PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not");
3773
3774 Value *BICond = BI->getCondition();
3775 if (BIOp)
3776 BICond = Builder.CreateNot(BICond, BICond->getName() + ".not");
3777
3778 // Merge the conditions.
3779 Value *Cond = Builder.CreateOr(PBICond, BICond, "brmerge");
3780
3781 // Modify PBI to branch on the new condition to the new dests.
3782 PBI->setCondition(Cond);
3783 PBI->setSuccessor(0, CommonDest);
3784 PBI->setSuccessor(1, OtherDest);
3785
3786 Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest});
3787 Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest});
3788
3789 if (DTU)
3790 DTU->applyUpdates(Updates);
3791
3792 // Update branch weight for PBI.
3793 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
3794 uint64_t PredCommon, PredOther, SuccCommon, SuccOther;
3795 bool HasWeights =
3796 extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
3797 SuccTrueWeight, SuccFalseWeight);
3798 if (HasWeights) {
3799 PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3800 PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3801 SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3802 SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3803 // The weight to CommonDest should be PredCommon * SuccTotal +
3804 // PredOther * SuccCommon.
3805 // The weight to OtherDest should be PredOther * SuccOther.
3806 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) +
3807 PredOther * SuccCommon,
3808 PredOther * SuccOther};
3809 // Halve the weights if any of them cannot fit in an uint32_t
3810 FitWeights(NewWeights);
3811
3812 setBranchWeights(PBI, NewWeights[0], NewWeights[1]);
3813 }
3814
3815 // OtherDest may have phi nodes. If so, add an entry from PBI's
3816 // block that are identical to the entries for BI's block.
3817 AddPredecessorToBlock(OtherDest, PBI->getParent(), BB);
3818
3819 // We know that the CommonDest already had an edge from PBI to
3820 // it. If it has PHIs though, the PHIs may have different
3821 // entries for BB and PBI's BB. If so, insert a select to make
3822 // them agree.
3823 for (PHINode &PN : CommonDest->phis()) {
3824 Value *BIV = PN.getIncomingValueForBlock(BB);
3825 unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent());
3826 Value *PBIV = PN.getIncomingValue(PBBIdx);
3827 if (BIV != PBIV) {
3828 // Insert a select in PBI to pick the right value.
3829 SelectInst *NV = cast<SelectInst>(
3830 Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux"));
3831 PN.setIncomingValue(PBBIdx, NV);
3832 // Although the select has the same condition as PBI, the original branch
3833 // weights for PBI do not apply to the new select because the select's
3834 // 'logical' edges are incoming edges of the phi that is eliminated, not
3835 // the outgoing edges of PBI.
3836 if (HasWeights) {
3837 uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3838 uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3839 uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3840 uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3841 // The weight to PredCommonDest should be PredCommon * SuccTotal.
3842 // The weight to PredOtherDest should be PredOther * SuccCommon.
3843 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther),
3844 PredOther * SuccCommon};
3845
3846 FitWeights(NewWeights);
3847
3848 setBranchWeights(NV, NewWeights[0], NewWeights[1]);
3849 }
3850 }
3851 }
3852
3853 LLVM_DEBUG(dbgs() << "INTO: " << *PBI->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "INTO: " << *PBI->
getParent(); } } while (false)
;
3854 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << *PBI->getParent()->getParent
(); } } while (false)
;
3855
3856 // This basic block is probably dead. We know it has at least
3857 // one fewer predecessor.
3858 return true;
3859}
3860
3861// Simplifies a terminator by replacing it with a branch to TrueBB if Cond is
3862// true or to FalseBB if Cond is false.
3863// Takes care of updating the successors and removing the old terminator.
3864// Also makes sure not to introduce new successors by assuming that edges to
3865// non-successor TrueBBs and FalseBBs aren't reachable.
3866bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm,
3867 Value *Cond, BasicBlock *TrueBB,
3868 BasicBlock *FalseBB,
3869 uint32_t TrueWeight,
3870 uint32_t FalseWeight) {
3871 auto *BB = OldTerm->getParent();
3872 // Remove any superfluous successor edges from the CFG.
3873 // First, figure out which successors to preserve.
3874 // If TrueBB and FalseBB are equal, only try to preserve one copy of that
3875 // successor.
3876 BasicBlock *KeepEdge1 = TrueBB;
3877 BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr;
3878
3879 SmallSetVector<BasicBlock *, 2> RemovedSuccessors;
3880
3881 // Then remove the rest.
3882 for (BasicBlock *Succ : successors(OldTerm)) {
3883 // Make sure only to keep exactly one copy of each edge.
3884 if (Succ == KeepEdge1)
3885 KeepEdge1 = nullptr;
3886 else if (Succ == KeepEdge2)
3887 KeepEdge2 = nullptr;
3888 else {
3889 Succ->removePredecessor(BB,
3890 /*KeepOneInputPHIs=*/true);
3891
3892 if (Succ != TrueBB && Succ != FalseBB)
3893 RemovedSuccessors.insert(Succ);
3894 }
3895 }
3896
3897 IRBuilder<> Builder(OldTerm);
3898 Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc());
3899
3900 // Insert an appropriate new terminator.
3901 if (!KeepEdge1 && !KeepEdge2) {
3902 if (TrueBB == FalseBB) {
3903 // We were only looking for one successor, and it was present.
3904 // Create an unconditional branch to it.
3905 Builder.CreateBr(TrueBB);
3906 } else {
3907 // We found both of the successors we were looking for.
3908 // Create a conditional branch sharing the condition of the select.
3909 BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
3910 if (TrueWeight != FalseWeight)
3911 setBranchWeights(NewBI, TrueWeight, FalseWeight);
3912 }
3913 } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
3914 // Neither of the selected blocks were successors, so this
3915 // terminator must be unreachable.
3916 new UnreachableInst(OldTerm->getContext(), OldTerm);
3917 } else {
3918 // One of the selected values was a successor, but the other wasn't.
3919 // Insert an unconditional branch to the one that was found;
3920 // the edge to the one that wasn't must be unreachable.
3921 if (!KeepEdge1) {
3922 // Only TrueBB was found.
3923 Builder.CreateBr(TrueBB);
3924 } else {
3925 // Only FalseBB was found.
3926 Builder.CreateBr(FalseBB);
3927 }
3928 }
3929
3930 EraseTerminatorAndDCECond(OldTerm);
3931
3932 if (DTU) {
3933 SmallVector<DominatorTree::UpdateType, 2> Updates;
3934 Updates.reserve(RemovedSuccessors.size());
3935 for (auto *RemovedSuccessor : RemovedSuccessors)
3936 Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor});
3937 DTU->applyUpdates(Updates);
3938 }
3939
3940 return true;
3941}
3942
3943// Replaces
3944// (switch (select cond, X, Y)) on constant X, Y
3945// with a branch - conditional if X and Y lead to distinct BBs,
3946// unconditional otherwise.
3947bool SimplifyCFGOpt::SimplifySwitchOnSelect(SwitchInst *SI,
3948 SelectInst *Select) {
3949 // Check for constant integer values in the select.
3950 ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue());
3951 ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue());
3952 if (!TrueVal || !FalseVal)
3953 return false;
3954
3955 // Find the relevant condition and destinations.
3956 Value *Condition = Select->getCondition();
3957 BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor();
3958 BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor();
3959
3960 // Get weight for TrueBB and FalseBB.
3961 uint32_t TrueWeight = 0, FalseWeight = 0;
3962 SmallVector<uint64_t, 8> Weights;
3963 bool HasWeights = HasBranchWeights(SI);
3964 if (HasWeights) {
3965 GetBranchWeights(SI, Weights);
3966 if (Weights.size() == 1 + SI->getNumCases()) {
3967 TrueWeight =
3968 (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()];
3969 FalseWeight =
3970 (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()];
3971 }
3972 }
3973
3974 // Perform the actual simplification.
3975 return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight,
3976 FalseWeight);
3977}
3978
3979// Replaces
3980// (indirectbr (select cond, blockaddress(@fn, BlockA),
3981// blockaddress(@fn, BlockB)))
3982// with
3983// (br cond, BlockA, BlockB).
3984bool SimplifyCFGOpt::SimplifyIndirectBrOnSelect(IndirectBrInst *IBI,
3985 SelectInst *SI) {
3986 // Check that both operands of the select are block addresses.
3987 BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue());
3988 BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue());
3989 if (!TBA || !FBA)
3990 return false;
3991
3992 // Extract the actual blocks.
3993 BasicBlock *TrueBB = TBA->getBasicBlock();
3994 BasicBlock *FalseBB = FBA->getBasicBlock();
3995
3996 // Perform the actual simplification.
3997 return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0,
3998 0);
3999}
4000
4001/// This is called when we find an icmp instruction
4002/// (a seteq/setne with a constant) as the only instruction in a
4003/// block that ends with an uncond branch. We are looking for a very specific
4004/// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In
4005/// this case, we merge the first two "or's of icmp" into a switch, but then the
4006/// default value goes to an uncond block with a seteq in it, we get something
4007/// like:
4008///
4009/// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ]
4010/// DEFAULT:
4011/// %tmp = icmp eq i8 %A, 92
4012/// br label %end
4013/// end:
4014/// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ]
4015///
4016/// We prefer to split the edge to 'end' so that there is a true/false entry to
4017/// the PHI, merging the third icmp into the switch.
4018bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt(
4019 ICmpInst *ICI, IRBuilder<> &Builder) {
4020 BasicBlock *BB = ICI->getParent();
4021
4022 // If the block has any PHIs in it or the icmp has multiple uses, it is too
4023 // complex.
4024 if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse())
4025 return false;
4026
4027 Value *V = ICI->getOperand(0);
4028 ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1));
4029
4030 // The pattern we're looking for is where our only predecessor is a switch on
4031 // 'V' and this block is the default case for the switch. In this case we can
4032 // fold the compared value into the switch to simplify things.
4033 BasicBlock *Pred = BB->getSinglePredecessor();
4034 if (!Pred || !isa<SwitchInst>(Pred->getTerminator()))
4035 return false;
4036
4037 SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator());
4038 if (SI->getCondition() != V)
4039 return false;
4040
4041 // If BB is reachable on a non-default case, then we simply know the value of
4042 // V in this block. Substitute it and constant fold the icmp instruction
4043 // away.
4044 if (SI->getDefaultDest() != BB) {
4045 ConstantInt *VVal = SI->findCaseDest(BB);
4046 assert(VVal && "Should have a unique destination value")((VVal && "Should have a unique destination value") ?
static_cast<void> (0) : __assert_fail ("VVal && \"Should have a unique destination value\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4046, __PRETTY_FUNCTION__))
;
4047 ICI->setOperand(0, VVal);
4048
4049 if (Value *V = SimplifyInstruction(ICI, {DL, ICI})) {
4050 ICI->replaceAllUsesWith(V);
4051 ICI->eraseFromParent();
4052 }
4053 // BB is now empty, so it is likely to simplify away.
4054 return requestResimplify();
4055 }
4056
4057 // Ok, the block is reachable from the default dest. If the constant we're
4058 // comparing exists in one of the other edges, then we can constant fold ICI
4059 // and zap it.
4060 if (SI->findCaseValue(Cst) != SI->case_default()) {
4061 Value *V;
4062 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
4063 V = ConstantInt::getFalse(BB->getContext());
4064 else
4065 V = ConstantInt::getTrue(BB->getContext());
4066
4067 ICI->replaceAllUsesWith(V);
4068 ICI->eraseFromParent();
4069 // BB is now empty, so it is likely to simplify away.
4070 return requestResimplify();
4071 }
4072
4073 // The use of the icmp has to be in the 'end' block, by the only PHI node in
4074 // the block.
4075 BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0);
4076 PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back());
4077 if (PHIUse == nullptr || PHIUse != &SuccBlock->front() ||
4078 isa<PHINode>(++BasicBlock::iterator(PHIUse)))
4079 return false;
4080
4081 // If the icmp is a SETEQ, then the default dest gets false, the new edge gets
4082 // true in the PHI.
4083 Constant *DefaultCst = ConstantInt::getTrue(BB->getContext());
4084 Constant *NewCst = ConstantInt::getFalse(BB->getContext());
4085
4086 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
4087 std::swap(DefaultCst, NewCst);
4088
4089 // Replace ICI (which is used by the PHI for the default value) with true or
4090 // false depending on if it is EQ or NE.
4091 ICI->replaceAllUsesWith(DefaultCst);
4092 ICI->eraseFromParent();
4093
4094 SmallVector<DominatorTree::UpdateType, 2> Updates;
4095
4096 // Okay, the switch goes to this block on a default value. Add an edge from
4097 // the switch to the merge point on the compared value.
4098 BasicBlock *NewBB =
4099 BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB);
4100 {
4101 SwitchInstProfUpdateWrapper SIW(*SI);
4102 auto W0 = SIW.getSuccessorWeight(0);
4103 SwitchInstProfUpdateWrapper::CaseWeightOpt NewW;
4104 if (W0) {
4105 NewW = ((uint64_t(*W0) + 1) >> 1);
4106 SIW.setSuccessorWeight(0, *NewW);
4107 }
4108 SIW.addCase(Cst, NewBB, NewW);
4109 Updates.push_back({DominatorTree::Insert, Pred, NewBB});
4110 }
4111
4112 // NewBB branches to the phi block, add the uncond branch and the phi entry.
4113 Builder.SetInsertPoint(NewBB);
4114 Builder.SetCurrentDebugLocation(SI->getDebugLoc());
4115 Builder.CreateBr(SuccBlock);
4116 Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock});
4117 PHIUse->addIncoming(NewCst, NewBB);
4118 if (DTU)
4119 DTU->applyUpdates(Updates);
4120 return true;
4121}
4122
4123/// The specified branch is a conditional branch.
4124/// Check to see if it is branching on an or/and chain of icmp instructions, and
4125/// fold it into a switch instruction if so.
4126bool SimplifyCFGOpt::SimplifyBranchOnICmpChain(BranchInst *BI,
4127 IRBuilder<> &Builder,
4128 const DataLayout &DL) {
4129 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
4130 if (!Cond)
4131 return false;
4132
4133 // Change br (X == 0 | X == 1), T, F into a switch instruction.
4134 // If this is a bunch of seteq's or'd together, or if it's a bunch of
4135 // 'setne's and'ed together, collect them.
4136
4137 // Try to gather values from a chain of and/or to be turned into a switch
4138 ConstantComparesGatherer ConstantCompare(Cond, DL);
4139 // Unpack the result
4140 SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals;
4141 Value *CompVal = ConstantCompare.CompValue;
4142 unsigned UsedICmps = ConstantCompare.UsedICmps;
4143 Value *ExtraCase = ConstantCompare.Extra;
4144
4145 // If we didn't have a multiply compared value, fail.
4146 if (!CompVal)
4147 return false;
4148
4149 // Avoid turning single icmps into a switch.
4150 if (UsedICmps <= 1)
4151 return false;
4152
4153 bool TrueWhenEqual = match(Cond, m_LogicalOr(m_Value(), m_Value()));
4154
4155 // There might be duplicate constants in the list, which the switch
4156 // instruction can't handle, remove them now.
4157 array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate);
4158 Values.erase(std::unique(Values.begin(), Values.end()), Values.end());
4159
4160 // If Extra was used, we require at least two switch values to do the
4161 // transformation. A switch with one value is just a conditional branch.
4162 if (ExtraCase && Values.size() < 2)
4163 return false;
4164
4165 // TODO: Preserve branch weight metadata, similarly to how
4166 // FoldValueComparisonIntoPredecessors preserves it.
4167
4168 // Figure out which block is which destination.
4169 BasicBlock *DefaultBB = BI->getSuccessor(1);
4170 BasicBlock *EdgeBB = BI->getSuccessor(0);
4171 if (!TrueWhenEqual)
4172 std::swap(DefaultBB, EdgeBB);
4173
4174 BasicBlock *BB = BI->getParent();
4175
4176 // MSAN does not like undefs as branch condition which can be introduced
4177 // with "explicit branch".
4178 if (ExtraCase && BB->getParent()->hasFnAttribute(Attribute::SanitizeMemory))
4179 return false;
4180
4181 LLVM_DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Converting 'icmp' chain with "
<< Values.size() << " cases into SWITCH. BB is:\n"
<< *BB; } } while (false)
4182 << " cases into SWITCH. BB is:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Converting 'icmp' chain with "
<< Values.size() << " cases into SWITCH. BB is:\n"
<< *BB; } } while (false)
4183 << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Converting 'icmp' chain with "
<< Values.size() << " cases into SWITCH. BB is:\n"
<< *BB; } } while (false)
;
4184
4185 SmallVector<DominatorTree::UpdateType, 2> Updates;
4186
4187 // If there are any extra values that couldn't be folded into the switch
4188 // then we evaluate them with an explicit branch first. Split the block
4189 // right before the condbr to handle it.
4190 if (ExtraCase) {
4191 BasicBlock *NewBB = SplitBlock(BB, BI, DTU, /*LI=*/nullptr,
4192 /*MSSAU=*/nullptr, "switch.early.test");
4193
4194 // Remove the uncond branch added to the old block.
4195 Instruction *OldTI = BB->getTerminator();
4196 Builder.SetInsertPoint(OldTI);
4197
4198 if (TrueWhenEqual)
4199 Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB);
4200 else
4201 Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB);
4202
4203 OldTI->eraseFromParent();
4204
4205 Updates.push_back({DominatorTree::Insert, BB, EdgeBB});
4206
4207 // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
4208 // for the edge we just added.
4209 AddPredecessorToBlock(EdgeBB, BB, NewBB);
4210
4211 LLVM_DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCasedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << " ** 'icmp' chain unhandled condition: "
<< *ExtraCase << "\nEXTRABB = " << *BB; } }
while (false)
4212 << "\nEXTRABB = " << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << " ** 'icmp' chain unhandled condition: "
<< *ExtraCase << "\nEXTRABB = " << *BB; } }
while (false)
;
4213 BB = NewBB;
4214 }
4215
4216 Builder.SetInsertPoint(BI);
4217 // Convert pointer to int before we switch.
4218 if (CompVal->getType()->isPointerTy()) {
4219 CompVal = Builder.CreatePtrToInt(
4220 CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr");
4221 }
4222
4223 // Create the new switch instruction now.
4224 SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size());
4225
4226 // Add all of the 'cases' to the switch instruction.
4227 for (unsigned i = 0, e = Values.size(); i != e; ++i)
4228 New->addCase(Values[i], EdgeBB);
4229
4230 // We added edges from PI to the EdgeBB. As such, if there were any
4231 // PHI nodes in EdgeBB, they need entries to be added corresponding to
4232 // the number of edges added.
4233 for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) {
4234 PHINode *PN = cast<PHINode>(BBI);
4235 Value *InVal = PN->getIncomingValueForBlock(BB);
4236 for (unsigned i = 0, e = Values.size() - 1; i != e; ++i)
4237 PN->addIncoming(InVal, BB);
4238 }
4239
4240 // Erase the old branch instruction.
4241 EraseTerminatorAndDCECond(BI);
4242 if (DTU)
4243 DTU->applyUpdates(Updates);
4244
4245 LLVM_DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << " ** 'icmp' chain result is:\n"
<< *BB << '\n'; } } while (false)
;
4246 return true;
4247}
4248
4249bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
4250 if (isa<PHINode>(RI->getValue()))
4251 return simplifyCommonResume(RI);
4252 else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) &&
4253 RI->getValue() == RI->getParent()->getFirstNonPHI())
4254 // The resume must unwind the exception that caused control to branch here.
4255 return simplifySingleResume(RI);
4256
4257 return false;
4258}
4259
4260// Check if cleanup block is empty
4261static bool isCleanupBlockEmpty(iterator_range<BasicBlock::iterator> R) {
4262 for (Instruction &I : R) {
4263 auto *II = dyn_cast<IntrinsicInst>(&I);
4264 if (!II)
4265 return false;
4266
4267 Intrinsic::ID IntrinsicID = II->getIntrinsicID();
4268 switch (IntrinsicID) {
4269 case Intrinsic::dbg_declare:
4270 case Intrinsic::dbg_value:
4271 case Intrinsic::dbg_label:
4272 case Intrinsic::lifetime_end:
4273 break;
4274 default:
4275 return false;
4276 }
4277 }
4278 return true;
4279}
4280
4281// Simplify resume that is shared by several landing pads (phi of landing pad).
4282bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
4283 BasicBlock *BB = RI->getParent();
4284
4285 // Check that there are no other instructions except for debug and lifetime
4286 // intrinsics between the phi's and resume instruction.
4287 if (!isCleanupBlockEmpty(
4288 make_range(RI->getParent()->getFirstNonPHI(), BB->getTerminator())))
4289 return false;
4290
4291 SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks;
4292 auto *PhiLPInst = cast<PHINode>(RI->getValue());
4293
4294 // Check incoming blocks to see if any of them are trivial.
4295 for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End;
4296 Idx++) {
4297 auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx);
4298 auto *IncomingValue = PhiLPInst->getIncomingValue(Idx);
4299
4300 // If the block has other successors, we can not delete it because
4301 // it has other dependents.
4302 if (IncomingBB->getUniqueSuccessor() != BB)
4303 continue;
4304
4305 auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI());
4306 // Not the landing pad that caused the control to branch here.
4307 if (IncomingValue != LandingPad)
4308 continue;
4309
4310 if (isCleanupBlockEmpty(
4311 make_range(LandingPad->getNextNode(), IncomingBB->getTerminator())))
4312 TrivialUnwindBlocks.insert(IncomingBB);
4313 }
4314
4315 // If no trivial unwind blocks, don't do any simplifications.
4316 if (TrivialUnwindBlocks.empty())
4317 return false;
4318
4319 // Turn all invokes that unwind here into calls.
4320 for (auto *TrivialBB : TrivialUnwindBlocks) {
4321 // Blocks that will be simplified should be removed from the phi node.
4322 // Note there could be multiple edges to the resume block, and we need
4323 // to remove them all.
4324 while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1)
4325 BB->removePredecessor(TrivialBB, true);
4326
4327 for (pred_iterator PI = pred_begin(TrivialBB), PE = pred_end(TrivialBB);
4328 PI != PE;) {
4329 BasicBlock *Pred = *PI++;
4330 removeUnwindEdge(Pred, DTU);
4331 ++NumInvokes;
4332 }
4333
4334 // In each SimplifyCFG run, only the current processed block can be erased.
4335 // Otherwise, it will break the iteration of SimplifyCFG pass. So instead
4336 // of erasing TrivialBB, we only remove the branch to the common resume
4337 // block so that we can later erase the resume block since it has no
4338 // predecessors.
4339 TrivialBB->getTerminator()->eraseFromParent();
4340 new UnreachableInst(RI->getContext(), TrivialBB);
4341 if (DTU)
4342 DTU->applyUpdates({{DominatorTree::Delete, TrivialBB, BB}});
4343 }
4344
4345 // Delete the resume block if all its predecessors have been removed.
4346 if (pred_empty(BB)) {
4347 if (DTU)
4348 DTU->deleteBB(BB);
4349 else
4350 BB->eraseFromParent();
4351 }
4352
4353 return !TrivialUnwindBlocks.empty();
4354}
4355
4356// Simplify resume that is only used by a single (non-phi) landing pad.
4357bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) {
4358 BasicBlock *BB = RI->getParent();
4359 auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI());
4360 assert(RI->getValue() == LPInst &&((RI->getValue() == LPInst && "Resume must unwind the exception that caused control to here"
) ? static_cast<void> (0) : __assert_fail ("RI->getValue() == LPInst && \"Resume must unwind the exception that caused control to here\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4361, __PRETTY_FUNCTION__))
4361 "Resume must unwind the exception that caused control to here")((RI->getValue() == LPInst && "Resume must unwind the exception that caused control to here"
) ? static_cast<void> (0) : __assert_fail ("RI->getValue() == LPInst && \"Resume must unwind the exception that caused control to here\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4361, __PRETTY_FUNCTION__))
;
4362
4363 // Check that there are no other instructions except for debug intrinsics.
4364 if (!isCleanupBlockEmpty(
4365 make_range<Instruction *>(LPInst->getNextNode(), RI)))
4366 return false;
4367
4368 // Turn all invokes that unwind here into calls and delete the basic block.
4369 for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE;) {
4370 BasicBlock *Pred = *PI++;
4371 removeUnwindEdge(Pred, DTU);
4372 ++NumInvokes;
4373 }
4374
4375 // The landingpad is now unreachable. Zap it.
4376 if (LoopHeaders)
4377 LoopHeaders->erase(BB);
4378 if (DTU)
4379 DTU->deleteBB(BB);
4380 else
4381 BB->eraseFromParent();
4382 return true;
4383}
4384
4385static bool removeEmptyCleanup(CleanupReturnInst *RI, DomTreeUpdater *DTU) {
4386 // If this is a trivial cleanup pad that executes no instructions, it can be
4387 // eliminated. If the cleanup pad continues to the caller, any predecessor
4388 // that is an EH pad will be updated to continue to the caller and any
4389 // predecessor that terminates with an invoke instruction will have its invoke
4390 // instruction converted to a call instruction. If the cleanup pad being
4391 // simplified does not continue to the caller, each predecessor will be
4392 // updated to continue to the unwind destination of the cleanup pad being
4393 // simplified.
4394 BasicBlock *BB = RI->getParent();
4395 CleanupPadInst *CPInst = RI->getCleanupPad();
4396 if (CPInst->getParent() != BB)
4397 // This isn't an empty cleanup.
4398 return false;
4399
4400 // We cannot kill the pad if it has multiple uses. This typically arises
4401 // from unreachable basic blocks.
4402 if (!CPInst->hasOneUse())
4403 return false;
4404
4405 // Check that there are no other instructions except for benign intrinsics.
4406 if (!isCleanupBlockEmpty(
4407 make_range<Instruction *>(CPInst->getNextNode(), RI)))
4408 return false;
4409
4410 // If the cleanup return we are simplifying unwinds to the caller, this will
4411 // set UnwindDest to nullptr.
4412 BasicBlock *UnwindDest = RI->getUnwindDest();
4413 Instruction *DestEHPad = UnwindDest ? UnwindDest->getFirstNonPHI() : nullptr;
4414
4415 // We're about to remove BB from the control flow. Before we do, sink any
4416 // PHINodes into the unwind destination. Doing this before changing the
4417 // control flow avoids some potentially slow checks, since we can currently
4418 // be certain that UnwindDest and BB have no common predecessors (since they
4419 // are both EH pads).
4420 if (UnwindDest) {
4421 // First, go through the PHI nodes in UnwindDest and update any nodes that
4422 // reference the block we are removing
4423 for (BasicBlock::iterator I = UnwindDest->begin(),
4424 IE = DestEHPad->getIterator();
4425 I != IE; ++I) {
4426 PHINode *DestPN = cast<PHINode>(I);
4427
4428 int Idx = DestPN->getBasicBlockIndex(BB);
4429 // Since BB unwinds to UnwindDest, it has to be in the PHI node.
4430 assert(Idx != -1)((Idx != -1) ? static_cast<void> (0) : __assert_fail ("Idx != -1"
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4430, __PRETTY_FUNCTION__))
;
4431 // This PHI node has an incoming value that corresponds to a control
4432 // path through the cleanup pad we are removing. If the incoming
4433 // value is in the cleanup pad, it must be a PHINode (because we
4434 // verified above that the block is otherwise empty). Otherwise, the
4435 // value is either a constant or a value that dominates the cleanup
4436 // pad being removed.
4437 //
4438 // Because BB and UnwindDest are both EH pads, all of their
4439 // predecessors must unwind to these blocks, and since no instruction
4440 // can have multiple unwind destinations, there will be no overlap in
4441 // incoming blocks between SrcPN and DestPN.
4442 Value *SrcVal = DestPN->getIncomingValue(Idx);
4443 PHINode *SrcPN = dyn_cast<PHINode>(SrcVal);
4444
4445 // Remove the entry for the block we are deleting.
4446 DestPN->removeIncomingValue(Idx, false);
4447
4448 if (SrcPN && SrcPN->getParent() == BB) {
4449 // If the incoming value was a PHI node in the cleanup pad we are
4450 // removing, we need to merge that PHI node's incoming values into
4451 // DestPN.
4452 for (unsigned SrcIdx = 0, SrcE = SrcPN->getNumIncomingValues();
4453 SrcIdx != SrcE; ++SrcIdx) {
4454 DestPN->addIncoming(SrcPN->getIncomingValue(SrcIdx),
4455 SrcPN->getIncomingBlock(SrcIdx));
4456 }
4457 } else {
4458 // Otherwise, the incoming value came from above BB and
4459 // so we can just reuse it. We must associate all of BB's
4460 // predecessors with this value.
4461 for (auto *pred : predecessors(BB)) {
4462 DestPN->addIncoming(SrcVal, pred);
4463 }
4464 }
4465 }
4466
4467 // Sink any remaining PHI nodes directly into UnwindDest.
4468 Instruction *InsertPt = DestEHPad;
4469 for (BasicBlock::iterator I = BB->begin(),
4470 IE = BB->getFirstNonPHI()->getIterator();
4471 I != IE;) {
4472 // The iterator must be incremented here because the instructions are
4473 // being moved to another block.
4474 PHINode *PN = cast<PHINode>(I++);
4475 if (PN->use_empty() || !PN->isUsedOutsideOfBlock(BB))
4476 // If the PHI node has no uses or all of its uses are in this basic
4477 // block (meaning they are debug or lifetime intrinsics), just leave
4478 // it. It will be erased when we erase BB below.
4479 continue;
4480
4481 // Otherwise, sink this PHI node into UnwindDest.
4482 // Any predecessors to UnwindDest which are not already represented
4483 // must be back edges which inherit the value from the path through
4484 // BB. In this case, the PHI value must reference itself.
4485 for (auto *pred : predecessors(UnwindDest))
4486 if (pred != BB)
4487 PN->addIncoming(PN, pred);
4488 PN->moveBefore(InsertPt);
4489 }
4490 }
4491
4492 std::vector<DominatorTree::UpdateType> Updates;
4493
4494 for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE;) {
4495 // The iterator must be updated here because we are removing this pred.
4496 BasicBlock *PredBB = *PI++;
4497 if (UnwindDest == nullptr) {
4498 if (DTU)
4499 DTU->applyUpdates(Updates);
4500 Updates.clear();
4501 removeUnwindEdge(PredBB, DTU);
4502 ++NumInvokes;
4503 } else {
4504 Instruction *TI = PredBB->getTerminator();
4505 TI->replaceUsesOfWith(BB, UnwindDest);
4506 Updates.push_back({DominatorTree::Insert, PredBB, UnwindDest});
4507 Updates.push_back({DominatorTree::Delete, PredBB, BB});
4508 }
4509 }
4510
4511 if (DTU) {
4512 DTU->applyUpdates(Updates);
4513 DTU->deleteBB(BB);
4514 } else
4515 // The cleanup pad is now unreachable. Zap it.
4516 BB->eraseFromParent();
4517
4518 return true;
4519}
4520
4521// Try to merge two cleanuppads together.
4522static bool mergeCleanupPad(CleanupReturnInst *RI) {
4523 // Skip any cleanuprets which unwind to caller, there is nothing to merge
4524 // with.
4525 BasicBlock *UnwindDest = RI->getUnwindDest();
4526 if (!UnwindDest)
4527 return false;
4528
4529 // This cleanupret isn't the only predecessor of this cleanuppad, it wouldn't
4530 // be safe to merge without code duplication.
4531 if (UnwindDest->getSinglePredecessor() != RI->getParent())
4532 return false;
4533
4534 // Verify that our cleanuppad's unwind destination is another cleanuppad.
4535 auto *SuccessorCleanupPad = dyn_cast<CleanupPadInst>(&UnwindDest->front());
4536 if (!SuccessorCleanupPad)
4537 return false;
4538
4539 CleanupPadInst *PredecessorCleanupPad = RI->getCleanupPad();
4540 // Replace any uses of the successor cleanupad with the predecessor pad
4541 // The only cleanuppad uses should be this cleanupret, it's cleanupret and
4542 // funclet bundle operands.
4543 SuccessorCleanupPad->replaceAllUsesWith(PredecessorCleanupPad);
4544 // Remove the old cleanuppad.
4545 SuccessorCleanupPad->eraseFromParent();
4546 // Now, we simply replace the cleanupret with a branch to the unwind
4547 // destination.
4548 BranchInst::Create(UnwindDest, RI->getParent());
4549 RI->eraseFromParent();
4550
4551 return true;
4552}
4553
4554bool SimplifyCFGOpt::simplifyCleanupReturn(CleanupReturnInst *RI) {
4555 // It is possible to transiantly have an undef cleanuppad operand because we
4556 // have deleted some, but not all, dead blocks.
4557 // Eventually, this block will be deleted.
4558 if (isa<UndefValue>(RI->getOperand(0)))
4559 return false;
4560
4561 if (mergeCleanupPad(RI))
4562 return true;
4563
4564 if (removeEmptyCleanup(RI, DTU))
4565 return true;
4566
4567 return false;
4568}
4569
4570bool SimplifyCFGOpt::simplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
4571 BasicBlock *BB = RI->getParent();
4572 if (!BB->getFirstNonPHIOrDbg()->isTerminator())
4573 return false;
4574
4575 // Find predecessors that end with branches.
4576 SmallVector<BasicBlock *, 8> UncondBranchPreds;
4577 SmallVector<BranchInst *, 8> CondBranchPreds;
4578 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
4579 BasicBlock *P = *PI;
4580 Instruction *PTI = P->getTerminator();
4581 if (BranchInst *BI = dyn_cast<BranchInst>(PTI)) {
4582 if (BI->isUnconditional())
4583 UncondBranchPreds.push_back(P);
4584 else
4585 CondBranchPreds.push_back(BI);
4586 }
4587 }
4588
4589 // If we found some, do the transformation!
4590 if (!UncondBranchPreds.empty() && DupRet) {
4591 while (!UncondBranchPreds.empty()) {
4592 BasicBlock *Pred = UncondBranchPreds.pop_back_val();
4593 LLVM_DEBUG(dbgs() << "FOLDING: " << *BBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING: " << *BB <<
"INTO UNCOND BRANCH PRED: " << *Pred; } } while (false
)
4594 << "INTO UNCOND BRANCH PRED: " << *Pred)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING: " << *BB <<
"INTO UNCOND BRANCH PRED: " << *Pred; } } while (false
)
;
4595 (void)FoldReturnIntoUncondBranch(RI, BB, Pred, DTU);
4596 }
4597
4598 // If we eliminated all predecessors of the block, delete the block now.
4599 if (pred_empty(BB)) {
4600 // We know there are no successors, so just nuke the block.
4601 if (LoopHeaders)
4602 LoopHeaders->erase(BB);
4603 if (DTU)
4604 DTU->deleteBB(BB);
4605 else
4606 BB->eraseFromParent();
4607 }
4608
4609 return true;
4610 }
4611
4612 // Check out all of the conditional branches going to this return
4613 // instruction. If any of them just select between returns, change the
4614 // branch itself into a select/return pair.
4615 while (!CondBranchPreds.empty()) {
4616 BranchInst *BI = CondBranchPreds.pop_back_val();
4617
4618 // Check to see if the non-BB successor is also a return block.
4619 if (isa<ReturnInst>(BI->getSuccessor(0)->getTerminator()) &&
4620 isa<ReturnInst>(BI->getSuccessor(1)->getTerminator()) &&
4621 SimplifyCondBranchToTwoReturns(BI, Builder))
4622 return true;
4623 }
4624 return false;
4625}
4626
4627bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
4628 BasicBlock *BB = UI->getParent();
4629
4630 bool Changed = false;
4631
4632 // If there are any instructions immediately before the unreachable that can
4633 // be removed, do so.
4634 while (UI->getIterator() != BB->begin()) {
4635 BasicBlock::iterator BBI = UI->getIterator();
4636 --BBI;
4637 // Do not delete instructions that can have side effects which might cause
4638 // the unreachable to not be reachable; specifically, calls and volatile
4639 // operations may have this effect.
4640 if (isa<CallInst>(BBI) && !isa<DbgInfoIntrinsic>(BBI))
4641 break;
4642
4643 if (BBI->mayHaveSideEffects()) {
4644 if (auto *SI = dyn_cast<StoreInst>(BBI)) {
4645 if (SI->isVolatile())
4646 break;
4647 } else if (auto *LI = dyn_cast<LoadInst>(BBI)) {
4648 if (LI->isVolatile())
4649 break;
4650 } else if (auto *RMWI = dyn_cast<AtomicRMWInst>(BBI)) {
4651 if (RMWI->isVolatile())
4652 break;
4653 } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(BBI)) {
4654 if (CXI->isVolatile())
4655 break;
4656 } else if (isa<CatchPadInst>(BBI)) {
4657 // A catchpad may invoke exception object constructors and such, which
4658 // in some languages can be arbitrary code, so be conservative by
4659 // default.
4660 // For CoreCLR, it just involves a type test, so can be removed.
4661 if (classifyEHPersonality(BB->getParent()->getPersonalityFn()) !=
4662 EHPersonality::CoreCLR)
4663 break;
4664 } else if (!isa<FenceInst>(BBI) && !isa<VAArgInst>(BBI) &&
4665 !isa<LandingPadInst>(BBI)) {
4666 break;
4667 }
4668 // Note that deleting LandingPad's here is in fact okay, although it
4669 // involves a bit of subtle reasoning. If this inst is a LandingPad,
4670 // all the predecessors of this block will be the unwind edges of Invokes,
4671 // and we can therefore guarantee this block will be erased.
4672 }
4673
4674 // Delete this instruction (any uses are guaranteed to be dead)
4675 if (!BBI->use_empty())
4676 BBI->replaceAllUsesWith(UndefValue::get(BBI->getType()));
4677 BBI->eraseFromParent();
4678 Changed = true;
4679 }
4680
4681 // If the unreachable instruction is the first in the block, take a gander
4682 // at all of the predecessors of this instruction, and simplify them.
4683 if (&BB->front() != UI)
4684 return Changed;
4685
4686 std::vector<DominatorTree::UpdateType> Updates;
4687
4688 SmallSetVector<BasicBlock *, 8> Preds(pred_begin(BB), pred_end(BB));
4689 for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
4690 auto *Predecessor = Preds[i];
4691 Instruction *TI = Predecessor->getTerminator();
4692 IRBuilder<> Builder(TI);
4693 if (auto *BI = dyn_cast<BranchInst>(TI)) {
4694 // We could either have a proper unconditional branch,
4695 // or a degenerate conditional branch with matching destinations.
4696 if (all_of(BI->successors(),
4697 [BB](auto *Successor) { return Successor == BB; })) {
4698 new UnreachableInst(TI->getContext(), TI);
4699 TI->eraseFromParent();
4700 Changed = true;
4701 } else {
4702 assert(BI->isConditional() && "Can't get here with an uncond branch.")((BI->isConditional() && "Can't get here with an uncond branch."
) ? static_cast<void> (0) : __assert_fail ("BI->isConditional() && \"Can't get here with an uncond branch.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4702, __PRETTY_FUNCTION__))
;
4703 Value* Cond = BI->getCondition();
4704 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&((BI->getSuccessor(0) != BI->getSuccessor(1) &&
"The destinations are guaranteed to be different here.") ? static_cast
<void> (0) : __assert_fail ("BI->getSuccessor(0) != BI->getSuccessor(1) && \"The destinations are guaranteed to be different here.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4705, __PRETTY_FUNCTION__))
4705 "The destinations are guaranteed to be different here.")((BI->getSuccessor(0) != BI->getSuccessor(1) &&
"The destinations are guaranteed to be different here.") ? static_cast
<void> (0) : __assert_fail ("BI->getSuccessor(0) != BI->getSuccessor(1) && \"The destinations are guaranteed to be different here.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4705, __PRETTY_FUNCTION__))
;
4706 if (BI->getSuccessor(0) == BB) {
4707 Builder.CreateAssumption(Builder.CreateNot(Cond));
4708 Builder.CreateBr(BI->getSuccessor(1));
4709 } else {
4710 assert(BI->getSuccessor(1) == BB && "Incorrect CFG")((BI->getSuccessor(1) == BB && "Incorrect CFG") ? static_cast
<void> (0) : __assert_fail ("BI->getSuccessor(1) == BB && \"Incorrect CFG\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4710, __PRETTY_FUNCTION__))
;
4711 Builder.CreateAssumption(Cond);
4712 Builder.CreateBr(BI->getSuccessor(0));
4713 }
4714 EraseTerminatorAndDCECond(BI);
4715 Changed = true;
4716 }
4717 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4718 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
4719 SwitchInstProfUpdateWrapper SU(*SI);
4720 for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) {
4721 if (i->getCaseSuccessor() != BB) {
4722 ++i;
4723 continue;
4724 }
4725 BB->removePredecessor(SU->getParent());
4726 i = SU.removeCase(i);
4727 e = SU->case_end();
4728 Changed = true;
4729 }
4730 // Note that the default destination can't be removed!
4731 if (SI->getDefaultDest() != BB)
4732 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4733 } else if (auto *II = dyn_cast<InvokeInst>(TI)) {
4734 if (II->getUnwindDest() == BB) {
4735 if (DTU)
4736 DTU->applyUpdates(Updates);
4737 Updates.clear();
4738 removeUnwindEdge(TI->getParent(), DTU);
4739 Changed = true;
4740 }
4741 } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) {
4742 if (CSI->getUnwindDest() == BB) {
4743 if (DTU)
4744 DTU->applyUpdates(Updates);
4745 Updates.clear();
4746 removeUnwindEdge(TI->getParent(), DTU);
4747 Changed = true;
4748 continue;
4749 }
4750
4751 for (CatchSwitchInst::handler_iterator I = CSI->handler_begin(),
4752 E = CSI->handler_end();
4753 I != E; ++I) {
4754 if (*I == BB) {
4755 CSI->removeHandler(I);
4756 --I;
4757 --E;
4758 Changed = true;
4759 }
4760 }
4761 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4762 if (CSI->getNumHandlers() == 0) {
4763 if (CSI->hasUnwindDest()) {
4764 // Redirect all predecessors of the block containing CatchSwitchInst
4765 // to instead branch to the CatchSwitchInst's unwind destination.
4766 for (auto *PredecessorOfPredecessor : predecessors(Predecessor)) {
4767 Updates.push_back({DominatorTree::Insert, PredecessorOfPredecessor,
4768 CSI->getUnwindDest()});
4769 Updates.push_back(
4770 {DominatorTree::Delete, PredecessorOfPredecessor, Predecessor});
4771 }
4772 Predecessor->replaceAllUsesWith(CSI->getUnwindDest());
4773 } else {
4774 // Rewrite all preds to unwind to caller (or from invoke to call).
4775 if (DTU)
4776 DTU->applyUpdates(Updates);
4777 Updates.clear();
4778 SmallVector<BasicBlock *, 8> EHPreds(predecessors(Predecessor));
4779 for (BasicBlock *EHPred : EHPreds)
4780 removeUnwindEdge(EHPred, DTU);
4781 }
4782 // The catchswitch is no longer reachable.
4783 new UnreachableInst(CSI->getContext(), CSI);
4784 CSI->eraseFromParent();
4785 Changed = true;
4786 }
4787 } else if (auto *CRI = dyn_cast<CleanupReturnInst>(TI)) {
4788 (void)CRI;
4789 assert(CRI->hasUnwindDest() && CRI->getUnwindDest() == BB &&((CRI->hasUnwindDest() && CRI->getUnwindDest() ==
BB && "Expected to always have an unwind to BB.") ? static_cast
<void> (0) : __assert_fail ("CRI->hasUnwindDest() && CRI->getUnwindDest() == BB && \"Expected to always have an unwind to BB.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4790, __PRETTY_FUNCTION__))
4790 "Expected to always have an unwind to BB.")((CRI->hasUnwindDest() && CRI->getUnwindDest() ==
BB && "Expected to always have an unwind to BB.") ? static_cast
<void> (0) : __assert_fail ("CRI->hasUnwindDest() && CRI->getUnwindDest() == BB && \"Expected to always have an unwind to BB.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4790, __PRETTY_FUNCTION__))
;
4791 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4792 new UnreachableInst(TI->getContext(), TI);
4793 TI->eraseFromParent();
4794 Changed = true;
4795 }
4796 }
4797
4798 if (DTU)
4799 DTU->applyUpdates(Updates);
4800
4801 // If this block is now dead, remove it.
4802 if (pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) {
4803 // We know there are no successors, so just nuke the block.
4804 if (LoopHeaders)
4805 LoopHeaders->erase(BB);
4806 if (DTU)
4807 DTU->deleteBB(BB);
4808 else
4809 BB->eraseFromParent();
4810 return true;
4811 }
4812
4813 return Changed;
4814}
4815
4816static bool CasesAreContiguous(SmallVectorImpl<ConstantInt *> &Cases) {
4817 assert(Cases.size() >= 1)((Cases.size() >= 1) ? static_cast<void> (0) : __assert_fail
("Cases.size() >= 1", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4817, __PRETTY_FUNCTION__))
;
4818
4819 array_pod_sort(Cases.begin(), Cases.end(), ConstantIntSortPredicate);
4820 for (size_t I = 1, E = Cases.size(); I != E; ++I) {
4821 if (Cases[I - 1]->getValue() != Cases[I]->getValue() + 1)
4822 return false;
4823 }
4824 return true;
4825}
4826
4827static void createUnreachableSwitchDefault(SwitchInst *Switch,
4828 DomTreeUpdater *DTU) {
4829 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch default is dead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SimplifyCFG: switch default is dead.\n"
; } } while (false)
;
4830 auto *BB = Switch->getParent();
4831 BasicBlock *NewDefaultBlock = SplitBlockPredecessors(
4832 Switch->getDefaultDest(), Switch->getParent(), "", DTU);
4833 auto *OrigDefaultBlock = Switch->getDefaultDest();
4834 Switch->setDefaultDest(&*NewDefaultBlock);
4835 if (DTU)
4836 DTU->applyUpdates({{DominatorTree::Insert, BB, &*NewDefaultBlock},
4837 {DominatorTree::Delete, BB, OrigDefaultBlock}});
4838 SplitBlock(&*NewDefaultBlock, &NewDefaultBlock->front(), DTU);
4839 SmallVector<DominatorTree::UpdateType, 2> Updates;
4840 for (auto *Successor : successors(NewDefaultBlock))
4841 Updates.push_back({DominatorTree::Delete, NewDefaultBlock, Successor});
4842 auto *NewTerminator = NewDefaultBlock->getTerminator();
4843 new UnreachableInst(Switch->getContext(), NewTerminator);
4844 EraseTerminatorAndDCECond(NewTerminator);
4845 if (DTU)
4846 DTU->applyUpdates(Updates);
4847}
4848
4849/// Turn a switch with two reachable destinations into an integer range
4850/// comparison and branch.
4851bool SimplifyCFGOpt::TurnSwitchRangeIntoICmp(SwitchInst *SI,
4852 IRBuilder<> &Builder) {
4853 assert(SI->getNumCases() > 1 && "Degenerate switch?")((SI->getNumCases() > 1 && "Degenerate switch?"
) ? static_cast<void> (0) : __assert_fail ("SI->getNumCases() > 1 && \"Degenerate switch?\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4853, __PRETTY_FUNCTION__))
;
4854
4855 bool HasDefault =
4856 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
4857
4858 auto *BB = SI->getParent();
4859
4860 // Partition the cases into two sets with different destinations.
4861 BasicBlock *DestA = HasDefault ? SI->getDefaultDest() : nullptr;
4862 BasicBlock *DestB = nullptr;
4863 SmallVector<ConstantInt *, 16> CasesA;
4864 SmallVector<ConstantInt *, 16> CasesB;
4865
4866 for (auto Case : SI->cases()) {
4867 BasicBlock *Dest = Case.getCaseSuccessor();
4868 if (!DestA)
4869 DestA = Dest;
4870 if (Dest == DestA) {
4871 CasesA.push_back(Case.getCaseValue());
4872 continue;
4873 }
4874 if (!DestB)
4875 DestB = Dest;
4876 if (Dest == DestB) {
4877 CasesB.push_back(Case.getCaseValue());
4878 continue;
4879 }
4880 return false; // More than two destinations.
4881 }
4882
4883 assert(DestA && DestB &&((DestA && DestB && "Single-destination switch should have been folded."
) ? static_cast<void> (0) : __assert_fail ("DestA && DestB && \"Single-destination switch should have been folded.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4884, __PRETTY_FUNCTION__))
4884 "Single-destination switch should have been folded.")((DestA && DestB && "Single-destination switch should have been folded."
) ? static_cast<void> (0) : __assert_fail ("DestA && DestB && \"Single-destination switch should have been folded.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4884, __PRETTY_FUNCTION__))
;
4885 assert(DestA != DestB)((DestA != DestB) ? static_cast<void> (0) : __assert_fail
("DestA != DestB", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4885, __PRETTY_FUNCTION__))
;
4886 assert(DestB != SI->getDefaultDest())((DestB != SI->getDefaultDest()) ? static_cast<void>
(0) : __assert_fail ("DestB != SI->getDefaultDest()", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4886, __PRETTY_FUNCTION__))
;
4887 assert(!CasesB.empty() && "There must be non-default cases.")((!CasesB.empty() && "There must be non-default cases."
) ? static_cast<void> (0) : __assert_fail ("!CasesB.empty() && \"There must be non-default cases.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4887, __PRETTY_FUNCTION__))
;
4888 assert(!CasesA.empty() || HasDefault)((!CasesA.empty() || HasDefault) ? static_cast<void> (0
) : __assert_fail ("!CasesA.empty() || HasDefault", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4888, __PRETTY_FUNCTION__))
;
4889
4890 // Figure out if one of the sets of cases form a contiguous range.
4891 SmallVectorImpl<ConstantInt *> *ContiguousCases = nullptr;
4892 BasicBlock *ContiguousDest = nullptr;
4893 BasicBlock *OtherDest = nullptr;
4894 if (!CasesA.empty() && CasesAreContiguous(CasesA)) {
4895 ContiguousCases = &CasesA;
4896 ContiguousDest = DestA;
4897 OtherDest = DestB;
4898 } else if (CasesAreContiguous(CasesB)) {
4899 ContiguousCases = &CasesB;
4900 ContiguousDest = DestB;
4901 OtherDest = DestA;
4902 } else
4903 return false;
4904
4905 // Start building the compare and branch.
4906
4907 Constant *Offset = ConstantExpr::getNeg(ContiguousCases->back());
4908 Constant *NumCases =
4909 ConstantInt::get(Offset->getType(), ContiguousCases->size());
4910
4911 Value *Sub = SI->getCondition();
4912 if (!Offset->isNullValue())
4913 Sub = Builder.CreateAdd(Sub, Offset, Sub->getName() + ".off");
4914
4915 Value *Cmp;
4916 // If NumCases overflowed, then all possible values jump to the successor.
4917 if (NumCases->isNullValue() && !ContiguousCases->empty())
4918 Cmp = ConstantInt::getTrue(SI->getContext());
4919 else
4920 Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch");
4921 BranchInst *NewBI = Builder.CreateCondBr(Cmp, ContiguousDest, OtherDest);
4922
4923 // Update weight for the newly-created conditional branch.
4924 if (HasBranchWeights(SI)) {
4925 SmallVector<uint64_t, 8> Weights;
4926 GetBranchWeights(SI, Weights);
4927 if (Weights.size() == 1 + SI->getNumCases()) {
4928 uint64_t TrueWeight = 0;
4929 uint64_t FalseWeight = 0;
4930 for (size_t I = 0, E = Weights.size(); I != E; ++I) {
4931 if (SI->getSuccessor(I) == ContiguousDest)
4932 TrueWeight += Weights[I];
4933 else
4934 FalseWeight += Weights[I];
4935 }
4936 while (TrueWeight > UINT32_MAX(4294967295U) || FalseWeight > UINT32_MAX(4294967295U)) {
4937 TrueWeight /= 2;
4938 FalseWeight /= 2;
4939 }
4940 setBranchWeights(NewBI, TrueWeight, FalseWeight);
4941 }
4942 }
4943
4944 // Prune obsolete incoming values off the successors' PHI nodes.
4945 for (auto BBI = ContiguousDest->begin(); isa<PHINode>(BBI); ++BBI) {
4946 unsigned PreviousEdges = ContiguousCases->size();
4947 if (ContiguousDest == SI->getDefaultDest())
4948 ++PreviousEdges;
4949 for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4950 cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4951 }
4952 for (auto BBI = OtherDest->begin(); isa<PHINode>(BBI); ++BBI) {
4953 unsigned PreviousEdges = SI->getNumCases() - ContiguousCases->size();
4954 if (OtherDest == SI->getDefaultDest())
4955 ++PreviousEdges;
4956 for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4957 cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4958 }
4959
4960 // Clean up the default block - it may have phis or other instructions before
4961 // the unreachable terminator.
4962 if (!HasDefault)
4963 createUnreachableSwitchDefault(SI, DTU);
4964
4965 auto *UnreachableDefault = SI->getDefaultDest();
4966
4967 // Drop the switch.
4968 SI->eraseFromParent();
4969
4970 if (!HasDefault && DTU)
4971 DTU->applyUpdates({{DominatorTree::Delete, BB, UnreachableDefault}});
4972
4973 return true;
4974}
4975
4976/// Compute masked bits for the condition of a switch
4977/// and use it to remove dead cases.
4978static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU,
4979 AssumptionCache *AC,
4980 const DataLayout &DL) {
4981 Value *Cond = SI->getCondition();
4982 unsigned Bits = Cond->getType()->getIntegerBitWidth();
4983 KnownBits Known = computeKnownBits(Cond, DL, 0, AC, SI);
4984
4985 // We can also eliminate cases by determining that their values are outside of
4986 // the limited range of the condition based on how many significant (non-sign)
4987 // bits are in the condition value.
4988 unsigned ExtraSignBits = ComputeNumSignBits(Cond, DL, 0, AC, SI) - 1;
4989 unsigned MaxSignificantBitsInCond = Bits - ExtraSignBits;
4990
4991 // Gather dead cases.
4992 SmallVector<ConstantInt *, 8> DeadCases;
4993 SmallMapVector<BasicBlock *, int, 8> NumPerSuccessorCases;
4994 for (auto &Case : SI->cases()) {
4995 auto *Successor = Case.getCaseSuccessor();
4996 ++NumPerSuccessorCases[Successor];
4997 const APInt &CaseVal = Case.getCaseValue()->getValue();
4998 if (Known.Zero.intersects(CaseVal) || !Known.One.isSubsetOf(CaseVal) ||
4999 (CaseVal.getMinSignedBits() > MaxSignificantBitsInCond)) {
5000 DeadCases.push_back(Case.getCaseValue());
5001 --NumPerSuccessorCases[Successor];
5002 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseValdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SimplifyCFG: switch case "
<< CaseVal << " is dead.\n"; } } while (false)
5003 << " is dead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SimplifyCFG: switch case "
<< CaseVal << " is dead.\n"; } } while (false)
;
5004 }
5005 }
5006
5007 // If we can prove that the cases must cover all possible values, the
5008 // default destination becomes dead and we can remove it. If we know some
5009 // of the bits in the value, we can use that to more precisely compute the
5010 // number of possible unique case values.
5011 bool HasDefault =
5012 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
5013 const unsigned NumUnknownBits =
5014 Bits - (Known.Zero | Known.One).countPopulation();
5015 assert(NumUnknownBits <= Bits)((NumUnknownBits <= Bits) ? static_cast<void> (0) : __assert_fail
("NumUnknownBits <= Bits", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5015, __PRETTY_FUNCTION__))
;
5016 if (HasDefault && DeadCases.empty() &&
5017 NumUnknownBits < 64 /* avoid overflow */ &&
5018 SI->getNumCases() == (1ULL << NumUnknownBits)) {
5019 createUnreachableSwitchDefault(SI, DTU);
5020 return true;
5021 }
5022
5023 if (DeadCases.empty())
5024 return false;
5025
5026 SwitchInstProfUpdateWrapper SIW(*SI);
5027 for (ConstantInt *DeadCase : DeadCases) {
5028 SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase);
5029 assert(CaseI != SI->case_default() &&((CaseI != SI->case_default() && "Case was not found. Probably mistake in DeadCases forming."
) ? static_cast<void> (0) : __assert_fail ("CaseI != SI->case_default() && \"Case was not found. Probably mistake in DeadCases forming.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5030, __PRETTY_FUNCTION__))
5030 "Case was not found. Probably mistake in DeadCases forming.")((CaseI != SI->case_default() && "Case was not found. Probably mistake in DeadCases forming."
) ? static_cast<void> (0) : __assert_fail ("CaseI != SI->case_default() && \"Case was not found. Probably mistake in DeadCases forming.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5030, __PRETTY_FUNCTION__))
;
5031 // Prune unused values from PHI nodes.
5032 CaseI->getCaseSuccessor()->removePredecessor(SI->getParent());
5033 SIW.removeCase(CaseI);
5034 }
5035
5036 std::vector<DominatorTree::UpdateType> Updates;
5037 for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
5038 if (I.second == 0)
5039 Updates.push_back({DominatorTree::Delete, SI->getParent(), I.first});
5040 if (DTU)
5041 DTU->applyUpdates(Updates);
5042
5043 return true;
5044}
5045
5046/// If BB would be eligible for simplification by
5047/// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated
5048/// by an unconditional branch), look at the phi node for BB in the successor
5049/// block and see if the incoming value is equal to CaseValue. If so, return
5050/// the phi node, and set PhiIndex to BB's index in the phi node.
5051static PHINode *FindPHIForConditionForwarding(ConstantInt *CaseValue,
5052 BasicBlock *BB, int *PhiIndex) {
5053 if (BB->getFirstNonPHIOrDbg() != BB->getTerminator())
5054 return nullptr; // BB must be empty to be a candidate for simplification.
5055 if (!BB->getSinglePredecessor())
5056 return nullptr; // BB must be dominated by the switch.
5057
5058 BranchInst *Branch = dyn_cast<BranchInst>(BB->getTerminator());
5059 if (!Branch || !Branch->isUnconditional())
5060 return nullptr; // Terminator must be unconditional branch.
5061
5062 BasicBlock *Succ = Branch->getSuccessor(0);
5063
5064 for (PHINode &PHI : Succ->phis()) {
5065 int Idx = PHI.getBasicBlockIndex(BB);
5066 assert(Idx >= 0 && "PHI has no entry for predecessor?")((Idx >= 0 && "PHI has no entry for predecessor?")
? static_cast<void> (0) : __assert_fail ("Idx >= 0 && \"PHI has no entry for predecessor?\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5066, __PRETTY_FUNCTION__))
;
5067
5068 Value *InValue = PHI.getIncomingValue(Idx);
5069 if (InValue != CaseValue)
5070 continue;
5071
5072 *PhiIndex = Idx;
5073 return &PHI;
5074 }
5075
5076 return nullptr;
5077}
5078
5079/// Try to forward the condition of a switch instruction to a phi node
5080/// dominated by the switch, if that would mean that some of the destination
5081/// blocks of the switch can be folded away. Return true if a change is made.
5082static bool ForwardSwitchConditionToPHI(SwitchInst *SI) {
5083 using ForwardingNodesMap = DenseMap<PHINode *, SmallVector<int, 4>>;
5084
5085 ForwardingNodesMap ForwardingNodes;
5086 BasicBlock *SwitchBlock = SI->getParent();
5087 bool Changed = false;
5088 for (auto &Case : SI->cases()) {
5089 ConstantInt *CaseValue = Case.getCaseValue();
5090 BasicBlock *CaseDest = Case.getCaseSuccessor();
5091
5092 // Replace phi operands in successor blocks that are using the constant case
5093 // value rather than the switch condition variable:
5094 // switchbb:
5095 // switch i32 %x, label %default [
5096 // i32 17, label %succ
5097 // ...
5098 // succ:
5099 // %r = phi i32 ... [ 17, %switchbb ] ...
5100 // -->
5101 // %r = phi i32 ... [ %x, %switchbb ] ...
5102
5103 for (PHINode &Phi : CaseDest->phis()) {
5104 // This only works if there is exactly 1 incoming edge from the switch to
5105 // a phi. If there is >1, that means multiple cases of the switch map to 1
5106 // value in the phi, and that phi value is not the switch condition. Thus,
5107 // this transform would not make sense (the phi would be invalid because
5108 // a phi can't have different incoming values from the same block).
5109 int SwitchBBIdx = Phi.getBasicBlockIndex(SwitchBlock);
5110 if (Phi.getIncomingValue(SwitchBBIdx) == CaseValue &&
5111 count(Phi.blocks(), SwitchBlock) == 1) {
5112 Phi.setIncomingValue(SwitchBBIdx, SI->getCondition());
5113 Changed = true;
5114 }
5115 }
5116
5117 // Collect phi nodes that are indirectly using this switch's case constants.
5118 int PhiIdx;
5119 if (auto *Phi = FindPHIForConditionForwarding(CaseValue, CaseDest, &PhiIdx))
5120 ForwardingNodes[Phi].push_back(PhiIdx);
5121 }
5122
5123 for (auto &ForwardingNode : ForwardingNodes) {
5124 PHINode *Phi = ForwardingNode.first;
5125 SmallVectorImpl<int> &Indexes = ForwardingNode.second;
5126 if (Indexes.size() < 2)
5127 continue;
5128
5129 for (int Index : Indexes)
5130 Phi->setIncomingValue(Index, SI->getCondition());
5131 Changed = true;
5132 }
5133
5134 return Changed;
5135}
5136
5137/// Return true if the backend will be able to handle
5138/// initializing an array of constants like C.
5139static bool ValidLookupTableConstant(Constant *C, const TargetTransformInfo &TTI) {
5140 if (C->isThreadDependent())
5141 return false;
5142 if (C->isDLLImportDependent())
5143 return false;
5144
5145 if (!isa<ConstantFP>(C) && !isa<ConstantInt>(C) &&
5146 !isa<ConstantPointerNull>(C) && !isa<GlobalValue>(C) &&
5147 !isa<UndefValue>(C) && !isa<ConstantExpr>(C))
5148 return false;
5149
5150 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
5151 if (!CE->isGEPWithNoNotionalOverIndexing())
5152 return false;
5153 if (!ValidLookupTableConstant(CE->getOperand(0), TTI))
5154 return false;
5155 }
5156
5157 if (!TTI.shouldBuildLookupTablesForConstant(C))
5158 return false;
5159
5160 return true;
5161}
5162
5163/// If V is a Constant, return it. Otherwise, try to look up
5164/// its constant value in ConstantPool, returning 0 if it's not there.
5165static Constant *
5166LookupConstant(Value *V,
5167 const SmallDenseMap<Value *, Constant *> &ConstantPool) {
5168 if (Constant *C = dyn_cast<Constant>(V))
5169 return C;
5170 return ConstantPool.lookup(V);
5171}
5172
5173/// Try to fold instruction I into a constant. This works for
5174/// simple instructions such as binary operations where both operands are
5175/// constant or can be replaced by constants from the ConstantPool. Returns the
5176/// resulting constant on success, 0 otherwise.
5177static Constant *
5178ConstantFold(Instruction *I, const DataLayout &DL,
5179 const SmallDenseMap<Value *, Constant *> &ConstantPool) {
5180 if (SelectInst *Select = dyn_cast<SelectInst>(I)) {
5181 Constant *A = LookupConstant(Select->getCondition(), ConstantPool);
5182 if (!A)
5183 return nullptr;
5184 if (A->isAllOnesValue())
5185 return LookupConstant(Select->getTrueValue(), ConstantPool);
5186 if (A->isNullValue())
5187 return LookupConstant(Select->getFalseValue(), ConstantPool);
5188 return nullptr;
5189 }
5190
5191 SmallVector<Constant *, 4> COps;
5192 for (unsigned N = 0, E = I->getNumOperands(); N != E; ++N) {
5193 if (Constant *A = LookupConstant(I->getOperand(N), ConstantPool))
5194 COps.push_back(A);
5195 else
5196 return nullptr;
5197 }
5198
5199 if (CmpInst *Cmp = dyn_cast<CmpInst>(I)) {
5200 return ConstantFoldCompareInstOperands(Cmp->getPredicate(), COps[0],
5201 COps[1], DL);
5202 }
5203
5204 return ConstantFoldInstOperands(I, COps, DL);
5205}
5206
5207/// Try to determine the resulting constant values in phi nodes
5208/// at the common destination basic block, *CommonDest, for one of the case
5209/// destionations CaseDest corresponding to value CaseVal (0 for the default
5210/// case), of a switch instruction SI.
5211static bool
5212GetCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest,
5213 BasicBlock **CommonDest,
5214 SmallVectorImpl<std::pair<PHINode *, Constant *>> &Res,
5215 const DataLayout &DL, const TargetTransformInfo &TTI) {
5216 // The block from which we enter the common destination.
5217 BasicBlock *Pred = SI->getParent();
5218
5219 // If CaseDest is empty except for some side-effect free instructions through
5220 // which we can constant-propagate the CaseVal, continue to its successor.
5221 SmallDenseMap<Value *, Constant *> ConstantPool;
5222 ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal));
5223 for (Instruction &I :CaseDest->instructionsWithoutDebug()) {
5224 if (I.isTerminator()) {
5225 // If the terminator is a simple branch, continue to the next block.
5226 if (I.getNumSuccessors() != 1 || I.isExceptionalTerminator())
5227 return false;
5228 Pred = CaseDest;
5229 CaseDest = I.getSuccessor(0);
5230 } else if (Constant *C = ConstantFold(&I, DL, ConstantPool)) {
5231 // Instruction is side-effect free and constant.
5232
5233 // If the instruction has uses outside this block or a phi node slot for
5234 // the block, it is not safe to bypass the instruction since it would then
5235 // no longer dominate all its uses.
5236 for (auto &Use : I.uses()) {
5237 User *User = Use.getUser();
5238 if (Instruction *I = dyn_cast<Instruction>(User))
5239 if (I->getParent() == CaseDest)
5240 continue;
5241 if (PHINode *Phi = dyn_cast<PHINode>(User))
5242 if (Phi->getIncomingBlock(Use) == CaseDest)
5243 continue;
5244 return false;
5245 }
5246
5247 ConstantPool.insert(std::make_pair(&I, C));
5248 } else {
5249 break;
5250 }
5251 }
5252
5253 // If we did not have a CommonDest before, use the current one.
5254 if (!*CommonDest)
5255 *CommonDest = CaseDest;
5256 // If the destination isn't the common one, abort.
5257 if (CaseDest != *CommonDest)
5258 return false;
5259
5260 // Get the values for this case from phi nodes in the destination block.
5261 for (PHINode &PHI : (*CommonDest)->phis()) {
5262 int Idx = PHI.getBasicBlockIndex(Pred);
5263 if (Idx == -1)
5264 continue;
5265
5266 Constant *ConstVal =
5267 LookupConstant(PHI.getIncomingValue(Idx), ConstantPool);
5268 if (!ConstVal)
5269 return false;
5270
5271 // Be conservative about which kinds of constants we support.
5272 if (!ValidLookupTableConstant(ConstVal, TTI))
5273 return false;
5274
5275 Res.push_back(std::make_pair(&PHI, ConstVal));
5276 }
5277
5278 return Res.size() > 0;
5279}
5280
5281// Helper function used to add CaseVal to the list of cases that generate
5282// Result. Returns the updated number of cases that generate this result.
5283static uintptr_t MapCaseToResult(ConstantInt *CaseVal,
5284 SwitchCaseResultVectorTy &UniqueResults,
5285 Constant *Result) {
5286 for (auto &I : UniqueResults) {
5287 if (I.first == Result) {
5288 I.second.push_back(CaseVal);
5289 return I.second.size();
5290 }
5291 }
5292 UniqueResults.push_back(
5293 std::make_pair(Result, SmallVector<ConstantInt *, 4>(1, CaseVal)));
5294 return 1;
5295}
5296
5297// Helper function that initializes a map containing
5298// results for the PHI node of the common destination block for a switch
5299// instruction. Returns false if multiple PHI nodes have been found or if
5300// there is not a common destination block for the switch.
5301static bool
5302InitializeUniqueCases(SwitchInst *SI, PHINode *&PHI, BasicBlock *&CommonDest,
5303 SwitchCaseResultVectorTy &UniqueResults,
5304 Constant *&DefaultResult, const DataLayout &DL,
5305 const TargetTransformInfo &TTI,
5306 uintptr_t MaxUniqueResults, uintptr_t MaxCasesPerResult) {
5307 for (auto &I : SI->cases()) {
5308 ConstantInt *CaseVal = I.getCaseValue();
5309
5310 // Resulting value at phi nodes for this case value.
5311 SwitchCaseResultsTy Results;
5312 if (!GetCaseResults(SI, CaseVal, I.getCaseSuccessor(), &CommonDest, Results,
5313 DL, TTI))
5314 return false;
5315
5316 // Only one value per case is permitted.
5317 if (Results.size() > 1)
5318 return false;
5319
5320 // Add the case->result mapping to UniqueResults.
5321 const uintptr_t NumCasesForResult =
5322 MapCaseToResult(CaseVal, UniqueResults, Results.begin()->second);
5323
5324 // Early out if there are too many cases for this result.
5325 if (NumCasesForResult > MaxCasesPerResult)
5326 return false;
5327
5328 // Early out if there are too many unique results.
5329 if (UniqueResults.size() > MaxUniqueResults)
5330 return false;
5331
5332 // Check the PHI consistency.
5333 if (!PHI)
5334 PHI = Results[0].first;
5335 else if (PHI != Results[0].first)
5336 return false;
5337 }
5338 // Find the default result value.
5339 SmallVector<std::pair<PHINode *, Constant *>, 1> DefaultResults;
5340 BasicBlock *DefaultDest = SI->getDefaultDest();
5341 GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, DefaultResults,
5342 DL, TTI);
5343 // If the default value is not found abort unless the default destination
5344 // is unreachable.
5345 DefaultResult =
5346 DefaultResults.size() == 1 ? DefaultResults.begin()->second : nullptr;
5347 if ((!DefaultResult &&
5348 !isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg())))
5349 return false;
5350
5351 return true;
5352}
5353
5354// Helper function that checks if it is possible to transform a switch with only
5355// two cases (or two cases + default) that produces a result into a select.
5356// Example:
5357// switch (a) {
5358// case 10: %0 = icmp eq i32 %a, 10
5359// return 10; %1 = select i1 %0, i32 10, i32 4
5360// case 20: ----> %2 = icmp eq i32 %a, 20
5361// return 2; %3 = select i1 %2, i32 2, i32 %1
5362// default:
5363// return 4;
5364// }
5365static Value *ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector,
5366 Constant *DefaultResult, Value *Condition,
5367 IRBuilder<> &Builder) {
5368 assert(ResultVector.size() == 2 &&((ResultVector.size() == 2 && "We should have exactly two unique results at this point"
) ? static_cast<void> (0) : __assert_fail ("ResultVector.size() == 2 && \"We should have exactly two unique results at this point\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5369, __PRETTY_FUNCTION__))
5369 "We should have exactly two unique results at this point")((ResultVector.size() == 2 && "We should have exactly two unique results at this point"
) ? static_cast<void> (0) : __assert_fail ("ResultVector.size() == 2 && \"We should have exactly two unique results at this point\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5369, __PRETTY_FUNCTION__))
;
5370 // If we are selecting between only two cases transform into a simple
5371 // select or a two-way select if default is possible.
5372 if (ResultVector[0].second.size() == 1 &&
5373 ResultVector[1].second.size() == 1) {
5374 ConstantInt *const FirstCase = ResultVector[0].second[0];
5375 ConstantInt *const SecondCase = ResultVector[1].second[0];
5376
5377 bool DefaultCanTrigger = DefaultResult;
5378 Value *SelectValue = ResultVector[1].first;
5379 if (DefaultCanTrigger) {
5380 Value *const ValueCompare =
5381 Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp");
5382 SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first,
5383 DefaultResult, "switch.select");
5384 }
5385 Value *const ValueCompare =
5386 Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp");
5387 return Builder.CreateSelect(ValueCompare, ResultVector[0].first,
5388 SelectValue, "switch.select");
5389 }
5390
5391 return nullptr;
5392}
5393
5394// Helper function to cleanup a switch instruction that has been converted into
5395// a select, fixing up PHI nodes and basic blocks.
5396static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI,
5397 Value *SelectValue,
5398 IRBuilder<> &Builder,
5399 DomTreeUpdater *DTU) {
5400 std::vector<DominatorTree::UpdateType> Updates;
5401
5402 BasicBlock *SelectBB = SI->getParent();
5403 BasicBlock *DestBB = PHI->getParent();
5404
5405 if (!is_contained(predecessors(DestBB), SelectBB))
5406 Updates.push_back({DominatorTree::Insert, SelectBB, DestBB});
5407 Builder.CreateBr(DestBB);
5408
5409 // Remove the switch.
5410
5411 while (PHI->getBasicBlockIndex(SelectBB) >= 0)
5412 PHI->removeIncomingValue(SelectBB);
5413 PHI->addIncoming(SelectValue, SelectBB);
5414
5415 for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
5416 BasicBlock *Succ = SI->getSuccessor(i);
5417
5418 if (Succ == DestBB)
5419 continue;
5420 Succ->removePredecessor(SelectBB);
5421 Updates.push_back({DominatorTree::Delete, SelectBB, Succ});
5422 }
5423 SI->eraseFromParent();
5424 if (DTU)
5425 DTU->applyUpdates(Updates);
5426}
5427
5428/// If the switch is only used to initialize one or more
5429/// phi nodes in a common successor block with only two different
5430/// constant values, replace the switch with select.
5431static bool switchToSelect(SwitchInst *SI, IRBuilder<> &Builder,
5432 DomTreeUpdater *DTU, const DataLayout &DL,
5433 const TargetTransformInfo &TTI) {
5434 Value *const Cond = SI->getCondition();
5435 PHINode *PHI = nullptr;
5436 BasicBlock *CommonDest = nullptr;
5437 Constant *DefaultResult;
5438 SwitchCaseResultVectorTy UniqueResults;
5439 // Collect all the cases that will deliver the same value from the switch.
5440 if (!InitializeUniqueCases(SI, PHI, CommonDest, UniqueResults, DefaultResult,
5441 DL, TTI, 2, 1))
5442 return false;
5443 // Selects choose between maximum two values.
5444 if (UniqueResults.size() != 2)
5445 return false;
5446 assert(PHI != nullptr && "PHI for value select not found")((PHI != nullptr && "PHI for value select not found")
? static_cast<void> (0) : __assert_fail ("PHI != nullptr && \"PHI for value select not found\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5446, __PRETTY_FUNCTION__))
;
5447
5448 Builder.SetInsertPoint(SI);
5449 Value *SelectValue =
5450 ConvertTwoCaseSwitch(UniqueResults, DefaultResult, Cond, Builder);
5451 if (SelectValue) {
5452 RemoveSwitchAfterSelectConversion(SI, PHI, SelectValue, Builder, DTU);
5453 return true;
5454 }
5455 // The switch couldn't be converted into a select.
5456 return false;
5457}
5458
5459namespace {
5460
5461/// This class represents a lookup table that can be used to replace a switch.
5462class SwitchLookupTable {
5463public:
5464 /// Create a lookup table to use as a switch replacement with the contents
5465 /// of Values, using DefaultValue to fill any holes in the table.
5466 SwitchLookupTable(
5467 Module &M, uint64_t TableSize, ConstantInt *Offset,
5468 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
5469 Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName);
5470
5471 /// Build instructions with Builder to retrieve the value at
5472 /// the position given by Index in the lookup table.
5473 Value *BuildLookup(Value *Index, IRBuilder<> &Builder);
5474
5475 /// Return true if a table with TableSize elements of
5476 /// type ElementType would fit in a target-legal register.
5477 static bool WouldFitInRegister(const DataLayout &DL, uint64_t TableSize,
5478 Type *ElementType);
5479
5480private:
5481 // Depending on the contents of the table, it can be represented in
5482 // different ways.
5483 enum {
5484 // For tables where each element contains the same value, we just have to
5485 // store that single value and return it for each lookup.
5486 SingleValueKind,
5487
5488 // For tables where there is a linear relationship between table index
5489 // and values. We calculate the result with a simple multiplication
5490 // and addition instead of a table lookup.
5491 LinearMapKind,
5492
5493 // For small tables with integer elements, we can pack them into a bitmap
5494 // that fits into a target-legal register. Values are retrieved by
5495 // shift and mask operations.
5496 BitMapKind,
5497
5498 // The table is stored as an array of values. Values are retrieved by load
5499 // instructions from the table.
5500 ArrayKind
5501 } Kind;
5502
5503 // For SingleValueKind, this is the single value.
5504 Constant *SingleValue = nullptr;
5505
5506 // For BitMapKind, this is the bitmap.
5507 ConstantInt *BitMap = nullptr;
5508 IntegerType *BitMapElementTy = nullptr;
5509
5510 // For LinearMapKind, these are the constants used to derive the value.
5511 ConstantInt *LinearOffset = nullptr;
5512 ConstantInt *LinearMultiplier = nullptr;
5513
5514 // For ArrayKind, this is the array.
5515 GlobalVariable *Array = nullptr;
5516};
5517
5518} // end anonymous namespace
5519
5520SwitchLookupTable::SwitchLookupTable(
5521 Module &M, uint64_t TableSize, ConstantInt *Offset,
5522 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
5523 Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName) {
5524 assert(Values.size() && "Can't build lookup table without values!")((Values.size() && "Can't build lookup table without values!"
) ? static_cast<void> (0) : __assert_fail ("Values.size() && \"Can't build lookup table without values!\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5524, __PRETTY_FUNCTION__))
;
5525 assert(TableSize >= Values.size() && "Can't fit values in table!")((TableSize >= Values.size() && "Can't fit values in table!"
) ? static_cast<void> (0) : __assert_fail ("TableSize >= Values.size() && \"Can't fit values in table!\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5525, __PRETTY_FUNCTION__))
;
5526
5527 // If all values in the table are equal, this is that value.
5528 SingleValue = Values.begin()->second;
5529
5530 Type *ValueType = Values.begin()->second->getType();
5531
5532 // Build up the table contents.
5533 SmallVector<Constant *, 64> TableContents(TableSize);
5534 for (size_t I = 0, E = Values.size(); I != E; ++I) {
5535 ConstantInt *CaseVal = Values[I].first;
5536 Constant *CaseRes = Values[I].second;
5537 assert(CaseRes->getType() == ValueType)((CaseRes->getType() == ValueType) ? static_cast<void>
(0) : __assert_fail ("CaseRes->getType() == ValueType", "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5537, __PRETTY_FUNCTION__))
;
5538
5539 uint64_t Idx = (CaseVal->getValue() - Offset->getValue()).getLimitedValue();
5540 TableContents[Idx] = CaseRes;
5541
5542 if (CaseRes != SingleValue)
5543 SingleValue = nullptr;
5544 }
5545
5546 // Fill in any holes in the table with the default result.
5547 if (Values.size() < TableSize) {
5548 assert(DefaultValue &&((DefaultValue && "Need a default value to fill the lookup table holes."
) ? static_cast<void> (0) : __assert_fail ("DefaultValue && \"Need a default value to fill the lookup table holes.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5549, __PRETTY_FUNCTION__))
5549 "Need a default value to fill the lookup table holes.")((DefaultValue && "Need a default value to fill the lookup table holes."
) ? static_cast<void> (0) : __assert_fail ("DefaultValue && \"Need a default value to fill the lookup table holes.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5549, __PRETTY_FUNCTION__))
;
5550 assert(DefaultValue->getType() == ValueType)((DefaultValue->getType() == ValueType) ? static_cast<void
> (0) : __assert_fail ("DefaultValue->getType() == ValueType"
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5550, __PRETTY_FUNCTION__))
;
5551 for (uint64_t I = 0; I < TableSize; ++I) {
5552 if (!TableContents[I])
5553 TableContents[I] = DefaultValue;
5554 }
5555
5556 if (DefaultValue != SingleValue)
5557 SingleValue = nullptr;
5558 }
5559
5560 // If each element in the table contains the same value, we only need to store
5561 // that single value.
5562 if (SingleValue) {
5563 Kind = SingleValueKind;
5564 return;
5565 }
5566
5567 // Check if we can derive the value with a linear transformation from the
5568 // table index.
5569 if (isa<IntegerType>(ValueType)) {
5570 bool LinearMappingPossible = true;
5571 APInt PrevVal;
5572 APInt DistToPrev;
5573 assert(TableSize >= 2 && "Should be a SingleValue table.")((TableSize >= 2 && "Should be a SingleValue table."
) ? static_cast<void> (0) : __assert_fail ("TableSize >= 2 && \"Should be a SingleValue table.\""
, "/build/llvm-toolchain-snapshot-12~++20210121111113+bee486851c1a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5573, __PRETTY_FUNCTION__))
;
5574 // Check if there is the same distance between two consecutive values.
5575 for (uint64_t I = 0; I < TableSize; ++I) {
5576 ConstantInt *ConstVal = dyn_cast<ConstantInt>(TableContents[I]);
5577 if (!ConstVal) {
5578 // This is an undef. We could deal with it, but undefs in lookup tables
5579 // are very seldom. It's probably not worth the additional complexity.
5580 LinearMappingPossible = false;
5581 break;
5582 }
5583 const APInt &Val = ConstVal->getValue();
5584 if (I != 0) {
5585 APInt Dist = Val - PrevVal;
5586 if (I == 1) {
5587 DistToPrev = Dist;
5588 } else if (Dist != DistToPrev) {
5589 LinearMappingPossible = false;
5590 break;
5591 }
5592 }
5593 PrevVal = Val;
5594 }
5595 if (LinearMappingPossible) {
5596 LinearOffset = cast<ConstantInt>(TableContents[0]);
5597 LinearMultiplier = ConstantInt::get(M.getContext(), DistToPrev);
5598 Kind = LinearMapKind;
5599 ++NumLinearMaps;
5600 return;
5601 }
5602 }
5603
5604 // If the type is integer and the table fits in a register, build a bitmap.
5605 if (WouldFitInRegister(DL, TableSize, ValueType)) {
5606 IntegerType *IT = cast<IntegerType>(ValueType);
5607 APInt TableInt(TableSize * IT->getBitWidth(), 0);
5608 for (uint64_t I = TableSize; I > 0; --I) {
5609 TableInt <<= IT->getBitWidth();
5610 // Insert values into the bitmap. Undef values are set to zero.
5611 if (!isa<UndefValue>(TableContents[I - 1])) {
5612 ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]);
5613 TableInt |= Val->getValue().zext(TableInt.getBitWidth());
5614 }
5615 }
5616 BitMap = ConstantInt::get(M.getContext(), TableInt);
5617 BitMapElementTy = IT;
5618 Kind = BitMapKind;
5619 ++NumBitMaps;
5620 return;
5621 }
5622
5623 // Store the table in an array.
5624 ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize);
5625 Constant *Initializer = ConstantArray::get(ArrayTy, TableContents);
5626
5627 Array = new GlobalVariable(M, ArrayTy, /*isConstant=*/true,
5628 GlobalVariable::PrivateLinkage, Initializer,
5629 "switch.table." + FuncName);
5630 Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
5631 // Set the alignment to that of an array items. We will be only loading one
5632 // value out of it.
5633 Array->setAlignment(Align(DL.getPrefTypeAlignment(ValueType)));
5634 Kind = ArrayKind;
5635}
5636
5637Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) {
5638 switch (Kind) {
5639 case SingleValueKind:
5640 return SingleValue;
5641 case LinearMapKind: {
5642 // Derive the result value from the input value.
5643 Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(),
5644 false, "switch.idx.cast");
5645 if (!LinearMultiplier->isOne())
5646 Result = Builder.CreateMul(Result, LinearMultiplier, "switch.idx.mult");
5647 if (!LinearOffset->isZero())
5648 Result = Builder.CreateAdd(Result, LinearOffset, "switch.offset");
5649 return Result;
5650 }
5651 case BitMapKind: {
5652 // Type of the bitmap (e.g. i59).
5653 IntegerType *MapTy = BitMap->getType();
5654
5655 // Cast Index to the same type as the bitmap.
5656 // Note: The Index is <= the number of elements in the table, so
5657 // truncating it to the width of the bitmask is safe.
5658 Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast");
5659
5660 // Multiply the shift amount by the element width.
5661 ShiftAmt = Builder.CreateMul(
5662 ShiftAmt, ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()),
5663 "switch.shiftamt");
5664
5665 // Shift down.
5666 Value *DownShifted =
5667 Builder.CreateLShr(BitMap, ShiftAmt, "switch.downshift");
5668 // Mask off.