Bug Summary

File:llvm/lib/Transforms/Utils/SimplifyCFG.cpp
Warning:line 4247, column 24
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SimplifyCFG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/lib/Transforms/Utils -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/lib/Transforms/Utils -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/lib/Transforms/Utils -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-04-14-063029-18377-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
1//===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Peephole optimize the CFG.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APInt.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/Optional.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/ScopeExit.h"
20#include "llvm/ADT/Sequence.h"
21#include "llvm/ADT/SetOperations.h"
22#include "llvm/ADT/SetVector.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/Analysis/AssumptionCache.h"
28#include "llvm/Analysis/ConstantFolding.h"
29#include "llvm/Analysis/EHPersonalities.h"
30#include "llvm/Analysis/GuardUtils.h"
31#include "llvm/Analysis/InstructionSimplify.h"
32#include "llvm/Analysis/MemorySSA.h"
33#include "llvm/Analysis/MemorySSAUpdater.h"
34#include "llvm/Analysis/TargetTransformInfo.h"
35#include "llvm/Analysis/ValueTracking.h"
36#include "llvm/IR/Attributes.h"
37#include "llvm/IR/BasicBlock.h"
38#include "llvm/IR/CFG.h"
39#include "llvm/IR/Constant.h"
40#include "llvm/IR/ConstantRange.h"
41#include "llvm/IR/Constants.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/DerivedTypes.h"
44#include "llvm/IR/Function.h"
45#include "llvm/IR/GlobalValue.h"
46#include "llvm/IR/GlobalVariable.h"
47#include "llvm/IR/IRBuilder.h"
48#include "llvm/IR/InstrTypes.h"
49#include "llvm/IR/Instruction.h"
50#include "llvm/IR/Instructions.h"
51#include "llvm/IR/IntrinsicInst.h"
52#include "llvm/IR/Intrinsics.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/MDBuilder.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/NoFolder.h"
58#include "llvm/IR/Operator.h"
59#include "llvm/IR/PatternMatch.h"
60#include "llvm/IR/PseudoProbe.h"
61#include "llvm/IR/Type.h"
62#include "llvm/IR/Use.h"
63#include "llvm/IR/User.h"
64#include "llvm/IR/Value.h"
65#include "llvm/IR/ValueHandle.h"
66#include "llvm/Support/BranchProbability.h"
67#include "llvm/Support/Casting.h"
68#include "llvm/Support/CommandLine.h"
69#include "llvm/Support/Debug.h"
70#include "llvm/Support/ErrorHandling.h"
71#include "llvm/Support/KnownBits.h"
72#include "llvm/Support/MathExtras.h"
73#include "llvm/Support/raw_ostream.h"
74#include "llvm/Transforms/Utils/BasicBlockUtils.h"
75#include "llvm/Transforms/Utils/Local.h"
76#include "llvm/Transforms/Utils/SSAUpdater.h"
77#include "llvm/Transforms/Utils/ValueMapper.h"
78#include <algorithm>
79#include <cassert>
80#include <climits>
81#include <cstddef>
82#include <cstdint>
83#include <iterator>
84#include <map>
85#include <set>
86#include <tuple>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace PatternMatch;
92
93#define DEBUG_TYPE"simplifycfg" "simplifycfg"
94
95cl::opt<bool> llvm::RequireAndPreserveDomTree(
96 "simplifycfg-require-and-preserve-domtree", cl::Hidden, cl::ZeroOrMore,
97 cl::init(false),
98 cl::desc("Temorary development switch used to gradually uplift SimplifyCFG "
99 "into preserving DomTree,"));
100
101// Chosen as 2 so as to be cheap, but still to have enough power to fold
102// a select, so the "clamp" idiom (of a min followed by a max) will be caught.
103// To catch this, we need to fold a compare and a select, hence '2' being the
104// minimum reasonable default.
105static cl::opt<unsigned> PHINodeFoldingThreshold(
106 "phi-node-folding-threshold", cl::Hidden, cl::init(2),
107 cl::desc(
108 "Control the amount of phi node folding to perform (default = 2)"));
109
110static cl::opt<unsigned> TwoEntryPHINodeFoldingThreshold(
111 "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4),
112 cl::desc("Control the maximal total instruction cost that we are willing "
113 "to speculatively execute to fold a 2-entry PHI node into a "
114 "select (default = 4)"));
115
116static cl::opt<bool> DupRet(
117 "simplifycfg-dup-ret", cl::Hidden, cl::init(false),
118 cl::desc("Duplicate return instructions into unconditional branches"));
119
120static cl::opt<bool>
121 HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true),
122 cl::desc("Hoist common instructions up to the parent block"));
123
124static cl::opt<bool>
125 SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
126 cl::desc("Sink common instructions down to the end block"));
127
128static cl::opt<bool> HoistCondStores(
129 "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true),
130 cl::desc("Hoist conditional stores if an unconditional store precedes"));
131
132static cl::opt<bool> MergeCondStores(
133 "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true),
134 cl::desc("Hoist conditional stores even if an unconditional store does not "
135 "precede - hoist multiple conditional stores into a single "
136 "predicated store"));
137
138static cl::opt<bool> MergeCondStoresAggressively(
139 "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false),
140 cl::desc("When merging conditional stores, do so even if the resultant "
141 "basic blocks are unlikely to be if-converted as a result"));
142
143static cl::opt<bool> SpeculateOneExpensiveInst(
144 "speculate-one-expensive-inst", cl::Hidden, cl::init(true),
145 cl::desc("Allow exactly one expensive instruction to be speculatively "
146 "executed"));
147
148static cl::opt<unsigned> MaxSpeculationDepth(
149 "max-speculation-depth", cl::Hidden, cl::init(10),
150 cl::desc("Limit maximum recursion depth when calculating costs of "
151 "speculatively executed instructions"));
152
153static cl::opt<int>
154MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden, cl::init(10),
155 cl::desc("Max size of a block which is still considered "
156 "small enough to thread through"));
157
158// Two is chosen to allow one negation and a logical combine.
159static cl::opt<unsigned>
160 BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden,
161 cl::init(2),
162 cl::desc("Maximum cost of combining conditions when "
163 "folding branches"));
164
165STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps")static llvm::Statistic NumBitMaps = {"simplifycfg", "NumBitMaps"
, "Number of switch instructions turned into bitmaps"}
;
166STATISTIC(NumLinearMaps,static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps"
, "Number of switch instructions turned into linear mapping"}
167 "Number of switch instructions turned into linear mapping")static llvm::Statistic NumLinearMaps = {"simplifycfg", "NumLinearMaps"
, "Number of switch instructions turned into linear mapping"}
;
168STATISTIC(NumLookupTables,static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables"
, "Number of switch instructions turned into lookup tables"}
169 "Number of switch instructions turned into lookup tables")static llvm::Statistic NumLookupTables = {"simplifycfg", "NumLookupTables"
, "Number of switch instructions turned into lookup tables"}
;
170STATISTIC(static llvm::Statistic NumLookupTablesHoles = {"simplifycfg",
"NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)"
}
171 NumLookupTablesHoles,static llvm::Statistic NumLookupTablesHoles = {"simplifycfg",
"NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)"
}
172 "Number of switch instructions turned into lookup tables (holes checked)")static llvm::Statistic NumLookupTablesHoles = {"simplifycfg",
"NumLookupTablesHoles", "Number of switch instructions turned into lookup tables (holes checked)"
}
;
173STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares")static llvm::Statistic NumTableCmpReuses = {"simplifycfg", "NumTableCmpReuses"
, "Number of reused switch table lookup compares"}
;
174STATISTIC(NumFoldValueComparisonIntoPredecessors,static llvm::Statistic NumFoldValueComparisonIntoPredecessors
= {"simplifycfg", "NumFoldValueComparisonIntoPredecessors", "Number of value comparisons folded into predecessor basic blocks"
}
175 "Number of value comparisons folded into predecessor basic blocks")static llvm::Statistic NumFoldValueComparisonIntoPredecessors
= {"simplifycfg", "NumFoldValueComparisonIntoPredecessors", "Number of value comparisons folded into predecessor basic blocks"
}
;
176STATISTIC(NumFoldBranchToCommonDest,static llvm::Statistic NumFoldBranchToCommonDest = {"simplifycfg"
, "NumFoldBranchToCommonDest", "Number of branches folded into predecessor basic block"
}
177 "Number of branches folded into predecessor basic block")static llvm::Statistic NumFoldBranchToCommonDest = {"simplifycfg"
, "NumFoldBranchToCommonDest", "Number of branches folded into predecessor basic block"
}
;
178STATISTIC(static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode"
, "Number of common instruction 'blocks' hoisted up to the begin block"
}
179 NumHoistCommonCode,static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode"
, "Number of common instruction 'blocks' hoisted up to the begin block"
}
180 "Number of common instruction 'blocks' hoisted up to the begin block")static llvm::Statistic NumHoistCommonCode = {"simplifycfg", "NumHoistCommonCode"
, "Number of common instruction 'blocks' hoisted up to the begin block"
}
;
181STATISTIC(NumHoistCommonInstrs,static llvm::Statistic NumHoistCommonInstrs = {"simplifycfg",
"NumHoistCommonInstrs", "Number of common instructions hoisted up to the begin block"
}
182 "Number of common instructions hoisted up to the begin block")static llvm::Statistic NumHoistCommonInstrs = {"simplifycfg",
"NumHoistCommonInstrs", "Number of common instructions hoisted up to the begin block"
}
;
183STATISTIC(NumSinkCommonCode,static llvm::Statistic NumSinkCommonCode = {"simplifycfg", "NumSinkCommonCode"
, "Number of common instruction 'blocks' sunk down to the end block"
}
184 "Number of common instruction 'blocks' sunk down to the end block")static llvm::Statistic NumSinkCommonCode = {"simplifycfg", "NumSinkCommonCode"
, "Number of common instruction 'blocks' sunk down to the end block"
}
;
185STATISTIC(NumSinkCommonInstrs,static llvm::Statistic NumSinkCommonInstrs = {"simplifycfg", "NumSinkCommonInstrs"
, "Number of common instructions sunk down to the end block"}
186 "Number of common instructions sunk down to the end block")static llvm::Statistic NumSinkCommonInstrs = {"simplifycfg", "NumSinkCommonInstrs"
, "Number of common instructions sunk down to the end block"}
;
187STATISTIC(NumSpeculations, "Number of speculative executed instructions")static llvm::Statistic NumSpeculations = {"simplifycfg", "NumSpeculations"
, "Number of speculative executed instructions"}
;
188STATISTIC(NumInvokes,static llvm::Statistic NumInvokes = {"simplifycfg", "NumInvokes"
, "Number of invokes with empty resume blocks simplified into calls"
}
189 "Number of invokes with empty resume blocks simplified into calls")static llvm::Statistic NumInvokes = {"simplifycfg", "NumInvokes"
, "Number of invokes with empty resume blocks simplified into calls"
}
;
190
191namespace {
192
193// The first field contains the value that the switch produces when a certain
194// case group is selected, and the second field is a vector containing the
195// cases composing the case group.
196using SwitchCaseResultVectorTy =
197 SmallVector<std::pair<Constant *, SmallVector<ConstantInt *, 4>>, 2>;
198
199// The first field contains the phi node that generates a result of the switch
200// and the second field contains the value generated for a certain case in the
201// switch for that PHI.
202using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
203
204/// ValueEqualityComparisonCase - Represents a case of a switch.
205struct ValueEqualityComparisonCase {
206 ConstantInt *Value;
207 BasicBlock *Dest;
208
209 ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest)
210 : Value(Value), Dest(Dest) {}
211
212 bool operator<(ValueEqualityComparisonCase RHS) const {
213 // Comparing pointers is ok as we only rely on the order for uniquing.
214 return Value < RHS.Value;
215 }
216
217 bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; }
218};
219
220class SimplifyCFGOpt {
221 const TargetTransformInfo &TTI;
222 DomTreeUpdater *DTU;
223 const DataLayout &DL;
224 ArrayRef<WeakVH> LoopHeaders;
225 const SimplifyCFGOptions &Options;
226 bool Resimplify;
227
228 Value *isValueEqualityComparison(Instruction *TI);
229 BasicBlock *GetValueEqualityComparisonCases(
230 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases);
231 bool SimplifyEqualityComparisonWithOnlyPredecessor(Instruction *TI,
232 BasicBlock *Pred,
233 IRBuilder<> &Builder);
234 bool PerformValueComparisonIntoPredecessorFolding(Instruction *TI, Value *&CV,
235 Instruction *PTI,
236 IRBuilder<> &Builder);
237 bool FoldValueComparisonIntoPredecessors(Instruction *TI,
238 IRBuilder<> &Builder);
239
240 bool simplifyReturn(ReturnInst *RI, IRBuilder<> &Builder);
241 bool simplifyResume(ResumeInst *RI, IRBuilder<> &Builder);
242 bool simplifySingleResume(ResumeInst *RI);
243 bool simplifyCommonResume(ResumeInst *RI);
244 bool simplifyCleanupReturn(CleanupReturnInst *RI);
245 bool simplifyUnreachable(UnreachableInst *UI);
246 bool simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder);
247 bool simplifyIndirectBr(IndirectBrInst *IBI);
248 bool simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder);
249 bool simplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder);
250 bool simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder);
251 bool SimplifyCondBranchToTwoReturns(BranchInst *BI, IRBuilder<> &Builder);
252
253 bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
254 IRBuilder<> &Builder);
255
256 bool HoistThenElseCodeToIf(BranchInst *BI, const TargetTransformInfo &TTI,
257 bool EqTermsOnly);
258 bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
259 const TargetTransformInfo &TTI);
260 bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
261 BasicBlock *TrueBB, BasicBlock *FalseBB,
262 uint32_t TrueWeight, uint32_t FalseWeight);
263 bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder,
264 const DataLayout &DL);
265 bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select);
266 bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI);
267 bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder);
268
269public:
270 SimplifyCFGOpt(const TargetTransformInfo &TTI, DomTreeUpdater *DTU,
271 const DataLayout &DL, ArrayRef<WeakVH> LoopHeaders,
272 const SimplifyCFGOptions &Opts)
273 : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) {
274 assert((!DTU || !DTU->hasPostDomTree()) &&(((!DTU || !DTU->hasPostDomTree()) && "SimplifyCFG is not yet capable of maintaining validity of a "
"PostDomTree, so don't ask for it.") ? static_cast<void>
(0) : __assert_fail ("(!DTU || !DTU->hasPostDomTree()) && \"SimplifyCFG is not yet capable of maintaining validity of a \" \"PostDomTree, so don't ask for it.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 276, __PRETTY_FUNCTION__))
275 "SimplifyCFG is not yet capable of maintaining validity of a "(((!DTU || !DTU->hasPostDomTree()) && "SimplifyCFG is not yet capable of maintaining validity of a "
"PostDomTree, so don't ask for it.") ? static_cast<void>
(0) : __assert_fail ("(!DTU || !DTU->hasPostDomTree()) && \"SimplifyCFG is not yet capable of maintaining validity of a \" \"PostDomTree, so don't ask for it.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 276, __PRETTY_FUNCTION__))
276 "PostDomTree, so don't ask for it.")(((!DTU || !DTU->hasPostDomTree()) && "SimplifyCFG is not yet capable of maintaining validity of a "
"PostDomTree, so don't ask for it.") ? static_cast<void>
(0) : __assert_fail ("(!DTU || !DTU->hasPostDomTree()) && \"SimplifyCFG is not yet capable of maintaining validity of a \" \"PostDomTree, so don't ask for it.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 276, __PRETTY_FUNCTION__))
;
277 }
278
279 bool simplifyOnce(BasicBlock *BB);
280 bool simplifyOnceImpl(BasicBlock *BB);
281 bool run(BasicBlock *BB);
282
283 // Helper to set Resimplify and return change indication.
284 bool requestResimplify() {
285 Resimplify = true;
286 return true;
287 }
288};
289
290} // end anonymous namespace
291
292/// Return true if it is safe to merge these two
293/// terminator instructions together.
294static bool
295SafeToMergeTerminators(Instruction *SI1, Instruction *SI2,
296 SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) {
297 if (SI1 == SI2)
298 return false; // Can't merge with self!
299
300 // It is not safe to merge these two switch instructions if they have a common
301 // successor, and if that successor has a PHI node, and if *that* PHI node has
302 // conflicting incoming values from the two switch blocks.
303 BasicBlock *SI1BB = SI1->getParent();
304 BasicBlock *SI2BB = SI2->getParent();
305
306 SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
307 bool Fail = false;
308 for (BasicBlock *Succ : successors(SI2BB))
309 if (SI1Succs.count(Succ))
310 for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) {
311 PHINode *PN = cast<PHINode>(BBI);
312 if (PN->getIncomingValueForBlock(SI1BB) !=
313 PN->getIncomingValueForBlock(SI2BB)) {
314 if (FailBlocks)
315 FailBlocks->insert(Succ);
316 Fail = true;
317 }
318 }
319
320 return !Fail;
321}
322
323/// Update PHI nodes in Succ to indicate that there will now be entries in it
324/// from the 'NewPred' block. The values that will be flowing into the PHI nodes
325/// will be the same as those coming in from ExistPred, an existing predecessor
326/// of Succ.
327static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
328 BasicBlock *ExistPred,
329 MemorySSAUpdater *MSSAU = nullptr) {
330 for (PHINode &PN : Succ->phis())
331 PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred);
332 if (MSSAU)
333 if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ))
334 MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred);
335}
336
337/// Compute an abstract "cost" of speculating the given instruction,
338/// which is assumed to be safe to speculate. TCC_Free means cheap,
339/// TCC_Basic means less cheap, and TCC_Expensive means prohibitively
340/// expensive.
341static InstructionCost computeSpeculationCost(const User *I,
342 const TargetTransformInfo &TTI) {
343 assert(isSafeToSpeculativelyExecute(I) &&((isSafeToSpeculativelyExecute(I) && "Instruction is not safe to speculatively execute!"
) ? static_cast<void> (0) : __assert_fail ("isSafeToSpeculativelyExecute(I) && \"Instruction is not safe to speculatively execute!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 344, __PRETTY_FUNCTION__))
344 "Instruction is not safe to speculatively execute!")((isSafeToSpeculativelyExecute(I) && "Instruction is not safe to speculatively execute!"
) ? static_cast<void> (0) : __assert_fail ("isSafeToSpeculativelyExecute(I) && \"Instruction is not safe to speculatively execute!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 344, __PRETTY_FUNCTION__))
;
345 return TTI.getUserCost(I, TargetTransformInfo::TCK_SizeAndLatency);
346}
347
348/// If we have a merge point of an "if condition" as accepted above,
349/// return true if the specified value dominates the block. We
350/// don't handle the true generality of domination here, just a special case
351/// which works well enough for us.
352///
353/// If AggressiveInsts is non-null, and if V does not dominate BB, we check to
354/// see if V (which must be an instruction) and its recursive operands
355/// that do not dominate BB have a combined cost lower than Budget and
356/// are non-trapping. If both are true, the instruction is inserted into the
357/// set and true is returned.
358///
359/// The cost for most non-trapping instructions is defined as 1 except for
360/// Select whose cost is 2.
361///
362/// After this function returns, Cost is increased by the cost of
363/// V plus its non-dominating operands. If that cost is greater than
364/// Budget, false is returned and Cost is undefined.
365static bool dominatesMergePoint(Value *V, BasicBlock *BB,
366 SmallPtrSetImpl<Instruction *> &AggressiveInsts,
367 InstructionCost &Cost,
368 InstructionCost Budget,
369 const TargetTransformInfo &TTI,
370 unsigned Depth = 0) {
371 // It is possible to hit a zero-cost cycle (phi/gep instructions for example),
372 // so limit the recursion depth.
373 // TODO: While this recursion limit does prevent pathological behavior, it
374 // would be better to track visited instructions to avoid cycles.
375 if (Depth == MaxSpeculationDepth)
376 return false;
377
378 Instruction *I = dyn_cast<Instruction>(V);
379 if (!I) {
380 // Non-instructions all dominate instructions, but not all constantexprs
381 // can be executed unconditionally.
382 if (ConstantExpr *C = dyn_cast<ConstantExpr>(V))
383 if (C->canTrap())
384 return false;
385 return true;
386 }
387 BasicBlock *PBB = I->getParent();
388
389 // We don't want to allow weird loops that might have the "if condition" in
390 // the bottom of this block.
391 if (PBB == BB)
392 return false;
393
394 // If this instruction is defined in a block that contains an unconditional
395 // branch to BB, then it must be in the 'conditional' part of the "if
396 // statement". If not, it definitely dominates the region.
397 BranchInst *BI = dyn_cast<BranchInst>(PBB->getTerminator());
398 if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB)
399 return true;
400
401 // If we have seen this instruction before, don't count it again.
402 if (AggressiveInsts.count(I))
403 return true;
404
405 // Okay, it looks like the instruction IS in the "condition". Check to
406 // see if it's a cheap instruction to unconditionally compute, and if it
407 // only uses stuff defined outside of the condition. If so, hoist it out.
408 if (!isSafeToSpeculativelyExecute(I))
409 return false;
410
411 Cost += computeSpeculationCost(I, TTI);
412
413 // Allow exactly one instruction to be speculated regardless of its cost
414 // (as long as it is safe to do so).
415 // This is intended to flatten the CFG even if the instruction is a division
416 // or other expensive operation. The speculation of an expensive instruction
417 // is expected to be undone in CodeGenPrepare if the speculation has not
418 // enabled further IR optimizations.
419 if (Cost > Budget &&
420 (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0 ||
421 !Cost.isValid()))
422 return false;
423
424 // Okay, we can only really hoist these out if their operands do
425 // not take us over the cost threshold.
426 for (Use &Op : I->operands())
427 if (!dominatesMergePoint(Op, BB, AggressiveInsts, Cost, Budget, TTI,
428 Depth + 1))
429 return false;
430 // Okay, it's safe to do this! Remember this instruction.
431 AggressiveInsts.insert(I);
432 return true;
433}
434
435/// Extract ConstantInt from value, looking through IntToPtr
436/// and PointerNullValue. Return NULL if value is not a constant int.
437static ConstantInt *GetConstantInt(Value *V, const DataLayout &DL) {
438 // Normal constant int.
439 ConstantInt *CI = dyn_cast<ConstantInt>(V);
440 if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy())
441 return CI;
442
443 // This is some kind of pointer constant. Turn it into a pointer-sized
444 // ConstantInt if possible.
445 IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType()));
446
447 // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
448 if (isa<ConstantPointerNull>(V))
449 return ConstantInt::get(PtrTy, 0);
450
451 // IntToPtr const int.
452 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
453 if (CE->getOpcode() == Instruction::IntToPtr)
454 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) {
455 // The constant is very likely to have the right type already.
456 if (CI->getType() == PtrTy)
457 return CI;
458 else
459 return cast<ConstantInt>(
460 ConstantExpr::getIntegerCast(CI, PtrTy, /*isSigned=*/false));
461 }
462 return nullptr;
463}
464
465namespace {
466
467/// Given a chain of or (||) or and (&&) comparison of a value against a
468/// constant, this will try to recover the information required for a switch
469/// structure.
470/// It will depth-first traverse the chain of comparison, seeking for patterns
471/// like %a == 12 or %a < 4 and combine them to produce a set of integer
472/// representing the different cases for the switch.
473/// Note that if the chain is composed of '||' it will build the set of elements
474/// that matches the comparisons (i.e. any of this value validate the chain)
475/// while for a chain of '&&' it will build the set elements that make the test
476/// fail.
477struct ConstantComparesGatherer {
478 const DataLayout &DL;
479
480 /// Value found for the switch comparison
481 Value *CompValue = nullptr;
482
483 /// Extra clause to be checked before the switch
484 Value *Extra = nullptr;
485
486 /// Set of integers to match in switch
487 SmallVector<ConstantInt *, 8> Vals;
488
489 /// Number of comparisons matched in the and/or chain
490 unsigned UsedICmps = 0;
491
492 /// Construct and compute the result for the comparison instruction Cond
493 ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) {
494 gather(Cond);
495 }
496
497 ConstantComparesGatherer(const ConstantComparesGatherer &) = delete;
498 ConstantComparesGatherer &
499 operator=(const ConstantComparesGatherer &) = delete;
500
501private:
502 /// Try to set the current value used for the comparison, it succeeds only if
503 /// it wasn't set before or if the new value is the same as the old one
504 bool setValueOnce(Value *NewVal) {
505 if (CompValue && CompValue != NewVal)
506 return false;
507 CompValue = NewVal;
508 return (CompValue != nullptr);
509 }
510
511 /// Try to match Instruction "I" as a comparison against a constant and
512 /// populates the array Vals with the set of values that match (or do not
513 /// match depending on isEQ).
514 /// Return false on failure. On success, the Value the comparison matched
515 /// against is placed in CompValue.
516 /// If CompValue is already set, the function is expected to fail if a match
517 /// is found but the value compared to is different.
518 bool matchInstruction(Instruction *I, bool isEQ) {
519 // If this is an icmp against a constant, handle this as one of the cases.
520 ICmpInst *ICI;
521 ConstantInt *C;
522 if (!((ICI = dyn_cast<ICmpInst>(I)) &&
523 (C = GetConstantInt(I->getOperand(1), DL)))) {
524 return false;
525 }
526
527 Value *RHSVal;
528 const APInt *RHSC;
529
530 // Pattern match a special case
531 // (x & ~2^z) == y --> x == y || x == y|2^z
532 // This undoes a transformation done by instcombine to fuse 2 compares.
533 if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
534 // It's a little bit hard to see why the following transformations are
535 // correct. Here is a CVC3 program to verify them for 64-bit values:
536
537 /*
538 ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63);
539 x : BITVECTOR(64);
540 y : BITVECTOR(64);
541 z : BITVECTOR(64);
542 mask : BITVECTOR(64) = BVSHL(ONE, z);
543 QUERY( (y & ~mask = y) =>
544 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
545 );
546 QUERY( (y | mask = y) =>
547 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
548 );
549 */
550
551 // Please note that each pattern must be a dual implication (<--> or
552 // iff). One directional implication can create spurious matches. If the
553 // implication is only one-way, an unsatisfiable condition on the left
554 // side can imply a satisfiable condition on the right side. Dual
555 // implication ensures that satisfiable conditions are transformed to
556 // other satisfiable conditions and unsatisfiable conditions are
557 // transformed to other unsatisfiable conditions.
558
559 // Here is a concrete example of a unsatisfiable condition on the left
560 // implying a satisfiable condition on the right:
561 //
562 // mask = (1 << z)
563 // (x & ~mask) == y --> (x == y || x == (y | mask))
564 //
565 // Substituting y = 3, z = 0 yields:
566 // (x & -2) == 3 --> (x == 3 || x == 2)
567
568 // Pattern match a special case:
569 /*
570 QUERY( (y & ~mask = y) =>
571 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
572 );
573 */
574 if (match(ICI->getOperand(0),
575 m_And(m_Value(RHSVal), m_APInt(RHSC)))) {
576 APInt Mask = ~*RHSC;
577 if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) {
578 // If we already have a value for the switch, it has to match!
579 if (!setValueOnce(RHSVal))
580 return false;
581
582 Vals.push_back(C);
583 Vals.push_back(
584 ConstantInt::get(C->getContext(),
585 C->getValue() | Mask));
586 UsedICmps++;
587 return true;
588 }
589 }
590
591 // Pattern match a special case:
592 /*
593 QUERY( (y | mask = y) =>
594 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
595 );
596 */
597 if (match(ICI->getOperand(0),
598 m_Or(m_Value(RHSVal), m_APInt(RHSC)))) {
599 APInt Mask = *RHSC;
600 if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) {
601 // If we already have a value for the switch, it has to match!
602 if (!setValueOnce(RHSVal))
603 return false;
604
605 Vals.push_back(C);
606 Vals.push_back(ConstantInt::get(C->getContext(),
607 C->getValue() & ~Mask));
608 UsedICmps++;
609 return true;
610 }
611 }
612
613 // If we already have a value for the switch, it has to match!
614 if (!setValueOnce(ICI->getOperand(0)))
615 return false;
616
617 UsedICmps++;
618 Vals.push_back(C);
619 return ICI->getOperand(0);
620 }
621
622 // If we have "x ult 3", for example, then we can add 0,1,2 to the set.
623 ConstantRange Span = ConstantRange::makeAllowedICmpRegion(
624 ICI->getPredicate(), C->getValue());
625
626 // Shift the range if the compare is fed by an add. This is the range
627 // compare idiom as emitted by instcombine.
628 Value *CandidateVal = I->getOperand(0);
629 if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) {
630 Span = Span.subtract(*RHSC);
631 CandidateVal = RHSVal;
632 }
633
634 // If this is an and/!= check, then we are looking to build the set of
635 // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into
636 // x != 0 && x != 1.
637 if (!isEQ)
638 Span = Span.inverse();
639
640 // If there are a ton of values, we don't want to make a ginormous switch.
641 if (Span.isSizeLargerThan(8) || Span.isEmptySet()) {
642 return false;
643 }
644
645 // If we already have a value for the switch, it has to match!
646 if (!setValueOnce(CandidateVal))
647 return false;
648
649 // Add all values from the range to the set
650 for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp)
651 Vals.push_back(ConstantInt::get(I->getContext(), Tmp));
652
653 UsedICmps++;
654 return true;
655 }
656
657 /// Given a potentially 'or'd or 'and'd together collection of icmp
658 /// eq/ne/lt/gt instructions that compare a value against a constant, extract
659 /// the value being compared, and stick the list constants into the Vals
660 /// vector.
661 /// One "Extra" case is allowed to differ from the other.
662 void gather(Value *V) {
663 bool isEQ = match(V, m_LogicalOr(m_Value(), m_Value()));
664
665 // Keep a stack (SmallVector for efficiency) for depth-first traversal
666 SmallVector<Value *, 8> DFT;
667 SmallPtrSet<Value *, 8> Visited;
668
669 // Initialize
670 Visited.insert(V);
671 DFT.push_back(V);
672
673 while (!DFT.empty()) {
674 V = DFT.pop_back_val();
675
676 if (Instruction *I = dyn_cast<Instruction>(V)) {
677 // If it is a || (or && depending on isEQ), process the operands.
678 Value *Op0, *Op1;
679 if (isEQ ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1)))
680 : match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
681 if (Visited.insert(Op1).second)
682 DFT.push_back(Op1);
683 if (Visited.insert(Op0).second)
684 DFT.push_back(Op0);
685
686 continue;
687 }
688
689 // Try to match the current instruction
690 if (matchInstruction(I, isEQ))
691 // Match succeed, continue the loop
692 continue;
693 }
694
695 // One element of the sequence of || (or &&) could not be match as a
696 // comparison against the same value as the others.
697 // We allow only one "Extra" case to be checked before the switch
698 if (!Extra) {
699 Extra = V;
700 continue;
701 }
702 // Failed to parse a proper sequence, abort now
703 CompValue = nullptr;
704 break;
705 }
706 }
707};
708
709} // end anonymous namespace
710
711static void EraseTerminatorAndDCECond(Instruction *TI,
712 MemorySSAUpdater *MSSAU = nullptr) {
713 Instruction *Cond = nullptr;
714 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
715 Cond = dyn_cast<Instruction>(SI->getCondition());
716 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
717 if (BI->isConditional())
718 Cond = dyn_cast<Instruction>(BI->getCondition());
719 } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) {
720 Cond = dyn_cast<Instruction>(IBI->getAddress());
721 }
722
723 TI->eraseFromParent();
724 if (Cond)
725 RecursivelyDeleteTriviallyDeadInstructions(Cond, nullptr, MSSAU);
726}
727
728/// Return true if the specified terminator checks
729/// to see if a value is equal to constant integer value.
730Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) {
731 Value *CV = nullptr;
732 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
733 // Do not permit merging of large switch instructions into their
734 // predecessors unless there is only one predecessor.
735 if (!SI->getParent()->hasNPredecessorsOrMore(128 / SI->getNumSuccessors()))
736 CV = SI->getCondition();
737 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI))
738 if (BI->isConditional() && BI->getCondition()->hasOneUse())
739 if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) {
740 if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL))
741 CV = ICI->getOperand(0);
742 }
743
744 // Unwrap any lossless ptrtoint cast.
745 if (CV) {
746 if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) {
747 Value *Ptr = PTII->getPointerOperand();
748 if (PTII->getType() == DL.getIntPtrType(Ptr->getType()))
749 CV = Ptr;
750 }
751 }
752 return CV;
753}
754
755/// Given a value comparison instruction,
756/// decode all of the 'cases' that it represents and return the 'default' block.
757BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases(
758 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases) {
759 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
760 Cases.reserve(SI->getNumCases());
761 for (auto Case : SI->cases())
762 Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(),
763 Case.getCaseSuccessor()));
764 return SI->getDefaultDest();
765 }
766
767 BranchInst *BI = cast<BranchInst>(TI);
768 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
769 BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE);
770 Cases.push_back(ValueEqualityComparisonCase(
771 GetConstantInt(ICI->getOperand(1), DL), Succ));
772 return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ);
773}
774
775/// Given a vector of bb/value pairs, remove any entries
776/// in the list that match the specified block.
777static void
778EliminateBlockCases(BasicBlock *BB,
779 std::vector<ValueEqualityComparisonCase> &Cases) {
780 llvm::erase_value(Cases, BB);
781}
782
783/// Return true if there are any keys in C1 that exist in C2 as well.
784static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
785 std::vector<ValueEqualityComparisonCase> &C2) {
786 std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2;
787
788 // Make V1 be smaller than V2.
789 if (V1->size() > V2->size())
790 std::swap(V1, V2);
791
792 if (V1->empty())
793 return false;
794 if (V1->size() == 1) {
795 // Just scan V2.
796 ConstantInt *TheVal = (*V1)[0].Value;
797 for (unsigned i = 0, e = V2->size(); i != e; ++i)
798 if (TheVal == (*V2)[i].Value)
799 return true;
800 }
801
802 // Otherwise, just sort both lists and compare element by element.
803 array_pod_sort(V1->begin(), V1->end());
804 array_pod_sort(V2->begin(), V2->end());
805 unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size();
806 while (i1 != e1 && i2 != e2) {
807 if ((*V1)[i1].Value == (*V2)[i2].Value)
808 return true;
809 if ((*V1)[i1].Value < (*V2)[i2].Value)
810 ++i1;
811 else
812 ++i2;
813 }
814 return false;
815}
816
817// Set branch weights on SwitchInst. This sets the metadata if there is at
818// least one non-zero weight.
819static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights) {
820 // Check that there is at least one non-zero weight. Otherwise, pass
821 // nullptr to setMetadata which will erase the existing metadata.
822 MDNode *N = nullptr;
823 if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; }))
824 N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights);
825 SI->setMetadata(LLVMContext::MD_prof, N);
826}
827
828// Similar to the above, but for branch and select instructions that take
829// exactly 2 weights.
830static void setBranchWeights(Instruction *I, uint32_t TrueWeight,
831 uint32_t FalseWeight) {
832 assert(isa<BranchInst>(I) || isa<SelectInst>(I))((isa<BranchInst>(I) || isa<SelectInst>(I)) ? static_cast
<void> (0) : __assert_fail ("isa<BranchInst>(I) || isa<SelectInst>(I)"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 832, __PRETTY_FUNCTION__))
;
833 // Check that there is at least one non-zero weight. Otherwise, pass
834 // nullptr to setMetadata which will erase the existing metadata.
835 MDNode *N = nullptr;
836 if (TrueWeight || FalseWeight)
837 N = MDBuilder(I->getParent()->getContext())
838 .createBranchWeights(TrueWeight, FalseWeight);
839 I->setMetadata(LLVMContext::MD_prof, N);
840}
841
842/// If TI is known to be a terminator instruction and its block is known to
843/// only have a single predecessor block, check to see if that predecessor is
844/// also a value comparison with the same value, and if that comparison
845/// determines the outcome of this comparison. If so, simplify TI. This does a
846/// very limited form of jump threading.
847bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
848 Instruction *TI, BasicBlock *Pred, IRBuilder<> &Builder) {
849 Value *PredVal = isValueEqualityComparison(Pred->getTerminator());
850 if (!PredVal)
851 return false; // Not a value comparison in predecessor.
852
853 Value *ThisVal = isValueEqualityComparison(TI);
854 assert(ThisVal && "This isn't a value comparison!!")((ThisVal && "This isn't a value comparison!!") ? static_cast
<void> (0) : __assert_fail ("ThisVal && \"This isn't a value comparison!!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 854, __PRETTY_FUNCTION__))
;
855 if (ThisVal != PredVal)
856 return false; // Different predicates.
857
858 // TODO: Preserve branch weight metadata, similarly to how
859 // FoldValueComparisonIntoPredecessors preserves it.
860
861 // Find out information about when control will move from Pred to TI's block.
862 std::vector<ValueEqualityComparisonCase> PredCases;
863 BasicBlock *PredDef =
864 GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases);
865 EliminateBlockCases(PredDef, PredCases); // Remove default from cases.
866
867 // Find information about how control leaves this block.
868 std::vector<ValueEqualityComparisonCase> ThisCases;
869 BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases);
870 EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases.
871
872 // If TI's block is the default block from Pred's comparison, potentially
873 // simplify TI based on this knowledge.
874 if (PredDef == TI->getParent()) {
875 // If we are here, we know that the value is none of those cases listed in
876 // PredCases. If there are any cases in ThisCases that are in PredCases, we
877 // can simplify TI.
878 if (!ValuesOverlap(PredCases, ThisCases))
879 return false;
880
881 if (isa<BranchInst>(TI)) {
882 // Okay, one of the successors of this condbr is dead. Convert it to a
883 // uncond br.
884 assert(ThisCases.size() == 1 && "Branch can only have one case!")((ThisCases.size() == 1 && "Branch can only have one case!"
) ? static_cast<void> (0) : __assert_fail ("ThisCases.size() == 1 && \"Branch can only have one case!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 884, __PRETTY_FUNCTION__))
;
885 // Insert the new branch.
886 Instruction *NI = Builder.CreateBr(ThisDef);
887 (void)NI;
888
889 // Remove PHI node entries for the dead edge.
890 ThisCases[0].Dest->removePredecessor(PredDef);
891
892 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
893 << "Through successor TI: " << *TI << "Leaving: " << *NIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
894 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
;
895
896 EraseTerminatorAndDCECond(TI);
897
898 if (DTU)
899 DTU->applyUpdates(
900 {{DominatorTree::Delete, PredDef, ThisCases[0].Dest}});
901
902 return true;
903 }
904
905 SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI);
906 // Okay, TI has cases that are statically dead, prune them away.
907 SmallPtrSet<Constant *, 16> DeadCases;
908 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
909 DeadCases.insert(PredCases[i].Value);
910
911 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI; } } while (false)
912 << "Through successor TI: " << *TI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI; } } while (false)
;
913
914 SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
915 for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
916 --i;
917 auto *Successor = i->getCaseSuccessor();
918 if (DTU)
919 ++NumPerSuccessorCases[Successor];
920 if (DeadCases.count(i->getCaseValue())) {
921 Successor->removePredecessor(PredDef);
922 SI.removeCase(i);
923 if (DTU)
924 --NumPerSuccessorCases[Successor];
925 }
926 }
927
928 if (DTU) {
929 std::vector<DominatorTree::UpdateType> Updates;
930 for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
931 if (I.second == 0)
932 Updates.push_back({DominatorTree::Delete, PredDef, I.first});
933 DTU->applyUpdates(Updates);
934 }
935
936 LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Leaving: " << *TI <<
"\n"; } } while (false)
;
937 return true;
938 }
939
940 // Otherwise, TI's block must correspond to some matched value. Find out
941 // which value (or set of values) this is.
942 ConstantInt *TIV = nullptr;
943 BasicBlock *TIBB = TI->getParent();
944 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
945 if (PredCases[i].Dest == TIBB) {
946 if (TIV)
947 return false; // Cannot handle multiple values coming to this block.
948 TIV = PredCases[i].Value;
949 }
950 assert(TIV && "No edge from pred to succ?")((TIV && "No edge from pred to succ?") ? static_cast<
void> (0) : __assert_fail ("TIV && \"No edge from pred to succ?\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 950, __PRETTY_FUNCTION__))
;
951
952 // Okay, we found the one constant that our value can be if we get into TI's
953 // BB. Find out which successor will unconditionally be branched to.
954 BasicBlock *TheRealDest = nullptr;
955 for (unsigned i = 0, e = ThisCases.size(); i != e; ++i)
956 if (ThisCases[i].Value == TIV) {
957 TheRealDest = ThisCases[i].Dest;
958 break;
959 }
960
961 // If not handled by any explicit cases, it is handled by the default case.
962 if (!TheRealDest)
963 TheRealDest = ThisDef;
964
965 SmallPtrSet<BasicBlock *, 2> RemovedSuccs;
966
967 // Remove PHI node entries for dead edges.
968 BasicBlock *CheckEdge = TheRealDest;
969 for (BasicBlock *Succ : successors(TIBB))
970 if (Succ != CheckEdge) {
971 if (Succ != TheRealDest)
972 RemovedSuccs.insert(Succ);
973 Succ->removePredecessor(TIBB);
974 } else
975 CheckEdge = nullptr;
976
977 // Insert the new branch.
978 Instruction *NI = Builder.CreateBr(TheRealDest);
979 (void)NI;
980
981 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
982 << "Through successor TI: " << *TI << "Leaving: " << *NIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
983 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Threading pred instr: " <<
*Pred->getTerminator() << "Through successor TI: " <<
*TI << "Leaving: " << *NI << "\n"; } } while
(false)
;
984
985 EraseTerminatorAndDCECond(TI);
986 if (DTU) {
987 SmallVector<DominatorTree::UpdateType, 2> Updates;
988 Updates.reserve(RemovedSuccs.size());
989 for (auto *RemovedSucc : RemovedSuccs)
990 Updates.push_back({DominatorTree::Delete, TIBB, RemovedSucc});
991 DTU->applyUpdates(Updates);
992 }
993 return true;
994}
995
996namespace {
997
998/// This class implements a stable ordering of constant
999/// integers that does not depend on their address. This is important for
1000/// applications that sort ConstantInt's to ensure uniqueness.
1001struct ConstantIntOrdering {
1002 bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const {
1003 return LHS->getValue().ult(RHS->getValue());
1004 }
1005};
1006
1007} // end anonymous namespace
1008
1009static int ConstantIntSortPredicate(ConstantInt *const *P1,
1010 ConstantInt *const *P2) {
1011 const ConstantInt *LHS = *P1;
1012 const ConstantInt *RHS = *P2;
1013 if (LHS == RHS)
1014 return 0;
1015 return LHS->getValue().ult(RHS->getValue()) ? 1 : -1;
1016}
1017
1018static inline bool HasBranchWeights(const Instruction *I) {
1019 MDNode *ProfMD = I->getMetadata(LLVMContext::MD_prof);
1020 if (ProfMD && ProfMD->getOperand(0))
1021 if (MDString *MDS = dyn_cast<MDString>(ProfMD->getOperand(0)))
1022 return MDS->getString().equals("branch_weights");
1023
1024 return false;
1025}
1026
1027/// Get Weights of a given terminator, the default weight is at the front
1028/// of the vector. If TI is a conditional eq, we need to swap the branch-weight
1029/// metadata.
1030static void GetBranchWeights(Instruction *TI,
1031 SmallVectorImpl<uint64_t> &Weights) {
1032 MDNode *MD = TI->getMetadata(LLVMContext::MD_prof);
1033 assert(MD)((MD) ? static_cast<void> (0) : __assert_fail ("MD", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1033, __PRETTY_FUNCTION__))
;
1034 for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) {
1035 ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(i));
1036 Weights.push_back(CI->getValue().getZExtValue());
1037 }
1038
1039 // If TI is a conditional eq, the default case is the false case,
1040 // and the corresponding branch-weight data is at index 2. We swap the
1041 // default weight to be the first entry.
1042 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
1043 assert(Weights.size() == 2)((Weights.size() == 2) ? static_cast<void> (0) : __assert_fail
("Weights.size() == 2", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1043, __PRETTY_FUNCTION__))
;
1044 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
1045 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
1046 std::swap(Weights.front(), Weights.back());
1047 }
1048}
1049
1050/// Keep halving the weights until all can fit in uint32_t.
1051static void FitWeights(MutableArrayRef<uint64_t> Weights) {
1052 uint64_t Max = *std::max_element(Weights.begin(), Weights.end());
1053 if (Max > UINT_MAX(2147483647 *2U +1U)) {
1054 unsigned Offset = 32 - countLeadingZeros(Max);
1055 for (uint64_t &I : Weights)
1056 I >>= Offset;
1057 }
1058}
1059
1060static void CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(
1061 BasicBlock *BB, BasicBlock *PredBlock, ValueToValueMapTy &VMap) {
1062 Instruction *PTI = PredBlock->getTerminator();
1063
1064 // If we have bonus instructions, clone them into the predecessor block.
1065 // Note that there may be multiple predecessor blocks, so we cannot move
1066 // bonus instructions to a predecessor block.
1067 for (Instruction &BonusInst : *BB) {
1068 if (isa<DbgInfoIntrinsic>(BonusInst) || BonusInst.isTerminator())
1069 continue;
1070
1071 Instruction *NewBonusInst = BonusInst.clone();
1072
1073 if (PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) {
1074 // Unless the instruction has the same !dbg location as the original
1075 // branch, drop it. When we fold the bonus instructions we want to make
1076 // sure we reset their debug locations in order to avoid stepping on
1077 // dead code caused by folding dead branches.
1078 NewBonusInst->setDebugLoc(DebugLoc());
1079 }
1080
1081 RemapInstruction(NewBonusInst, VMap,
1082 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
1083 VMap[&BonusInst] = NewBonusInst;
1084
1085 // If we moved a load, we cannot any longer claim any knowledge about
1086 // its potential value. The previous information might have been valid
1087 // only given the branch precondition.
1088 // For an analogous reason, we must also drop all the metadata whose
1089 // semantics we don't understand. We *can* preserve !annotation, because
1090 // it is tied to the instruction itself, not the value or position.
1091 NewBonusInst->dropUnknownNonDebugMetadata(LLVMContext::MD_annotation);
1092
1093 PredBlock->getInstList().insert(PTI->getIterator(), NewBonusInst);
1094 NewBonusInst->takeName(&BonusInst);
1095 BonusInst.setName(NewBonusInst->getName() + ".old");
1096
1097 // Update (liveout) uses of bonus instructions,
1098 // now that the bonus instruction has been cloned into predecessor.
1099 SSAUpdater SSAUpdate;
1100 SSAUpdate.Initialize(BonusInst.getType(),
1101 (NewBonusInst->getName() + ".merge").str());
1102 SSAUpdate.AddAvailableValue(BB, &BonusInst);
1103 SSAUpdate.AddAvailableValue(PredBlock, NewBonusInst);
1104 for (Use &U : make_early_inc_range(BonusInst.uses())) {
1105 auto *UI = cast<Instruction>(U.getUser());
1106 if (UI->getParent() != PredBlock)
1107 SSAUpdate.RewriteUseAfterInsertions(U);
1108 else // Use is in the same block as, and comes before, NewBonusInst.
1109 SSAUpdate.RewriteUse(U);
1110 }
1111 }
1112}
1113
1114bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding(
1115 Instruction *TI, Value *&CV, Instruction *PTI, IRBuilder<> &Builder) {
1116 BasicBlock *BB = TI->getParent();
1117 BasicBlock *Pred = PTI->getParent();
1118
1119 SmallVector<DominatorTree::UpdateType, 32> Updates;
1120
1121 // Figure out which 'cases' to copy from SI to PSI.
1122 std::vector<ValueEqualityComparisonCase> BBCases;
1123 BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases);
1124
1125 std::vector<ValueEqualityComparisonCase> PredCases;
1126 BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases);
1127
1128 // Based on whether the default edge from PTI goes to BB or not, fill in
1129 // PredCases and PredDefault with the new switch cases we would like to
1130 // build.
1131 SmallMapVector<BasicBlock *, int, 8> NewSuccessors;
1132
1133 // Update the branch weight metadata along the way
1134 SmallVector<uint64_t, 8> Weights;
1135 bool PredHasWeights = HasBranchWeights(PTI);
1136 bool SuccHasWeights = HasBranchWeights(TI);
1137
1138 if (PredHasWeights) {
1139 GetBranchWeights(PTI, Weights);
1140 // branch-weight metadata is inconsistent here.
1141 if (Weights.size() != 1 + PredCases.size())
1142 PredHasWeights = SuccHasWeights = false;
1143 } else if (SuccHasWeights)
1144 // If there are no predecessor weights but there are successor weights,
1145 // populate Weights with 1, which will later be scaled to the sum of
1146 // successor's weights
1147 Weights.assign(1 + PredCases.size(), 1);
1148
1149 SmallVector<uint64_t, 8> SuccWeights;
1150 if (SuccHasWeights) {
1151 GetBranchWeights(TI, SuccWeights);
1152 // branch-weight metadata is inconsistent here.
1153 if (SuccWeights.size() != 1 + BBCases.size())
1154 PredHasWeights = SuccHasWeights = false;
1155 } else if (PredHasWeights)
1156 SuccWeights.assign(1 + BBCases.size(), 1);
1157
1158 if (PredDefault == BB) {
1159 // If this is the default destination from PTI, only the edges in TI
1160 // that don't occur in PTI, or that branch to BB will be activated.
1161 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1162 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1163 if (PredCases[i].Dest != BB)
1164 PTIHandled.insert(PredCases[i].Value);
1165 else {
1166 // The default destination is BB, we don't need explicit targets.
1167 std::swap(PredCases[i], PredCases.back());
1168
1169 if (PredHasWeights || SuccHasWeights) {
1170 // Increase weight for the default case.
1171 Weights[0] += Weights[i + 1];
1172 std::swap(Weights[i + 1], Weights.back());
1173 Weights.pop_back();
1174 }
1175
1176 PredCases.pop_back();
1177 --i;
1178 --e;
1179 }
1180
1181 // Reconstruct the new switch statement we will be building.
1182 if (PredDefault != BBDefault) {
1183 PredDefault->removePredecessor(Pred);
1184 if (DTU && PredDefault != BB)
1185 Updates.push_back({DominatorTree::Delete, Pred, PredDefault});
1186 PredDefault = BBDefault;
1187 ++NewSuccessors[BBDefault];
1188 }
1189
1190 unsigned CasesFromPred = Weights.size();
1191 uint64_t ValidTotalSuccWeight = 0;
1192 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1193 if (!PTIHandled.count(BBCases[i].Value) && BBCases[i].Dest != BBDefault) {
1194 PredCases.push_back(BBCases[i]);
1195 ++NewSuccessors[BBCases[i].Dest];
1196 if (SuccHasWeights || PredHasWeights) {
1197 // The default weight is at index 0, so weight for the ith case
1198 // should be at index i+1. Scale the cases from successor by
1199 // PredDefaultWeight (Weights[0]).
1200 Weights.push_back(Weights[0] * SuccWeights[i + 1]);
1201 ValidTotalSuccWeight += SuccWeights[i + 1];
1202 }
1203 }
1204
1205 if (SuccHasWeights || PredHasWeights) {
1206 ValidTotalSuccWeight += SuccWeights[0];
1207 // Scale the cases from predecessor by ValidTotalSuccWeight.
1208 for (unsigned i = 1; i < CasesFromPred; ++i)
1209 Weights[i] *= ValidTotalSuccWeight;
1210 // Scale the default weight by SuccDefaultWeight (SuccWeights[0]).
1211 Weights[0] *= SuccWeights[0];
1212 }
1213 } else {
1214 // If this is not the default destination from PSI, only the edges
1215 // in SI that occur in PSI with a destination of BB will be
1216 // activated.
1217 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1218 std::map<ConstantInt *, uint64_t> WeightsForHandled;
1219 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1220 if (PredCases[i].Dest == BB) {
1221 PTIHandled.insert(PredCases[i].Value);
1222
1223 if (PredHasWeights || SuccHasWeights) {
1224 WeightsForHandled[PredCases[i].Value] = Weights[i + 1];
1225 std::swap(Weights[i + 1], Weights.back());
1226 Weights.pop_back();
1227 }
1228
1229 std::swap(PredCases[i], PredCases.back());
1230 PredCases.pop_back();
1231 --i;
1232 --e;
1233 }
1234
1235 // Okay, now we know which constants were sent to BB from the
1236 // predecessor. Figure out where they will all go now.
1237 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1238 if (PTIHandled.count(BBCases[i].Value)) {
1239 // If this is one we are capable of getting...
1240 if (PredHasWeights || SuccHasWeights)
1241 Weights.push_back(WeightsForHandled[BBCases[i].Value]);
1242 PredCases.push_back(BBCases[i]);
1243 ++NewSuccessors[BBCases[i].Dest];
1244 PTIHandled.erase(BBCases[i].Value); // This constant is taken care of
1245 }
1246
1247 // If there are any constants vectored to BB that TI doesn't handle,
1248 // they must go to the default destination of TI.
1249 for (ConstantInt *I : PTIHandled) {
1250 if (PredHasWeights || SuccHasWeights)
1251 Weights.push_back(WeightsForHandled[I]);
1252 PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault));
1253 ++NewSuccessors[BBDefault];
1254 }
1255 }
1256
1257 // Okay, at this point, we know which new successor Pred will get. Make
1258 // sure we update the number of entries in the PHI nodes for these
1259 // successors.
1260 SmallPtrSet<BasicBlock *, 2> SuccsOfPred;
1261 if (DTU) {
1262 SuccsOfPred = {succ_begin(Pred), succ_end(Pred)};
1263 Updates.reserve(Updates.size() + NewSuccessors.size());
1264 }
1265 for (const std::pair<BasicBlock *, int /*Num*/> &NewSuccessor :
1266 NewSuccessors) {
1267 for (auto I : seq(0, NewSuccessor.second)) {
1268 (void)I;
1269 AddPredecessorToBlock(NewSuccessor.first, Pred, BB);
1270 }
1271 if (DTU && !SuccsOfPred.contains(NewSuccessor.first))
1272 Updates.push_back({DominatorTree::Insert, Pred, NewSuccessor.first});
1273 }
1274
1275 Builder.SetInsertPoint(PTI);
1276 // Convert pointer to int before we switch.
1277 if (CV->getType()->isPointerTy()) {
1278 CV =
1279 Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()), "magicptr");
1280 }
1281
1282 // Now that the successors are updated, create the new Switch instruction.
1283 SwitchInst *NewSI = Builder.CreateSwitch(CV, PredDefault, PredCases.size());
1284 NewSI->setDebugLoc(PTI->getDebugLoc());
1285 for (ValueEqualityComparisonCase &V : PredCases)
1286 NewSI->addCase(V.Value, V.Dest);
1287
1288 if (PredHasWeights || SuccHasWeights) {
1289 // Halve the weights if any of them cannot fit in an uint32_t
1290 FitWeights(Weights);
1291
1292 SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
1293
1294 setBranchWeights(NewSI, MDWeights);
1295 }
1296
1297 EraseTerminatorAndDCECond(PTI);
1298
1299 // Okay, last check. If BB is still a successor of PSI, then we must
1300 // have an infinite loop case. If so, add an infinitely looping block
1301 // to handle the case to preserve the behavior of the code.
1302 BasicBlock *InfLoopBlock = nullptr;
1303 for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i)
1304 if (NewSI->getSuccessor(i) == BB) {
1305 if (!InfLoopBlock) {
1306 // Insert it at the end of the function, because it's either code,
1307 // or it won't matter if it's hot. :)
1308 InfLoopBlock =
1309 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
1310 BranchInst::Create(InfLoopBlock, InfLoopBlock);
1311 if (DTU)
1312 Updates.push_back(
1313 {DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
1314 }
1315 NewSI->setSuccessor(i, InfLoopBlock);
1316 }
1317
1318 if (DTU) {
1319 if (InfLoopBlock)
1320 Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock});
1321
1322 Updates.push_back({DominatorTree::Delete, Pred, BB});
1323
1324 DTU->applyUpdates(Updates);
1325 }
1326
1327 ++NumFoldValueComparisonIntoPredecessors;
1328 return true;
1329}
1330
1331/// The specified terminator is a value equality comparison instruction
1332/// (either a switch or a branch on "X == c").
1333/// See if any of the predecessors of the terminator block are value comparisons
1334/// on the same value. If so, and if safe to do so, fold them together.
1335bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(Instruction *TI,
1336 IRBuilder<> &Builder) {
1337 BasicBlock *BB = TI->getParent();
1338 Value *CV = isValueEqualityComparison(TI); // CondVal
1339 assert(CV && "Not a comparison?")((CV && "Not a comparison?") ? static_cast<void>
(0) : __assert_fail ("CV && \"Not a comparison?\"", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1339, __PRETTY_FUNCTION__))
;
1340
1341 bool Changed = false;
1342
1343 SmallSetVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB));
1344 while (!Preds.empty()) {
1345 BasicBlock *Pred = Preds.pop_back_val();
1346 Instruction *PTI = Pred->getTerminator();
1347
1348 // Don't try to fold into itself.
1349 if (Pred == BB)
1350 continue;
1351
1352 // See if the predecessor is a comparison with the same value.
1353 Value *PCV = isValueEqualityComparison(PTI); // PredCondVal
1354 if (PCV != CV)
1355 continue;
1356
1357 SmallSetVector<BasicBlock *, 4> FailBlocks;
1358 if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) {
1359 for (auto *Succ : FailBlocks) {
1360 if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split", DTU))
1361 return false;
1362 }
1363 }
1364
1365 PerformValueComparisonIntoPredecessorFolding(TI, CV, PTI, Builder);
1366 Changed = true;
1367 }
1368 return Changed;
1369}
1370
1371// If we would need to insert a select that uses the value of this invoke
1372// (comments in HoistThenElseCodeToIf explain why we would need to do this), we
1373// can't hoist the invoke, as there is nowhere to put the select in this case.
1374static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2,
1375 Instruction *I1, Instruction *I2) {
1376 for (BasicBlock *Succ : successors(BB1)) {
1377 for (const PHINode &PN : Succ->phis()) {
1378 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1379 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1380 if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) {
1381 return false;
1382 }
1383 }
1384 }
1385 return true;
1386}
1387
1388static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified = false);
1389
1390/// Given a conditional branch that goes to BB1 and BB2, hoist any common code
1391/// in the two blocks up into the branch block. The caller of this function
1392/// guarantees that BI's block dominates BB1 and BB2. If EqTermsOnly is given,
1393/// only perform hoisting in case both blocks only contain a terminator. In that
1394/// case, only the original BI will be replaced and selects for PHIs are added.
1395bool SimplifyCFGOpt::HoistThenElseCodeToIf(BranchInst *BI,
1396 const TargetTransformInfo &TTI,
1397 bool EqTermsOnly) {
1398 // This does very trivial matching, with limited scanning, to find identical
1399 // instructions in the two blocks. In particular, we don't want to get into
1400 // O(M*N) situations here where M and N are the sizes of BB1 and BB2. As
1401 // such, we currently just scan for obviously identical instructions in an
1402 // identical order.
1403 BasicBlock *BB1 = BI->getSuccessor(0); // The true destination.
1404 BasicBlock *BB2 = BI->getSuccessor(1); // The false destination
1405
1406 BasicBlock::iterator BB1_Itr = BB1->begin();
1407 BasicBlock::iterator BB2_Itr = BB2->begin();
1408
1409 Instruction *I1 = &*BB1_Itr++, *I2 = &*BB2_Itr++;
1410 // Skip debug info if it is not identical.
1411 DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1);
1412 DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2);
1413 if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1414 while (isa<DbgInfoIntrinsic>(I1))
1415 I1 = &*BB1_Itr++;
1416 while (isa<DbgInfoIntrinsic>(I2))
1417 I2 = &*BB2_Itr++;
1418 }
1419 // FIXME: Can we define a safety predicate for CallBr?
1420 if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) ||
1421 (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) ||
1422 isa<CallBrInst>(I1))
1423 return false;
1424
1425 BasicBlock *BIParent = BI->getParent();
1426
1427 bool Changed = false;
1428
1429 auto _ = make_scope_exit([&]() {
1430 if (Changed)
1431 ++NumHoistCommonCode;
1432 });
1433
1434 // Check if only hoisting terminators is allowed. This does not add new
1435 // instructions to the hoist location.
1436 if (EqTermsOnly) {
1437 if (!I1->isIdenticalToWhenDefined(I2))
1438 return false;
1439 if (!I1->isTerminator())
1440 return false;
1441 goto HoistTerminator;
1442 }
1443
1444 do {
1445 // If we are hoisting the terminator instruction, don't move one (making a
1446 // broken BB), instead clone it, and remove BI.
1447 if (I1->isTerminator())
1448 goto HoistTerminator;
1449
1450 // If we're going to hoist a call, make sure that the two instructions we're
1451 // commoning/hoisting are both marked with musttail, or neither of them is
1452 // marked as such. Otherwise, we might end up in a situation where we hoist
1453 // from a block where the terminator is a `ret` to a block where the terminator
1454 // is a `br`, and `musttail` calls expect to be followed by a return.
1455 auto *C1 = dyn_cast<CallInst>(I1);
1456 auto *C2 = dyn_cast<CallInst>(I2);
1457 if (C1 && C2)
1458 if (C1->isMustTailCall() != C2->isMustTailCall())
1459 return Changed;
1460
1461 if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2))
1462 return Changed;
1463
1464 // If any of the two call sites has nomerge attribute, stop hoisting.
1465 if (const auto *CB1 = dyn_cast<CallBase>(I1))
1466 if (CB1->cannotMerge())
1467 return Changed;
1468 if (const auto *CB2 = dyn_cast<CallBase>(I2))
1469 if (CB2->cannotMerge())
1470 return Changed;
1471
1472 if (isa<DbgInfoIntrinsic>(I1) || isa<DbgInfoIntrinsic>(I2)) {
1473 assert (isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2))((isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic
>(I2)) ? static_cast<void> (0) : __assert_fail ("isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2)"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1473, __PRETTY_FUNCTION__))
;
1474 // The debug location is an integral part of a debug info intrinsic
1475 // and can't be separated from it or replaced. Instead of attempting
1476 // to merge locations, simply hoist both copies of the intrinsic.
1477 BIParent->getInstList().splice(BI->getIterator(),
1478 BB1->getInstList(), I1);
1479 BIParent->getInstList().splice(BI->getIterator(),
1480 BB2->getInstList(), I2);
1481 Changed = true;
1482 } else {
1483 // For a normal instruction, we just move one to right before the branch,
1484 // then replace all uses of the other with the first. Finally, we remove
1485 // the now redundant second instruction.
1486 BIParent->getInstList().splice(BI->getIterator(),
1487 BB1->getInstList(), I1);
1488 if (!I2->use_empty())
1489 I2->replaceAllUsesWith(I1);
1490 I1->andIRFlags(I2);
1491 unsigned KnownIDs[] = {LLVMContext::MD_tbaa,
1492 LLVMContext::MD_range,
1493 LLVMContext::MD_fpmath,
1494 LLVMContext::MD_invariant_load,
1495 LLVMContext::MD_nonnull,
1496 LLVMContext::MD_invariant_group,
1497 LLVMContext::MD_align,
1498 LLVMContext::MD_dereferenceable,
1499 LLVMContext::MD_dereferenceable_or_null,
1500 LLVMContext::MD_mem_parallel_loop_access,
1501 LLVMContext::MD_access_group,
1502 LLVMContext::MD_preserve_access_index};
1503 combineMetadata(I1, I2, KnownIDs, true);
1504
1505 // I1 and I2 are being combined into a single instruction. Its debug
1506 // location is the merged locations of the original instructions.
1507 I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1508
1509 I2->eraseFromParent();
1510 Changed = true;
1511 }
1512 ++NumHoistCommonInstrs;
1513
1514 I1 = &*BB1_Itr++;
1515 I2 = &*BB2_Itr++;
1516 // Skip debug info if it is not identical.
1517 DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1);
1518 DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2);
1519 if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1520 while (isa<DbgInfoIntrinsic>(I1))
1521 I1 = &*BB1_Itr++;
1522 while (isa<DbgInfoIntrinsic>(I2))
1523 I2 = &*BB2_Itr++;
1524 }
1525 } while (I1->isIdenticalToWhenDefined(I2));
1526
1527 return true;
1528
1529HoistTerminator:
1530 // It may not be possible to hoist an invoke.
1531 // FIXME: Can we define a safety predicate for CallBr?
1532 if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))
1533 return Changed;
1534
1535 // TODO: callbr hoisting currently disabled pending further study.
1536 if (isa<CallBrInst>(I1))
1537 return Changed;
1538
1539 for (BasicBlock *Succ : successors(BB1)) {
1540 for (PHINode &PN : Succ->phis()) {
1541 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1542 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1543 if (BB1V == BB2V)
1544 continue;
1545
1546 // Check for passingValueIsAlwaysUndefined here because we would rather
1547 // eliminate undefined control flow then converting it to a select.
1548 if (passingValueIsAlwaysUndefined(BB1V, &PN) ||
1549 passingValueIsAlwaysUndefined(BB2V, &PN))
1550 return Changed;
1551
1552 if (isa<ConstantExpr>(BB1V) && !isSafeToSpeculativelyExecute(BB1V))
1553 return Changed;
1554 if (isa<ConstantExpr>(BB2V) && !isSafeToSpeculativelyExecute(BB2V))
1555 return Changed;
1556 }
1557 }
1558
1559 // Okay, it is safe to hoist the terminator.
1560 Instruction *NT = I1->clone();
1561 BIParent->getInstList().insert(BI->getIterator(), NT);
1562 if (!NT->getType()->isVoidTy()) {
1563 I1->replaceAllUsesWith(NT);
1564 I2->replaceAllUsesWith(NT);
1565 NT->takeName(I1);
1566 }
1567 Changed = true;
1568 ++NumHoistCommonInstrs;
1569
1570 // Ensure terminator gets a debug location, even an unknown one, in case
1571 // it involves inlinable calls.
1572 NT->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1573
1574 // PHIs created below will adopt NT's merged DebugLoc.
1575 IRBuilder<NoFolder> Builder(NT);
1576
1577 // Hoisting one of the terminators from our successor is a great thing.
1578 // Unfortunately, the successors of the if/else blocks may have PHI nodes in
1579 // them. If they do, all PHI entries for BB1/BB2 must agree for all PHI
1580 // nodes, so we insert select instruction to compute the final result.
1581 std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects;
1582 for (BasicBlock *Succ : successors(BB1)) {
1583 for (PHINode &PN : Succ->phis()) {
1584 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1585 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1586 if (BB1V == BB2V)
1587 continue;
1588
1589 // These values do not agree. Insert a select instruction before NT
1590 // that determines the right value.
1591 SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)];
1592 if (!SI) {
1593 // Propagate fast-math-flags from phi node to its replacement select.
1594 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
1595 if (isa<FPMathOperator>(PN))
1596 Builder.setFastMathFlags(PN.getFastMathFlags());
1597
1598 SI = cast<SelectInst>(
1599 Builder.CreateSelect(BI->getCondition(), BB1V, BB2V,
1600 BB1V->getName() + "." + BB2V->getName(), BI));
1601 }
1602
1603 // Make the PHI node use the select for all incoming values for BB1/BB2
1604 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
1605 if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2)
1606 PN.setIncomingValue(i, SI);
1607 }
1608 }
1609
1610 SmallVector<DominatorTree::UpdateType, 4> Updates;
1611
1612 // Update any PHI nodes in our new successors.
1613 for (BasicBlock *Succ : successors(BB1)) {
1614 AddPredecessorToBlock(Succ, BIParent, BB1);
1615 if (DTU)
1616 Updates.push_back({DominatorTree::Insert, BIParent, Succ});
1617 }
1618
1619 if (DTU)
1620 for (BasicBlock *Succ : successors(BI))
1621 Updates.push_back({DominatorTree::Delete, BIParent, Succ});
1622
1623 EraseTerminatorAndDCECond(BI);
1624 if (DTU)
1625 DTU->applyUpdates(Updates);
1626 return Changed;
1627}
1628
1629// Check lifetime markers.
1630static bool isLifeTimeMarker(const Instruction *I) {
1631 if (auto II = dyn_cast<IntrinsicInst>(I)) {
1632 switch (II->getIntrinsicID()) {
1633 default:
1634 break;
1635 case Intrinsic::lifetime_start:
1636 case Intrinsic::lifetime_end:
1637 return true;
1638 }
1639 }
1640 return false;
1641}
1642
1643// TODO: Refine this. This should avoid cases like turning constant memcpy sizes
1644// into variables.
1645static bool replacingOperandWithVariableIsCheap(const Instruction *I,
1646 int OpIdx) {
1647 return !isa<IntrinsicInst>(I);
1648}
1649
1650// All instructions in Insts belong to different blocks that all unconditionally
1651// branch to a common successor. Analyze each instruction and return true if it
1652// would be possible to sink them into their successor, creating one common
1653// instruction instead. For every value that would be required to be provided by
1654// PHI node (because an operand varies in each input block), add to PHIOperands.
1655static bool canSinkInstructions(
1656 ArrayRef<Instruction *> Insts,
1657 DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) {
1658 // Prune out obviously bad instructions to move. Each instruction must have
1659 // exactly zero or one use, and we check later that use is by a single, common
1660 // PHI instruction in the successor.
1661 bool HasUse = !Insts.front()->user_empty();
1662 for (auto *I : Insts) {
1663 // These instructions may change or break semantics if moved.
1664 if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) ||
1665 I->getType()->isTokenTy())
1666 return false;
1667
1668 // Do not try to sink an instruction in an infinite loop - it can cause
1669 // this algorithm to infinite loop.
1670 if (I->getParent()->getSingleSuccessor() == I->getParent())
1671 return false;
1672
1673 // Conservatively return false if I is an inline-asm instruction. Sinking
1674 // and merging inline-asm instructions can potentially create arguments
1675 // that cannot satisfy the inline-asm constraints.
1676 // If the instruction has nomerge attribute, return false.
1677 if (const auto *C = dyn_cast<CallBase>(I))
1678 if (C->isInlineAsm() || C->cannotMerge())
1679 return false;
1680
1681 // Each instruction must have zero or one use.
1682 if (HasUse && !I->hasOneUse())
1683 return false;
1684 if (!HasUse && !I->user_empty())
1685 return false;
1686 }
1687
1688 const Instruction *I0 = Insts.front();
1689 for (auto *I : Insts)
1690 if (!I->isSameOperationAs(I0))
1691 return false;
1692
1693 // All instructions in Insts are known to be the same opcode. If they have a
1694 // use, check that the only user is a PHI or in the same block as the
1695 // instruction, because if a user is in the same block as an instruction we're
1696 // contemplating sinking, it must already be determined to be sinkable.
1697 if (HasUse) {
1698 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1699 auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0);
1700 if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool {
1701 auto *U = cast<Instruction>(*I->user_begin());
1702 return (PNUse &&
1703 PNUse->getParent() == Succ &&
1704 PNUse->getIncomingValueForBlock(I->getParent()) == I) ||
1705 U->getParent() == I->getParent();
1706 }))
1707 return false;
1708 }
1709
1710 // Because SROA can't handle speculating stores of selects, try not to sink
1711 // loads, stores or lifetime markers of allocas when we'd have to create a
1712 // PHI for the address operand. Also, because it is likely that loads or
1713 // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink
1714 // them.
1715 // This can cause code churn which can have unintended consequences down
1716 // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244.
1717 // FIXME: This is a workaround for a deficiency in SROA - see
1718 // https://llvm.org/bugs/show_bug.cgi?id=30188
1719 if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) {
1720 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
1721 }))
1722 return false;
1723 if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) {
1724 return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
1725 }))
1726 return false;
1727 if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) {
1728 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
1729 }))
1730 return false;
1731
1732 for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) {
1733 Value *Op = I0->getOperand(OI);
1734 if (Op->getType()->isTokenTy())
1735 // Don't touch any operand of token type.
1736 return false;
1737
1738 auto SameAsI0 = [&I0, OI](const Instruction *I) {
1739 assert(I->getNumOperands() == I0->getNumOperands())((I->getNumOperands() == I0->getNumOperands()) ? static_cast
<void> (0) : __assert_fail ("I->getNumOperands() == I0->getNumOperands()"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1739, __PRETTY_FUNCTION__))
;
1740 return I->getOperand(OI) == I0->getOperand(OI);
1741 };
1742 if (!all_of(Insts, SameAsI0)) {
1743 if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) ||
1744 !canReplaceOperandWithVariable(I0, OI))
1745 // We can't create a PHI from this GEP.
1746 return false;
1747 // Don't create indirect calls! The called value is the final operand.
1748 if (isa<CallBase>(I0) && OI == OE - 1) {
1749 // FIXME: if the call was *already* indirect, we should do this.
1750 return false;
1751 }
1752 for (auto *I : Insts)
1753 PHIOperands[I].push_back(I->getOperand(OI));
1754 }
1755 }
1756 return true;
1757}
1758
1759// Assuming canSinkInstructions(Blocks) has returned true, sink the last
1760// instruction of every block in Blocks to their common successor, commoning
1761// into one instruction.
1762static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
1763 auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0);
1764
1765 // canSinkInstructions returning true guarantees that every block has at
1766 // least one non-terminator instruction.
1767 SmallVector<Instruction*,4> Insts;
1768 for (auto *BB : Blocks) {
1769 Instruction *I = BB->getTerminator();
1770 do {
1771 I = I->getPrevNode();
1772 } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front());
1773 if (!isa<DbgInfoIntrinsic>(I))
1774 Insts.push_back(I);
1775 }
1776
1777 // The only checking we need to do now is that all users of all instructions
1778 // are the same PHI node. canSinkInstructions should have checked this but
1779 // it is slightly over-aggressive - it gets confused by commutative
1780 // instructions so double-check it here.
1781 Instruction *I0 = Insts.front();
1782 if (!I0->user_empty()) {
1783 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1784 if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool {
1785 auto *U = cast<Instruction>(*I->user_begin());
1786 return U == PNUse;
1787 }))
1788 return false;
1789 }
1790
1791 // We don't need to do any more checking here; canSinkInstructions should
1792 // have done it all for us.
1793 SmallVector<Value*, 4> NewOperands;
1794 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) {
1795 // This check is different to that in canSinkInstructions. There, we
1796 // cared about the global view once simplifycfg (and instcombine) have
1797 // completed - it takes into account PHIs that become trivially
1798 // simplifiable. However here we need a more local view; if an operand
1799 // differs we create a PHI and rely on instcombine to clean up the very
1800 // small mess we may make.
1801 bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) {
1802 return I->getOperand(O) != I0->getOperand(O);
1803 });
1804 if (!NeedPHI) {
1805 NewOperands.push_back(I0->getOperand(O));
1806 continue;
1807 }
1808
1809 // Create a new PHI in the successor block and populate it.
1810 auto *Op = I0->getOperand(O);
1811 assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!")((!Op->getType()->isTokenTy() && "Can't PHI tokens!"
) ? static_cast<void> (0) : __assert_fail ("!Op->getType()->isTokenTy() && \"Can't PHI tokens!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 1811, __PRETTY_FUNCTION__))
;
1812 auto *PN = PHINode::Create(Op->getType(), Insts.size(),
1813 Op->getName() + ".sink", &BBEnd->front());
1814 for (auto *I : Insts)
1815 PN->addIncoming(I->getOperand(O), I->getParent());
1816 NewOperands.push_back(PN);
1817 }
1818
1819 // Arbitrarily use I0 as the new "common" instruction; remap its operands
1820 // and move it to the start of the successor block.
1821 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O)
1822 I0->getOperandUse(O).set(NewOperands[O]);
1823 I0->moveBefore(&*BBEnd->getFirstInsertionPt());
1824
1825 // Update metadata and IR flags, and merge debug locations.
1826 for (auto *I : Insts)
1827 if (I != I0) {
1828 // The debug location for the "common" instruction is the merged locations
1829 // of all the commoned instructions. We start with the original location
1830 // of the "common" instruction and iteratively merge each location in the
1831 // loop below.
1832 // This is an N-way merge, which will be inefficient if I0 is a CallInst.
1833 // However, as N-way merge for CallInst is rare, so we use simplified API
1834 // instead of using complex API for N-way merge.
1835 I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc());
1836 combineMetadataForCSE(I0, I, true);
1837 I0->andIRFlags(I);
1838 }
1839
1840 if (!I0->user_empty()) {
1841 // canSinkLastInstruction checked that all instructions were used by
1842 // one and only one PHI node. Find that now, RAUW it to our common
1843 // instruction and nuke it.
1844 auto *PN = cast<PHINode>(*I0->user_begin());
1845 PN->replaceAllUsesWith(I0);
1846 PN->eraseFromParent();
1847 }
1848
1849 // Finally nuke all instructions apart from the common instruction.
1850 for (auto *I : Insts)
1851 if (I != I0)
1852 I->eraseFromParent();
1853
1854 return true;
1855}
1856
1857namespace {
1858
1859 // LockstepReverseIterator - Iterates through instructions
1860 // in a set of blocks in reverse order from the first non-terminator.
1861 // For example (assume all blocks have size n):
1862 // LockstepReverseIterator I([B1, B2, B3]);
1863 // *I-- = [B1[n], B2[n], B3[n]];
1864 // *I-- = [B1[n-1], B2[n-1], B3[n-1]];
1865 // *I-- = [B1[n-2], B2[n-2], B3[n-2]];
1866 // ...
1867 class LockstepReverseIterator {
1868 ArrayRef<BasicBlock*> Blocks;
1869 SmallVector<Instruction*,4> Insts;
1870 bool Fail;
1871
1872 public:
1873 LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) : Blocks(Blocks) {
1874 reset();
1875 }
1876
1877 void reset() {
1878 Fail = false;
1879 Insts.clear();
1880 for (auto *BB : Blocks) {
1881 Instruction *Inst = BB->getTerminator();
1882 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1883 Inst = Inst->getPrevNode();
1884 if (!Inst) {
1885 // Block wasn't big enough.
1886 Fail = true;
1887 return;
1888 }
1889 Insts.push_back(Inst);
1890 }
1891 }
1892
1893 bool isValid() const {
1894 return !Fail;
1895 }
1896
1897 void operator--() {
1898 if (Fail)
1899 return;
1900 for (auto *&Inst : Insts) {
1901 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1902 Inst = Inst->getPrevNode();
1903 // Already at beginning of block.
1904 if (!Inst) {
1905 Fail = true;
1906 return;
1907 }
1908 }
1909 }
1910
1911 ArrayRef<Instruction*> operator * () const {
1912 return Insts;
1913 }
1914 };
1915
1916} // end anonymous namespace
1917
1918/// Check whether BB's predecessors end with unconditional branches. If it is
1919/// true, sink any common code from the predecessors to BB.
1920/// We also allow one predecessor to end with conditional branch (but no more
1921/// than one).
1922static bool SinkCommonCodeFromPredecessors(BasicBlock *BB,
1923 DomTreeUpdater *DTU) {
1924 // We support two situations:
1925 // (1) all incoming arcs are unconditional
1926 // (2) one incoming arc is conditional
1927 //
1928 // (2) is very common in switch defaults and
1929 // else-if patterns;
1930 //
1931 // if (a) f(1);
1932 // else if (b) f(2);
1933 //
1934 // produces:
1935 //
1936 // [if]
1937 // / \
1938 // [f(1)] [if]
1939 // | | \
1940 // | | |
1941 // | [f(2)]|
1942 // \ | /
1943 // [ end ]
1944 //
1945 // [end] has two unconditional predecessor arcs and one conditional. The
1946 // conditional refers to the implicit empty 'else' arc. This conditional
1947 // arc can also be caused by an empty default block in a switch.
1948 //
1949 // In this case, we attempt to sink code from all *unconditional* arcs.
1950 // If we can sink instructions from these arcs (determined during the scan
1951 // phase below) we insert a common successor for all unconditional arcs and
1952 // connect that to [end], to enable sinking:
1953 //
1954 // [if]
1955 // / \
1956 // [x(1)] [if]
1957 // | | \
1958 // | | \
1959 // | [x(2)] |
1960 // \ / |
1961 // [sink.split] |
1962 // \ /
1963 // [ end ]
1964 //
1965 SmallVector<BasicBlock*,4> UnconditionalPreds;
1966 Instruction *Cond = nullptr;
1967 for (auto *B : predecessors(BB)) {
1968 auto *T = B->getTerminator();
1969 if (isa<BranchInst>(T) && cast<BranchInst>(T)->isUnconditional())
1970 UnconditionalPreds.push_back(B);
1971 else if ((isa<BranchInst>(T) || isa<SwitchInst>(T)) && !Cond)
1972 Cond = T;
1973 else
1974 return false;
1975 }
1976 if (UnconditionalPreds.size() < 2)
1977 return false;
1978
1979 // We take a two-step approach to tail sinking. First we scan from the end of
1980 // each block upwards in lockstep. If the n'th instruction from the end of each
1981 // block can be sunk, those instructions are added to ValuesToSink and we
1982 // carry on. If we can sink an instruction but need to PHI-merge some operands
1983 // (because they're not identical in each instruction) we add these to
1984 // PHIOperands.
1985 unsigned ScanIdx = 0;
1986 SmallPtrSet<Value*,4> InstructionsToSink;
1987 DenseMap<Instruction*, SmallVector<Value*,4>> PHIOperands;
1988 LockstepReverseIterator LRI(UnconditionalPreds);
1989 while (LRI.isValid() &&
1990 canSinkInstructions(*LRI, PHIOperands)) {
1991 LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: instruction can be sunk: "
<< *(*LRI)[0] << "\n"; } } while (false)
1992 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: instruction can be sunk: "
<< *(*LRI)[0] << "\n"; } } while (false)
;
1993 InstructionsToSink.insert((*LRI).begin(), (*LRI).end());
1994 ++ScanIdx;
1995 --LRI;
1996 }
1997
1998 // If no instructions can be sunk, early-return.
1999 if (ScanIdx == 0)
2000 return false;
2001
2002 bool Changed = false;
2003
2004 auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) {
2005 unsigned NumPHIdValues = 0;
2006 for (auto *I : *LRI)
2007 for (auto *V : PHIOperands[I])
2008 if (InstructionsToSink.count(V) == 0)
2009 ++NumPHIdValues;
2010 LLVM_DEBUG(dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: #phid values: " <<
NumPHIdValues << "\n"; } } while (false)
;
2011 unsigned NumPHIInsts = NumPHIdValues / UnconditionalPreds.size();
2012 if ((NumPHIdValues % UnconditionalPreds.size()) != 0)
2013 NumPHIInsts++;
2014
2015 return NumPHIInsts <= 1;
2016 };
2017
2018 if (Cond) {
2019 // Check if we would actually sink anything first! This mutates the CFG and
2020 // adds an extra block. The goal in doing this is to allow instructions that
2021 // couldn't be sunk before to be sunk - obviously, speculatable instructions
2022 // (such as trunc, add) can be sunk and predicated already. So we check that
2023 // we're going to sink at least one non-speculatable instruction.
2024 LRI.reset();
2025 unsigned Idx = 0;
2026 bool Profitable = false;
2027 while (ProfitableToSinkInstruction(LRI) && Idx < ScanIdx) {
2028 if (!isSafeToSpeculativelyExecute((*LRI)[0])) {
2029 Profitable = true;
2030 break;
2031 }
2032 --LRI;
2033 ++Idx;
2034 }
2035 if (!Profitable)
2036 return false;
2037
2038 LLVM_DEBUG(dbgs() << "SINK: Splitting edge\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Splitting edge\n"; }
} while (false)
;
2039 // We have a conditional edge and we're going to sink some instructions.
2040 // Insert a new block postdominating all blocks we're going to sink from.
2041 if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split", DTU))
2042 // Edges couldn't be split.
2043 return false;
2044 Changed = true;
2045 }
2046
2047 // Now that we've analyzed all potential sinking candidates, perform the
2048 // actual sink. We iteratively sink the last non-terminator of the source
2049 // blocks into their common successor unless doing so would require too
2050 // many PHI instructions to be generated (currently only one PHI is allowed
2051 // per sunk instruction).
2052 //
2053 // We can use InstructionsToSink to discount values needing PHI-merging that will
2054 // actually be sunk in a later iteration. This allows us to be more
2055 // aggressive in what we sink. This does allow a false positive where we
2056 // sink presuming a later value will also be sunk, but stop half way through
2057 // and never actually sink it which means we produce more PHIs than intended.
2058 // This is unlikely in practice though.
2059 unsigned SinkIdx = 0;
2060 for (; SinkIdx != ScanIdx; ++SinkIdx) {
2061 LLVM_DEBUG(dbgs() << "SINK: Sink: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds
[0]->getTerminator()->getPrevNode() << "\n"; } } while
(false)
2062 << *UnconditionalPreds[0]->getTerminator()->getPrevNode()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds
[0]->getTerminator()->getPrevNode() << "\n"; } } while
(false)
2063 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: Sink: " << *UnconditionalPreds
[0]->getTerminator()->getPrevNode() << "\n"; } } while
(false)
;
2064
2065 // Because we've sunk every instruction in turn, the current instruction to
2066 // sink is always at index 0.
2067 LRI.reset();
2068 if (!ProfitableToSinkInstruction(LRI)) {
2069 // Too many PHIs would be created.
2070 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, too many PHIs would be created!\n"
; } } while (false)
2071 dbgs() << "SINK: stopping here, too many PHIs would be created!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, too many PHIs would be created!\n"
; } } while (false)
;
2072 break;
2073 }
2074
2075 if (!sinkLastInstruction(UnconditionalPreds)) {
2076 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, failed to actually sink instruction!\n"
; } } while (false)
2077 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, failed to actually sink instruction!\n"
; } } while (false)
2078 << "SINK: stopping here, failed to actually sink instruction!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SINK: stopping here, failed to actually sink instruction!\n"
; } } while (false)
;
2079 break;
2080 }
2081
2082 NumSinkCommonInstrs++;
2083 Changed = true;
2084 }
2085 if (SinkIdx != 0)
2086 ++NumSinkCommonCode;
2087 return Changed;
2088}
2089
2090/// Determine if we can hoist sink a sole store instruction out of a
2091/// conditional block.
2092///
2093/// We are looking for code like the following:
2094/// BrBB:
2095/// store i32 %add, i32* %arrayidx2
2096/// ... // No other stores or function calls (we could be calling a memory
2097/// ... // function).
2098/// %cmp = icmp ult %x, %y
2099/// br i1 %cmp, label %EndBB, label %ThenBB
2100/// ThenBB:
2101/// store i32 %add5, i32* %arrayidx2
2102/// br label EndBB
2103/// EndBB:
2104/// ...
2105/// We are going to transform this into:
2106/// BrBB:
2107/// store i32 %add, i32* %arrayidx2
2108/// ... //
2109/// %cmp = icmp ult %x, %y
2110/// %add.add5 = select i1 %cmp, i32 %add, %add5
2111/// store i32 %add.add5, i32* %arrayidx2
2112/// ...
2113///
2114/// \return The pointer to the value of the previous store if the store can be
2115/// hoisted into the predecessor block. 0 otherwise.
2116static Value *isSafeToSpeculateStore(Instruction *I, BasicBlock *BrBB,
2117 BasicBlock *StoreBB, BasicBlock *EndBB) {
2118 StoreInst *StoreToHoist = dyn_cast<StoreInst>(I);
2119 if (!StoreToHoist)
2120 return nullptr;
2121
2122 // Volatile or atomic.
2123 if (!StoreToHoist->isSimple())
2124 return nullptr;
2125
2126 Value *StorePtr = StoreToHoist->getPointerOperand();
2127
2128 // Look for a store to the same pointer in BrBB.
2129 unsigned MaxNumInstToLookAt = 9;
2130 // Skip pseudo probe intrinsic calls which are not really killing any memory
2131 // accesses.
2132 for (Instruction &CurI : reverse(BrBB->instructionsWithoutDebug(true))) {
2133 if (!MaxNumInstToLookAt)
2134 break;
2135 --MaxNumInstToLookAt;
2136
2137 // Could be calling an instruction that affects memory like free().
2138 if (CurI.mayHaveSideEffects() && !isa<StoreInst>(CurI))
2139 return nullptr;
2140
2141 if (auto *SI = dyn_cast<StoreInst>(&CurI)) {
2142 // Found the previous store make sure it stores to the same location.
2143 if (SI->getPointerOperand() == StorePtr)
2144 // Found the previous store, return its value operand.
2145 return SI->getValueOperand();
2146 return nullptr; // Unknown store.
2147 }
2148 }
2149
2150 return nullptr;
2151}
2152
2153/// Estimate the cost of the insertion(s) and check that the PHI nodes can be
2154/// converted to selects.
2155static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB,
2156 BasicBlock *EndBB,
2157 unsigned &SpeculatedInstructions,
2158 InstructionCost &Cost,
2159 const TargetTransformInfo &TTI) {
2160 TargetTransformInfo::TargetCostKind CostKind =
2161 BB->getParent()->hasMinSize()
2162 ? TargetTransformInfo::TCK_CodeSize
2163 : TargetTransformInfo::TCK_SizeAndLatency;
2164
2165 bool HaveRewritablePHIs = false;
2166 for (PHINode &PN : EndBB->phis()) {
2167 Value *OrigV = PN.getIncomingValueForBlock(BB);
2168 Value *ThenV = PN.getIncomingValueForBlock(ThenBB);
2169
2170 // FIXME: Try to remove some of the duplication with HoistThenElseCodeToIf.
2171 // Skip PHIs which are trivial.
2172 if (ThenV == OrigV)
2173 continue;
2174
2175 Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr,
2176 CmpInst::BAD_ICMP_PREDICATE, CostKind);
2177
2178 // Don't convert to selects if we could remove undefined behavior instead.
2179 if (passingValueIsAlwaysUndefined(OrigV, &PN) ||
2180 passingValueIsAlwaysUndefined(ThenV, &PN))
2181 return false;
2182
2183 HaveRewritablePHIs = true;
2184 ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV);
2185 ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV);
2186 if (!OrigCE && !ThenCE)
2187 continue; // Known safe and cheap.
2188
2189 if ((ThenCE && !isSafeToSpeculativelyExecute(ThenCE)) ||
2190 (OrigCE && !isSafeToSpeculativelyExecute(OrigCE)))
2191 return false;
2192 InstructionCost OrigCost = OrigCE ? computeSpeculationCost(OrigCE, TTI) : 0;
2193 InstructionCost ThenCost = ThenCE ? computeSpeculationCost(ThenCE, TTI) : 0;
2194 InstructionCost MaxCost =
2195 2 * PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2196 if (OrigCost + ThenCost > MaxCost)
2197 return false;
2198
2199 // Account for the cost of an unfolded ConstantExpr which could end up
2200 // getting expanded into Instructions.
2201 // FIXME: This doesn't account for how many operations are combined in the
2202 // constant expression.
2203 ++SpeculatedInstructions;
2204 if (SpeculatedInstructions > 1)
2205 return false;
2206 }
2207
2208 return HaveRewritablePHIs;
2209}
2210
2211/// Speculate a conditional basic block flattening the CFG.
2212///
2213/// Note that this is a very risky transform currently. Speculating
2214/// instructions like this is most often not desirable. Instead, there is an MI
2215/// pass which can do it with full awareness of the resource constraints.
2216/// However, some cases are "obvious" and we should do directly. An example of
2217/// this is speculating a single, reasonably cheap instruction.
2218///
2219/// There is only one distinct advantage to flattening the CFG at the IR level:
2220/// it makes very common but simplistic optimizations such as are common in
2221/// instcombine and the DAG combiner more powerful by removing CFG edges and
2222/// modeling their effects with easier to reason about SSA value graphs.
2223///
2224///
2225/// An illustration of this transform is turning this IR:
2226/// \code
2227/// BB:
2228/// %cmp = icmp ult %x, %y
2229/// br i1 %cmp, label %EndBB, label %ThenBB
2230/// ThenBB:
2231/// %sub = sub %x, %y
2232/// br label BB2
2233/// EndBB:
2234/// %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ]
2235/// ...
2236/// \endcode
2237///
2238/// Into this IR:
2239/// \code
2240/// BB:
2241/// %cmp = icmp ult %x, %y
2242/// %sub = sub %x, %y
2243/// %cond = select i1 %cmp, 0, %sub
2244/// ...
2245/// \endcode
2246///
2247/// \returns true if the conditional block is removed.
2248bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
2249 const TargetTransformInfo &TTI) {
2250 // Be conservative for now. FP select instruction can often be expensive.
2251 Value *BrCond = BI->getCondition();
2252 if (isa<FCmpInst>(BrCond))
2253 return false;
2254
2255 BasicBlock *BB = BI->getParent();
2256 BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0);
2257 InstructionCost Budget =
2258 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2259
2260 // If ThenBB is actually on the false edge of the conditional branch, remember
2261 // to swap the select operands later.
2262 bool Invert = false;
2263 if (ThenBB != BI->getSuccessor(0)) {
2264 assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?")((ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?"
) ? static_cast<void> (0) : __assert_fail ("ThenBB == BI->getSuccessor(1) && \"No edge from 'if' block?\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2264, __PRETTY_FUNCTION__))
;
2265 Invert = true;
2266 }
2267 assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block")((EndBB == BI->getSuccessor(!Invert) && "No edge from to end block"
) ? static_cast<void> (0) : __assert_fail ("EndBB == BI->getSuccessor(!Invert) && \"No edge from to end block\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2267, __PRETTY_FUNCTION__))
;
2268
2269 // Keep a count of how many times instructions are used within ThenBB when
2270 // they are candidates for sinking into ThenBB. Specifically:
2271 // - They are defined in BB, and
2272 // - They have no side effects, and
2273 // - All of their uses are in ThenBB.
2274 SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts;
2275
2276 SmallVector<Instruction *, 4> SpeculatedDbgIntrinsics;
2277
2278 unsigned SpeculatedInstructions = 0;
2279 Value *SpeculatedStoreValue = nullptr;
2280 StoreInst *SpeculatedStore = nullptr;
2281 for (BasicBlock::iterator BBI = ThenBB->begin(),
2282 BBE = std::prev(ThenBB->end());
2283 BBI != BBE; ++BBI) {
2284 Instruction *I = &*BBI;
2285 // Skip debug info.
2286 if (isa<DbgInfoIntrinsic>(I)) {
2287 SpeculatedDbgIntrinsics.push_back(I);
2288 continue;
2289 }
2290
2291 // Skip pseudo probes. The consequence is we lose track of the branch
2292 // probability for ThenBB, which is fine since the optimization here takes
2293 // place regardless of the branch probability.
2294 if (isa<PseudoProbeInst>(I)) {
2295 continue;
2296 }
2297
2298 // Only speculatively execute a single instruction (not counting the
2299 // terminator) for now.
2300 ++SpeculatedInstructions;
2301 if (SpeculatedInstructions > 1)
2302 return false;
2303
2304 // Don't hoist the instruction if it's unsafe or expensive.
2305 if (!isSafeToSpeculativelyExecute(I) &&
2306 !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore(
2307 I, BB, ThenBB, EndBB))))
2308 return false;
2309 if (!SpeculatedStoreValue &&
2310 computeSpeculationCost(I, TTI) >
2311 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic)
2312 return false;
2313
2314 // Store the store speculation candidate.
2315 if (SpeculatedStoreValue)
2316 SpeculatedStore = cast<StoreInst>(I);
2317
2318 // Do not hoist the instruction if any of its operands are defined but not
2319 // used in BB. The transformation will prevent the operand from
2320 // being sunk into the use block.
2321 for (Use &Op : I->operands()) {
2322 Instruction *OpI = dyn_cast<Instruction>(Op);
2323 if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects())
2324 continue; // Not a candidate for sinking.
2325
2326 ++SinkCandidateUseCounts[OpI];
2327 }
2328 }
2329
2330 // Consider any sink candidates which are only used in ThenBB as costs for
2331 // speculation. Note, while we iterate over a DenseMap here, we are summing
2332 // and so iteration order isn't significant.
2333 for (SmallDenseMap<Instruction *, unsigned, 4>::iterator
2334 I = SinkCandidateUseCounts.begin(),
2335 E = SinkCandidateUseCounts.end();
2336 I != E; ++I)
2337 if (I->first->hasNUses(I->second)) {
2338 ++SpeculatedInstructions;
2339 if (SpeculatedInstructions > 1)
2340 return false;
2341 }
2342
2343 // Check that we can insert the selects and that it's not too expensive to do
2344 // so.
2345 bool Convert = SpeculatedStore != nullptr;
2346 InstructionCost Cost = 0;
2347 Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB,
2348 SpeculatedInstructions,
2349 Cost, TTI);
2350 if (!Convert || Cost > Budget)
2351 return false;
2352
2353 // If we get here, we can hoist the instruction and if-convert.
2354 LLVM_DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SPECULATIVELY EXECUTING BB"
<< *ThenBB << "\n";; } } while (false)
;
2355
2356 // Insert a select of the value of the speculated store.
2357 if (SpeculatedStoreValue) {
2358 IRBuilder<NoFolder> Builder(BI);
2359 Value *TrueV = SpeculatedStore->getValueOperand();
2360 Value *FalseV = SpeculatedStoreValue;
2361 if (Invert)
2362 std::swap(TrueV, FalseV);
2363 Value *S = Builder.CreateSelect(
2364 BrCond, TrueV, FalseV, "spec.store.select", BI);
2365 SpeculatedStore->setOperand(0, S);
2366 SpeculatedStore->applyMergedLocation(BI->getDebugLoc(),
2367 SpeculatedStore->getDebugLoc());
2368 }
2369
2370 // Metadata can be dependent on the condition we are hoisting above.
2371 // Conservatively strip all metadata on the instruction. Drop the debug loc
2372 // to avoid making it appear as if the condition is a constant, which would
2373 // be misleading while debugging.
2374 for (auto &I : *ThenBB) {
2375 if (!SpeculatedStoreValue || &I != SpeculatedStore)
2376 I.setDebugLoc(DebugLoc());
2377 I.dropUnknownNonDebugMetadata();
2378 }
2379
2380 // A hoisted conditional probe should be treated as dangling so that it will
2381 // not be over-counted when the samples collected on the non-conditional path
2382 // are counted towards the conditional path. We leave it for the counts
2383 // inference algorithm to figure out a proper count for a danglng probe.
2384 moveAndDanglePseudoProbes(ThenBB, BI);
2385
2386 // Hoist the instructions.
2387 BB->getInstList().splice(BI->getIterator(), ThenBB->getInstList(),
2388 ThenBB->begin(), std::prev(ThenBB->end()));
2389
2390 // Insert selects and rewrite the PHI operands.
2391 IRBuilder<NoFolder> Builder(BI);
2392 for (PHINode &PN : EndBB->phis()) {
2393 unsigned OrigI = PN.getBasicBlockIndex(BB);
2394 unsigned ThenI = PN.getBasicBlockIndex(ThenBB);
2395 Value *OrigV = PN.getIncomingValue(OrigI);
2396 Value *ThenV = PN.getIncomingValue(ThenI);
2397
2398 // Skip PHIs which are trivial.
2399 if (OrigV == ThenV)
2400 continue;
2401
2402 // Create a select whose true value is the speculatively executed value and
2403 // false value is the pre-existing value. Swap them if the branch
2404 // destinations were inverted.
2405 Value *TrueV = ThenV, *FalseV = OrigV;
2406 if (Invert)
2407 std::swap(TrueV, FalseV);
2408 Value *V = Builder.CreateSelect(BrCond, TrueV, FalseV, "spec.select", BI);
2409 PN.setIncomingValue(OrigI, V);
2410 PN.setIncomingValue(ThenI, V);
2411 }
2412
2413 // Remove speculated dbg intrinsics.
2414 // FIXME: Is it possible to do this in a more elegant way? Moving/merging the
2415 // dbg value for the different flows and inserting it after the select.
2416 for (Instruction *I : SpeculatedDbgIntrinsics)
2417 I->eraseFromParent();
2418
2419 ++NumSpeculations;
2420 return true;
2421}
2422
2423/// Return true if we can thread a branch across this block.
2424static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
2425 int Size = 0;
2426
2427 for (Instruction &I : BB->instructionsWithoutDebug()) {
2428 if (Size > MaxSmallBlockSize)
2429 return false; // Don't clone large BB's.
2430
2431 // Can't fold blocks that contain noduplicate or convergent calls.
2432 if (CallInst *CI = dyn_cast<CallInst>(&I))
2433 if (CI->cannotDuplicate() || CI->isConvergent())
2434 return false;
2435
2436 // We will delete Phis while threading, so Phis should not be accounted in
2437 // block's size
2438 if (!isa<PHINode>(I))
2439 ++Size;
2440
2441 // We can only support instructions that do not define values that are
2442 // live outside of the current basic block.
2443 for (User *U : I.users()) {
2444 Instruction *UI = cast<Instruction>(U);
2445 if (UI->getParent() != BB || isa<PHINode>(UI))
2446 return false;
2447 }
2448
2449 // Looks ok, continue checking.
2450 }
2451
2452 return true;
2453}
2454
2455/// If we have a conditional branch on a PHI node value that is defined in the
2456/// same block as the branch and if any PHI entries are constants, thread edges
2457/// corresponding to that entry to be branches to their ultimate destination.
2458static bool FoldCondBranchOnPHI(BranchInst *BI, DomTreeUpdater *DTU,
2459 const DataLayout &DL, AssumptionCache *AC) {
2460 BasicBlock *BB = BI->getParent();
2461 PHINode *PN = dyn_cast<PHINode>(BI->getCondition());
2462 // NOTE: we currently cannot transform this case if the PHI node is used
2463 // outside of the block.
2464 if (!PN || PN->getParent() != BB || !PN->hasOneUse())
2465 return false;
2466
2467 // Degenerate case of a single entry PHI.
2468 if (PN->getNumIncomingValues() == 1) {
2469 FoldSingleEntryPHINodes(PN->getParent());
2470 return true;
2471 }
2472
2473 // Now we know that this block has multiple preds and two succs.
2474 if (!BlockIsSimpleEnoughToThreadThrough(BB))
2475 return false;
2476
2477 // Okay, this is a simple enough basic block. See if any phi values are
2478 // constants.
2479 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
2480 ConstantInt *CB = dyn_cast<ConstantInt>(PN->getIncomingValue(i));
2481 if (!CB || !CB->getType()->isIntegerTy(1))
2482 continue;
2483
2484 // Okay, we now know that all edges from PredBB should be revectored to
2485 // branch to RealDest.
2486 BasicBlock *PredBB = PN->getIncomingBlock(i);
2487 BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue());
2488
2489 if (RealDest == BB)
2490 continue; // Skip self loops.
2491 // Skip if the predecessor's terminator is an indirect branch.
2492 if (isa<IndirectBrInst>(PredBB->getTerminator()))
2493 continue;
2494
2495 SmallVector<DominatorTree::UpdateType, 3> Updates;
2496
2497 // The dest block might have PHI nodes, other predecessors and other
2498 // difficult cases. Instead of being smart about this, just insert a new
2499 // block that jumps to the destination block, effectively splitting
2500 // the edge we are about to create.
2501 BasicBlock *EdgeBB =
2502 BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge",
2503 RealDest->getParent(), RealDest);
2504 BranchInst *CritEdgeBranch = BranchInst::Create(RealDest, EdgeBB);
2505 if (DTU)
2506 Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest});
2507 CritEdgeBranch->setDebugLoc(BI->getDebugLoc());
2508
2509 // Update PHI nodes.
2510 AddPredecessorToBlock(RealDest, EdgeBB, BB);
2511
2512 // BB may have instructions that are being threaded over. Clone these
2513 // instructions into EdgeBB. We know that there will be no uses of the
2514 // cloned instructions outside of EdgeBB.
2515 BasicBlock::iterator InsertPt = EdgeBB->begin();
2516 DenseMap<Value *, Value *> TranslateMap; // Track translated values.
2517 for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
2518 if (PHINode *PN = dyn_cast<PHINode>(BBI)) {
2519 TranslateMap[PN] = PN->getIncomingValueForBlock(PredBB);
2520 continue;
2521 }
2522 // Clone the instruction.
2523 Instruction *N = BBI->clone();
2524 if (BBI->hasName())
2525 N->setName(BBI->getName() + ".c");
2526
2527 // Update operands due to translation.
2528 for (Use &Op : N->operands()) {
2529 DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(Op);
2530 if (PI != TranslateMap.end())
2531 Op = PI->second;
2532 }
2533
2534 // Check for trivial simplification.
2535 if (Value *V = SimplifyInstruction(N, {DL, nullptr, nullptr, AC})) {
2536 if (!BBI->use_empty())
2537 TranslateMap[&*BBI] = V;
2538 if (!N->mayHaveSideEffects()) {
2539 N->deleteValue(); // Instruction folded away, don't need actual inst
2540 N = nullptr;
2541 }
2542 } else {
2543 if (!BBI->use_empty())
2544 TranslateMap[&*BBI] = N;
2545 }
2546 if (N) {
2547 // Insert the new instruction into its new home.
2548 EdgeBB->getInstList().insert(InsertPt, N);
2549
2550 // Register the new instruction with the assumption cache if necessary.
2551 if (auto *Assume = dyn_cast<AssumeInst>(N))
2552 if (AC)
2553 AC->registerAssumption(Assume);
2554 }
2555 }
2556
2557 // Loop over all of the edges from PredBB to BB, changing them to branch
2558 // to EdgeBB instead.
2559 Instruction *PredBBTI = PredBB->getTerminator();
2560 for (unsigned i = 0, e = PredBBTI->getNumSuccessors(); i != e; ++i)
2561 if (PredBBTI->getSuccessor(i) == BB) {
2562 BB->removePredecessor(PredBB);
2563 PredBBTI->setSuccessor(i, EdgeBB);
2564 }
2565
2566 if (DTU) {
2567 Updates.push_back({DominatorTree::Insert, PredBB, EdgeBB});
2568 Updates.push_back({DominatorTree::Delete, PredBB, BB});
2569
2570 DTU->applyUpdates(Updates);
2571 }
2572
2573 // Recurse, simplifying any other constants.
2574 return FoldCondBranchOnPHI(BI, DTU, DL, AC) || true;
2575 }
2576
2577 return false;
2578}
2579
2580/// Given a BB that starts with the specified two-entry PHI node,
2581/// see if we can eliminate it.
2582static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
2583 DomTreeUpdater *DTU, const DataLayout &DL) {
2584 // Ok, this is a two entry PHI node. Check to see if this is a simple "if
2585 // statement", which has a very simple dominance structure. Basically, we
2586 // are trying to find the condition that is being branched on, which
2587 // subsequently causes this merge to happen. We really want control
2588 // dependence information for this check, but simplifycfg can't keep it up
2589 // to date, and this catches most of the cases we care about anyway.
2590 BasicBlock *BB = PN->getParent();
2591
2592 BasicBlock *IfTrue, *IfFalse;
2593 Value *IfCond = GetIfCondition(BB, IfTrue, IfFalse);
2594 if (!IfCond ||
2595 // Don't bother if the branch will be constant folded trivially.
2596 isa<ConstantInt>(IfCond))
2597 return false;
2598
2599 // Okay, we found that we can merge this two-entry phi node into a select.
2600 // Doing so would require us to fold *all* two entry phi nodes in this block.
2601 // At some point this becomes non-profitable (particularly if the target
2602 // doesn't support cmov's). Only do this transformation if there are two or
2603 // fewer PHI nodes in this block.
2604 unsigned NumPhis = 0;
2605 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I)
2606 if (NumPhis > 2)
2607 return false;
2608
2609 // Loop over the PHI's seeing if we can promote them all to select
2610 // instructions. While we are at it, keep track of the instructions
2611 // that need to be moved to the dominating block.
2612 SmallPtrSet<Instruction *, 4> AggressiveInsts;
2613 InstructionCost Cost = 0;
2614 InstructionCost Budget =
2615 TwoEntryPHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2616
2617 bool Changed = false;
2618 for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
2619 PHINode *PN = cast<PHINode>(II++);
2620 if (Value *V = SimplifyInstruction(PN, {DL, PN})) {
2621 PN->replaceAllUsesWith(V);
2622 PN->eraseFromParent();
2623 Changed = true;
2624 continue;
2625 }
2626
2627 if (!dominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts,
2628 Cost, Budget, TTI) ||
2629 !dominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts,
2630 Cost, Budget, TTI))
2631 return Changed;
2632 }
2633
2634 // If we folded the first phi, PN dangles at this point. Refresh it. If
2635 // we ran out of PHIs then we simplified them all.
2636 PN = dyn_cast<PHINode>(BB->begin());
2637 if (!PN)
2638 return true;
2639
2640 // Return true if at least one of these is a 'not', and another is either
2641 // a 'not' too, or a constant.
2642 auto CanHoistNotFromBothValues = [](Value *V0, Value *V1) {
2643 if (!match(V0, m_Not(m_Value())))
2644 std::swap(V0, V1);
2645 auto Invertible = m_CombineOr(m_Not(m_Value()), m_AnyIntegralConstant());
2646 return match(V0, m_Not(m_Value())) && match(V1, Invertible);
2647 };
2648
2649 // Don't fold i1 branches on PHIs which contain binary operators or
2650 // select form of or/ands, unless one of the incoming values is an 'not' and
2651 // another one is freely invertible.
2652 // These can often be turned into switches and other things.
2653 auto IsBinOpOrAnd = [](Value *V) {
2654 return match(
2655 V, m_CombineOr(m_BinOp(), m_CombineOr(m_LogicalAnd(), m_LogicalOr())));
2656 };
2657 if (PN->getType()->isIntegerTy(1) &&
2658 (IsBinOpOrAnd(PN->getIncomingValue(0)) ||
2659 IsBinOpOrAnd(PN->getIncomingValue(1)) || IsBinOpOrAnd(IfCond)) &&
2660 !CanHoistNotFromBothValues(PN->getIncomingValue(0),
2661 PN->getIncomingValue(1)))
2662 return Changed;
2663
2664 // If all PHI nodes are promotable, check to make sure that all instructions
2665 // in the predecessor blocks can be promoted as well. If not, we won't be able
2666 // to get rid of the control flow, so it's not worth promoting to select
2667 // instructions.
2668 BasicBlock *DomBlock = nullptr;
2669 BasicBlock *IfBlock1 = PN->getIncomingBlock(0);
2670 BasicBlock *IfBlock2 = PN->getIncomingBlock(1);
2671 if (cast<BranchInst>(IfBlock1->getTerminator())->isConditional()) {
2672 IfBlock1 = nullptr;
2673 } else {
2674 DomBlock = *pred_begin(IfBlock1);
2675 for (BasicBlock::iterator I = IfBlock1->begin(); !I->isTerminator(); ++I)
2676 if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I) &&
2677 !isa<PseudoProbeInst>(I)) {
2678 // This is not an aggressive instruction that we can promote.
2679 // Because of this, we won't be able to get rid of the control flow, so
2680 // the xform is not worth it.
2681 return Changed;
2682 }
2683 }
2684
2685 if (cast<BranchInst>(IfBlock2->getTerminator())->isConditional()) {
2686 IfBlock2 = nullptr;
2687 } else {
2688 DomBlock = *pred_begin(IfBlock2);
2689 for (BasicBlock::iterator I = IfBlock2->begin(); !I->isTerminator(); ++I)
2690 if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I) &&
2691 !isa<PseudoProbeInst>(I)) {
2692 // This is not an aggressive instruction that we can promote.
2693 // Because of this, we won't be able to get rid of the control flow, so
2694 // the xform is not worth it.
2695 return Changed;
2696 }
2697 }
2698 assert(DomBlock && "Failed to find root DomBlock")((DomBlock && "Failed to find root DomBlock") ? static_cast
<void> (0) : __assert_fail ("DomBlock && \"Failed to find root DomBlock\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2698, __PRETTY_FUNCTION__))
;
2699
2700 LLVM_DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfConddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " <<
*IfCond << " T: " << IfTrue->getName() <<
" F: " << IfFalse->getName() << "\n"; } } while
(false)
2701 << " T: " << IfTrue->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " <<
*IfCond << " T: " << IfTrue->getName() <<
" F: " << IfFalse->getName() << "\n"; } } while
(false)
2702 << " F: " << IfFalse->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOUND IF CONDITION! " <<
*IfCond << " T: " << IfTrue->getName() <<
" F: " << IfFalse->getName() << "\n"; } } while
(false)
;
2703
2704 // If we can still promote the PHI nodes after this gauntlet of tests,
2705 // do all of the PHI's now.
2706 Instruction *InsertPt = DomBlock->getTerminator();
2707 IRBuilder<NoFolder> Builder(InsertPt);
2708
2709 // Move all 'aggressive' instructions, which are defined in the
2710 // conditional parts of the if's up to the dominating block.
2711 if (IfBlock1)
2712 hoistAllInstructionsInto(DomBlock, InsertPt, IfBlock1);
2713 if (IfBlock2)
2714 hoistAllInstructionsInto(DomBlock, InsertPt, IfBlock2);
2715
2716 // Propagate fast-math-flags from phi nodes to replacement selects.
2717 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2718 while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
2719 if (isa<FPMathOperator>(PN))
2720 Builder.setFastMathFlags(PN->getFastMathFlags());
2721
2722 // Change the PHI node into a select instruction.
2723 Value *TrueVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfFalse);
2724 Value *FalseVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfTrue);
2725
2726 Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", InsertPt);
2727 PN->replaceAllUsesWith(Sel);
2728 Sel->takeName(PN);
2729 PN->eraseFromParent();
2730 }
2731
2732 // At this point, IfBlock1 and IfBlock2 are both empty, so our if statement
2733 // has been flattened. Change DomBlock to jump directly to our new block to
2734 // avoid other simplifycfg's kicking in on the diamond.
2735 Instruction *OldTI = DomBlock->getTerminator();
2736 Builder.SetInsertPoint(OldTI);
2737 Builder.CreateBr(BB);
2738
2739 SmallVector<DominatorTree::UpdateType, 3> Updates;
2740 if (DTU) {
2741 Updates.push_back({DominatorTree::Insert, DomBlock, BB});
2742 for (auto *Successor : successors(DomBlock))
2743 Updates.push_back({DominatorTree::Delete, DomBlock, Successor});
2744 }
2745
2746 OldTI->eraseFromParent();
2747 if (DTU)
2748 DTU->applyUpdates(Updates);
2749
2750 return true;
2751}
2752
2753/// If we found a conditional branch that goes to two returning blocks,
2754/// try to merge them together into one return,
2755/// introducing a select if the return values disagree.
2756bool SimplifyCFGOpt::SimplifyCondBranchToTwoReturns(BranchInst *BI,
2757 IRBuilder<> &Builder) {
2758 auto *BB = BI->getParent();
2759 assert(BI->isConditional() && "Must be a conditional branch")((BI->isConditional() && "Must be a conditional branch"
) ? static_cast<void> (0) : __assert_fail ("BI->isConditional() && \"Must be a conditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2759, __PRETTY_FUNCTION__))
;
2760 BasicBlock *TrueSucc = BI->getSuccessor(0);
2761 BasicBlock *FalseSucc = BI->getSuccessor(1);
2762 // NOTE: destinations may match, this could be degenerate uncond branch.
2763 ReturnInst *TrueRet = cast<ReturnInst>(TrueSucc->getTerminator());
2764 ReturnInst *FalseRet = cast<ReturnInst>(FalseSucc->getTerminator());
2765
2766 // Check to ensure both blocks are empty (just a return) or optionally empty
2767 // with PHI nodes. If there are other instructions, merging would cause extra
2768 // computation on one path or the other.
2769 if (!TrueSucc->getFirstNonPHIOrDbg()->isTerminator())
2770 return false;
2771 if (!FalseSucc->getFirstNonPHIOrDbg()->isTerminator())
2772 return false;
2773
2774 Builder.SetInsertPoint(BI);
2775 // Okay, we found a branch that is going to two return nodes. If
2776 // there is no return value for this function, just change the
2777 // branch into a return.
2778 if (FalseRet->getNumOperands() == 0) {
2779 TrueSucc->removePredecessor(BB);
2780 FalseSucc->removePredecessor(BB);
2781 Builder.CreateRetVoid();
2782 EraseTerminatorAndDCECond(BI);
2783 if (DTU) {
2784 SmallVector<DominatorTree::UpdateType, 2> Updates;
2785 Updates.push_back({DominatorTree::Delete, BB, TrueSucc});
2786 if (TrueSucc != FalseSucc)
2787 Updates.push_back({DominatorTree::Delete, BB, FalseSucc});
2788 DTU->applyUpdates(Updates);
2789 }
2790 return true;
2791 }
2792
2793 // Otherwise, figure out what the true and false return values are
2794 // so we can insert a new select instruction.
2795 Value *TrueValue = TrueRet->getReturnValue();
2796 Value *FalseValue = FalseRet->getReturnValue();
2797
2798 // Unwrap any PHI nodes in the return blocks.
2799 if (PHINode *TVPN = dyn_cast_or_null<PHINode>(TrueValue))
2800 if (TVPN->getParent() == TrueSucc)
2801 TrueValue = TVPN->getIncomingValueForBlock(BB);
2802 if (PHINode *FVPN = dyn_cast_or_null<PHINode>(FalseValue))
2803 if (FVPN->getParent() == FalseSucc)
2804 FalseValue = FVPN->getIncomingValueForBlock(BB);
2805
2806 // In order for this transformation to be safe, we must be able to
2807 // unconditionally execute both operands to the return. This is
2808 // normally the case, but we could have a potentially-trapping
2809 // constant expression that prevents this transformation from being
2810 // safe.
2811 if (ConstantExpr *TCV = dyn_cast_or_null<ConstantExpr>(TrueValue))
2812 if (TCV->canTrap())
2813 return false;
2814 if (ConstantExpr *FCV = dyn_cast_or_null<ConstantExpr>(FalseValue))
2815 if (FCV->canTrap())
2816 return false;
2817
2818 // Okay, we collected all the mapped values and checked them for sanity, and
2819 // defined to really do this transformation. First, update the CFG.
2820 TrueSucc->removePredecessor(BB);
2821 FalseSucc->removePredecessor(BB);
2822
2823 // Insert select instructions where needed.
2824 Value *BrCond = BI->getCondition();
2825 if (TrueValue) {
2826 // Insert a select if the results differ.
2827 if (TrueValue == FalseValue || isa<UndefValue>(FalseValue)) {
2828 } else if (isa<UndefValue>(TrueValue)) {
2829 TrueValue = FalseValue;
2830 } else {
2831 TrueValue =
2832 Builder.CreateSelect(BrCond, TrueValue, FalseValue, "retval", BI);
2833 }
2834 }
2835
2836 Value *RI =
2837 !TrueValue ? Builder.CreateRetVoid() : Builder.CreateRet(TrueValue);
2838
2839 (void)RI;
2840
2841 LLVM_DEBUG(dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
<< "\n " << *BI << "\nNewRet = " <<
*RI << "\nTRUEBLOCK: " << *TrueSucc << "\nFALSEBLOCK: "
<< *FalseSucc; } } while (false)
2842 << "\n " << *BI << "\nNewRet = " << *RI << "\nTRUEBLOCK: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
<< "\n " << *BI << "\nNewRet = " <<
*RI << "\nTRUEBLOCK: " << *TrueSucc << "\nFALSEBLOCK: "
<< *FalseSucc; } } while (false)
2843 << *TrueSucc << "\nFALSEBLOCK: " << *FalseSucc)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
<< "\n " << *BI << "\nNewRet = " <<
*RI << "\nTRUEBLOCK: " << *TrueSucc << "\nFALSEBLOCK: "
<< *FalseSucc; } } while (false)
;
2844
2845 EraseTerminatorAndDCECond(BI);
2846 if (DTU) {
2847 SmallVector<DominatorTree::UpdateType, 2> Updates;
2848 Updates.push_back({DominatorTree::Delete, BB, TrueSucc});
2849 if (TrueSucc != FalseSucc)
2850 Updates.push_back({DominatorTree::Delete, BB, FalseSucc});
2851 DTU->applyUpdates(Updates);
2852 }
2853
2854 return true;
2855}
2856
2857/// Return true if either PBI or BI has branch weight available, and store
2858/// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does
2859/// not have branch weight, use 1:1 as its weight.
2860static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI,
2861 uint64_t &PredTrueWeight,
2862 uint64_t &PredFalseWeight,
2863 uint64_t &SuccTrueWeight,
2864 uint64_t &SuccFalseWeight) {
2865 bool PredHasWeights =
2866 PBI->extractProfMetadata(PredTrueWeight, PredFalseWeight);
2867 bool SuccHasWeights =
2868 BI->extractProfMetadata(SuccTrueWeight, SuccFalseWeight);
2869 if (PredHasWeights || SuccHasWeights) {
2870 if (!PredHasWeights)
2871 PredTrueWeight = PredFalseWeight = 1;
2872 if (!SuccHasWeights)
2873 SuccTrueWeight = SuccFalseWeight = 1;
2874 return true;
2875 } else {
2876 return false;
2877 }
2878}
2879
2880/// Determine if the two branches share a common destination and deduce a glue
2881/// that joins the branches' conditions to arrive at the common destination if
2882/// that would be profitable.
2883static Optional<std::pair<Instruction::BinaryOps, bool>>
2884shouldFoldCondBranchesToCommonDestination(BranchInst *BI, BranchInst *PBI,
2885 const TargetTransformInfo *TTI) {
2886 assert(BI && PBI && BI->isConditional() && PBI->isConditional() &&((BI && PBI && BI->isConditional() &&
PBI->isConditional() && "Both blocks must end with a conditional branches."
) ? static_cast<void> (0) : __assert_fail ("BI && PBI && BI->isConditional() && PBI->isConditional() && \"Both blocks must end with a conditional branches.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2887, __PRETTY_FUNCTION__))
2887 "Both blocks must end with a conditional branches.")((BI && PBI && BI->isConditional() &&
PBI->isConditional() && "Both blocks must end with a conditional branches."
) ? static_cast<void> (0) : __assert_fail ("BI && PBI && BI->isConditional() && PBI->isConditional() && \"Both blocks must end with a conditional branches.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2887, __PRETTY_FUNCTION__))
;
2888 assert(is_contained(predecessors(BI->getParent()), PBI->getParent()) &&((is_contained(predecessors(BI->getParent()), PBI->getParent
()) && "PredBB must be a predecessor of BB.") ? static_cast
<void> (0) : __assert_fail ("is_contained(predecessors(BI->getParent()), PBI->getParent()) && \"PredBB must be a predecessor of BB.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2889, __PRETTY_FUNCTION__))
2889 "PredBB must be a predecessor of BB.")((is_contained(predecessors(BI->getParent()), PBI->getParent
()) && "PredBB must be a predecessor of BB.") ? static_cast
<void> (0) : __assert_fail ("is_contained(predecessors(BI->getParent()), PBI->getParent()) && \"PredBB must be a predecessor of BB.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 2889, __PRETTY_FUNCTION__))
;
2890
2891 // We have the potential to fold the conditions together, but if the
2892 // predecessor branch is predictable, we may not want to merge them.
2893 uint64_t PTWeight, PFWeight;
2894 BranchProbability PBITrueProb, Likely;
2895 if (TTI && PBI->extractProfMetadata(PTWeight, PFWeight) &&
2896 (PTWeight + PFWeight) != 0) {
2897 PBITrueProb =
2898 BranchProbability::getBranchProbability(PTWeight, PTWeight + PFWeight);
2899 Likely = TTI->getPredictableBranchThreshold();
2900 }
2901
2902 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
2903 // Speculate the 2nd condition unless the 1st is probably true.
2904 if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
2905 return {{Instruction::Or, false}};
2906 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
2907 // Speculate the 2nd condition unless the 1st is probably false.
2908 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
2909 return {{Instruction::And, false}};
2910 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
2911 // Speculate the 2nd condition unless the 1st is probably true.
2912 if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
2913 return {{Instruction::And, true}};
2914 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
2915 // Speculate the 2nd condition unless the 1st is probably false.
2916 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
2917 return {{Instruction::Or, true}};
2918 }
2919 return None;
2920}
2921
2922static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
2923 DomTreeUpdater *DTU,
2924 MemorySSAUpdater *MSSAU,
2925 const TargetTransformInfo *TTI) {
2926 BasicBlock *BB = BI->getParent();
2927 BasicBlock *PredBlock = PBI->getParent();
2928
2929 // Determine if the two branches share a common destination.
2930 Instruction::BinaryOps Opc;
2931 bool InvertPredCond;
2932 std::tie(Opc, InvertPredCond) =
2933 *shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI);
2934
2935 LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING BRANCH TO COMMON DEST:\n"
<< *PBI << *BB; } } while (false)
;
2936
2937 IRBuilder<> Builder(PBI);
2938 // The builder is used to create instructions to eliminate the branch in BB.
2939 // If BB's terminator has !annotation metadata, add it to the new
2940 // instructions.
2941 Builder.CollectMetadataToCopy(BB->getTerminator(),
2942 {LLVMContext::MD_annotation});
2943
2944 // If we need to invert the condition in the pred block to match, do so now.
2945 if (InvertPredCond) {
2946 Value *NewCond = PBI->getCondition();
2947 if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) {
2948 CmpInst *CI = cast<CmpInst>(NewCond);
2949 CI->setPredicate(CI->getInversePredicate());
2950 } else {
2951 NewCond =
2952 Builder.CreateNot(NewCond, PBI->getCondition()->getName() + ".not");
2953 }
2954
2955 PBI->setCondition(NewCond);
2956 PBI->swapSuccessors();
2957 }
2958
2959 BasicBlock *UniqueSucc =
2960 PBI->getSuccessor(0) == BB ? BI->getSuccessor(0) : BI->getSuccessor(1);
2961
2962 // Before cloning instructions, notify the successor basic block that it
2963 // is about to have a new predecessor. This will update PHI nodes,
2964 // which will allow us to update live-out uses of bonus instructions.
2965 AddPredecessorToBlock(UniqueSucc, PredBlock, BB, MSSAU);
2966
2967 // Try to update branch weights.
2968 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
2969 if (extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
2970 SuccTrueWeight, SuccFalseWeight)) {
2971 SmallVector<uint64_t, 8> NewWeights;
2972
2973 if (PBI->getSuccessor(0) == BB) {
2974 // PBI: br i1 %x, BB, FalseDest
2975 // BI: br i1 %y, UniqueSucc, FalseDest
2976 // TrueWeight is TrueWeight for PBI * TrueWeight for BI.
2977 NewWeights.push_back(PredTrueWeight * SuccTrueWeight);
2978 // FalseWeight is FalseWeight for PBI * TotalWeight for BI +
2979 // TrueWeight for PBI * FalseWeight for BI.
2980 // We assume that total weights of a BranchInst can fit into 32 bits.
2981 // Therefore, we will not have overflow using 64-bit arithmetic.
2982 NewWeights.push_back(PredFalseWeight *
2983 (SuccFalseWeight + SuccTrueWeight) +
2984 PredTrueWeight * SuccFalseWeight);
2985 } else {
2986 // PBI: br i1 %x, TrueDest, BB
2987 // BI: br i1 %y, TrueDest, UniqueSucc
2988 // TrueWeight is TrueWeight for PBI * TotalWeight for BI +
2989 // FalseWeight for PBI * TrueWeight for BI.
2990 NewWeights.push_back(PredTrueWeight * (SuccFalseWeight + SuccTrueWeight) +
2991 PredFalseWeight * SuccTrueWeight);
2992 // FalseWeight is FalseWeight for PBI * FalseWeight for BI.
2993 NewWeights.push_back(PredFalseWeight * SuccFalseWeight);
2994 }
2995
2996 // Halve the weights if any of them cannot fit in an uint32_t
2997 FitWeights(NewWeights);
2998
2999 SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), NewWeights.end());
3000 setBranchWeights(PBI, MDWeights[0], MDWeights[1]);
3001
3002 // TODO: If BB is reachable from all paths through PredBlock, then we
3003 // could replace PBI's branch probabilities with BI's.
3004 } else
3005 PBI->setMetadata(LLVMContext::MD_prof, nullptr);
3006
3007 // Now, update the CFG.
3008 PBI->setSuccessor(PBI->getSuccessor(0) != BB, UniqueSucc);
3009
3010 if (DTU)
3011 DTU->applyUpdates({{DominatorTree::Insert, PredBlock, UniqueSucc},
3012 {DominatorTree::Delete, PredBlock, BB}});
3013
3014 // If BI was a loop latch, it may have had associated loop metadata.
3015 // We need to copy it to the new latch, that is, PBI.
3016 if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop))
3017 PBI->setMetadata(LLVMContext::MD_loop, LoopMD);
3018
3019 ValueToValueMapTy VMap; // maps original values to cloned values
3020 CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(BB, PredBlock, VMap);
3021
3022 // Now that the Cond was cloned into the predecessor basic block,
3023 // or/and the two conditions together.
3024 Value *NewCond = nullptr;
3025 Value *BICond = VMap[BI->getCondition()];
3026
3027 if (impliesPoison(BICond, PBI->getCondition()))
3028 NewCond = Builder.CreateBinOp(Opc, PBI->getCondition(), BICond, "or.cond");
3029 else
3030 NewCond =
3031 Opc == Instruction::And
3032 ? Builder.CreateLogicalAnd(PBI->getCondition(), BICond, "or.cond")
3033 : Builder.CreateLogicalOr(PBI->getCondition(), BICond, "or.cond");
3034 PBI->setCondition(NewCond);
3035
3036 // Copy any debug value intrinsics into the end of PredBlock.
3037 for (Instruction &I : *BB) {
3038 if (isa<DbgInfoIntrinsic>(I)) {
3039 Instruction *NewI = I.clone();
3040 RemapInstruction(NewI, VMap,
3041 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
3042 NewI->insertBefore(PBI);
3043 }
3044 }
3045
3046 ++NumFoldBranchToCommonDest;
3047 return true;
3048}
3049
3050/// If this basic block is simple enough, and if a predecessor branches to us
3051/// and one of our successors, fold the block into the predecessor and use
3052/// logical operations to pick the right destination.
3053bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
3054 MemorySSAUpdater *MSSAU,
3055 const TargetTransformInfo *TTI,
3056 unsigned BonusInstThreshold) {
3057 // If this block ends with an unconditional branch,
3058 // let SpeculativelyExecuteBB() deal with it.
3059 if (!BI->isConditional())
3060 return false;
3061
3062 BasicBlock *BB = BI->getParent();
3063
3064 bool Changed = false;
3065
3066 TargetTransformInfo::TargetCostKind CostKind =
3067 BB->getParent()->hasMinSize() ? TargetTransformInfo::TCK_CodeSize
3068 : TargetTransformInfo::TCK_SizeAndLatency;
3069
3070 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
3071
3072 if (!Cond || (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond)) ||
3073 Cond->getParent() != BB || !Cond->hasOneUse())
3074 return Changed;
3075
3076 // Cond is known to be a compare or binary operator. Check to make sure that
3077 // neither operand is a potentially-trapping constant expression.
3078 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(0)))
3079 if (CE->canTrap())
3080 return Changed;
3081 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(1)))
3082 if (CE->canTrap())
3083 return Changed;
3084
3085 // Finally, don't infinitely unroll conditional loops.
3086 if (is_contained(successors(BB), BB))
3087 return Changed;
3088
3089 // With which predecessors will we want to deal with?
3090 SmallVector<BasicBlock *, 8> Preds;
3091 for (BasicBlock *PredBlock : predecessors(BB)) {
3092 BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
3093
3094 // Check that we have two conditional branches. If there is a PHI node in
3095 // the common successor, verify that the same value flows in from both
3096 // blocks.
3097 if (!PBI || PBI->isUnconditional() || !SafeToMergeTerminators(BI, PBI))
3098 continue;
3099
3100 // Determine if the two branches share a common destination.
3101 Instruction::BinaryOps Opc;
3102 bool InvertPredCond;
3103 if (auto Recipe = shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI))
3104 std::tie(Opc, InvertPredCond) = *Recipe;
3105 else
3106 continue;
3107
3108 // Check the cost of inserting the necessary logic before performing the
3109 // transformation.
3110 if (TTI) {
3111 Type *Ty = BI->getCondition()->getType();
3112 InstructionCost Cost = TTI->getArithmeticInstrCost(Opc, Ty, CostKind);
3113 if (InvertPredCond && (!PBI->getCondition()->hasOneUse() ||
3114 !isa<CmpInst>(PBI->getCondition())))
3115 Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind);
3116
3117 if (Cost > BranchFoldThreshold)
3118 continue;
3119 }
3120
3121 // Ok, we do want to deal with this predecessor. Record it.
3122 Preds.emplace_back(PredBlock);
3123 }
3124
3125 // If there aren't any predecessors into which we can fold,
3126 // don't bother checking the cost.
3127 if (Preds.empty())
3128 return Changed;
3129
3130 // Only allow this transformation if computing the condition doesn't involve
3131 // too many instructions and these involved instructions can be executed
3132 // unconditionally. We denote all involved instructions except the condition
3133 // as "bonus instructions", and only allow this transformation when the
3134 // number of the bonus instructions we'll need to create when cloning into
3135 // each predecessor does not exceed a certain threshold.
3136 unsigned NumBonusInsts = 0;
3137 const unsigned PredCount = Preds.size();
3138 for (Instruction &I : *BB) {
3139 // Don't check the branch condition comparison itself.
3140 if (&I == Cond)
3141 continue;
3142 // Ignore dbg intrinsics, and the terminator.
3143 if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I))
3144 continue;
3145 // I must be safe to execute unconditionally.
3146 if (!isSafeToSpeculativelyExecute(&I))
3147 return Changed;
3148
3149 // Account for the cost of duplicating this instruction into each
3150 // predecessor.
3151 NumBonusInsts += PredCount;
3152 // Early exits once we reach the limit.
3153 if (NumBonusInsts > BonusInstThreshold)
3154 return Changed;
3155 }
3156
3157 // Ok, we have the budget. Perform the transformation.
3158 for (BasicBlock *PredBlock : Preds) {
3159 auto *PBI = cast<BranchInst>(PredBlock->getTerminator());
3160 return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI);
3161 }
3162 return Changed;
3163}
3164
3165// If there is only one store in BB1 and BB2, return it, otherwise return
3166// nullptr.
3167static StoreInst *findUniqueStoreInBlocks(BasicBlock *BB1, BasicBlock *BB2) {
3168 StoreInst *S = nullptr;
3169 for (auto *BB : {BB1, BB2}) {
3170 if (!BB)
3171 continue;
3172 for (auto &I : *BB)
3173 if (auto *SI = dyn_cast<StoreInst>(&I)) {
3174 if (S)
3175 // Multiple stores seen.
3176 return nullptr;
3177 else
3178 S = SI;
3179 }
3180 }
3181 return S;
3182}
3183
3184static Value *ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB,
3185 Value *AlternativeV = nullptr) {
3186 // PHI is going to be a PHI node that allows the value V that is defined in
3187 // BB to be referenced in BB's only successor.
3188 //
3189 // If AlternativeV is nullptr, the only value we care about in PHI is V. It
3190 // doesn't matter to us what the other operand is (it'll never get used). We
3191 // could just create a new PHI with an undef incoming value, but that could
3192 // increase register pressure if EarlyCSE/InstCombine can't fold it with some
3193 // other PHI. So here we directly look for some PHI in BB's successor with V
3194 // as an incoming operand. If we find one, we use it, else we create a new
3195 // one.
3196 //
3197 // If AlternativeV is not nullptr, we care about both incoming values in PHI.
3198 // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV]
3199 // where OtherBB is the single other predecessor of BB's only successor.
3200 PHINode *PHI = nullptr;
3201 BasicBlock *Succ = BB->getSingleSuccessor();
3202
3203 for (auto I = Succ->begin(); isa<PHINode>(I); ++I)
3204 if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) {
3205 PHI = cast<PHINode>(I);
3206 if (!AlternativeV)
3207 break;
3208
3209 assert(Succ->hasNPredecessors(2))((Succ->hasNPredecessors(2)) ? static_cast<void> (0)
: __assert_fail ("Succ->hasNPredecessors(2)", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3209, __PRETTY_FUNCTION__))
;
3210 auto PredI = pred_begin(Succ);
3211 BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI;
3212 if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV)
3213 break;
3214 PHI = nullptr;
3215 }
3216 if (PHI)
3217 return PHI;
3218
3219 // If V is not an instruction defined in BB, just return it.
3220 if (!AlternativeV &&
3221 (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB))
3222 return V;
3223
3224 PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge", &Succ->front());
3225 PHI->addIncoming(V, BB);
3226 for (BasicBlock *PredBB : predecessors(Succ))
3227 if (PredBB != BB)
3228 PHI->addIncoming(
3229 AlternativeV ? AlternativeV : UndefValue::get(V->getType()), PredBB);
3230 return PHI;
3231}
3232
3233static bool mergeConditionalStoreToAddress(
3234 BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB,
3235 BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond,
3236 DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI) {
3237 // For every pointer, there must be exactly two stores, one coming from
3238 // PTB or PFB, and the other from QTB or QFB. We don't support more than one
3239 // store (to any address) in PTB,PFB or QTB,QFB.
3240 // FIXME: We could relax this restriction with a bit more work and performance
3241 // testing.
3242 StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB);
3243 StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB);
3244 if (!PStore || !QStore)
3245 return false;
3246
3247 // Now check the stores are compatible.
3248 if (!QStore->isUnordered() || !PStore->isUnordered())
3249 return false;
3250
3251 // Check that sinking the store won't cause program behavior changes. Sinking
3252 // the store out of the Q blocks won't change any behavior as we're sinking
3253 // from a block to its unconditional successor. But we're moving a store from
3254 // the P blocks down through the middle block (QBI) and past both QFB and QTB.
3255 // So we need to check that there are no aliasing loads or stores in
3256 // QBI, QTB and QFB. We also need to check there are no conflicting memory
3257 // operations between PStore and the end of its parent block.
3258 //
3259 // The ideal way to do this is to query AliasAnalysis, but we don't
3260 // preserve AA currently so that is dangerous. Be super safe and just
3261 // check there are no other memory operations at all.
3262 for (auto &I : *QFB->getSinglePredecessor())
3263 if (I.mayReadOrWriteMemory())
3264 return false;
3265 for (auto &I : *QFB)
3266 if (&I != QStore && I.mayReadOrWriteMemory())
3267 return false;
3268 if (QTB)
3269 for (auto &I : *QTB)
3270 if (&I != QStore && I.mayReadOrWriteMemory())
3271 return false;
3272 for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end();
3273 I != E; ++I)
3274 if (&*I != PStore && I->mayReadOrWriteMemory())
3275 return false;
3276
3277 // If we're not in aggressive mode, we only optimize if we have some
3278 // confidence that by optimizing we'll allow P and/or Q to be if-converted.
3279 auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) {
3280 if (!BB)
3281 return true;
3282 // Heuristic: if the block can be if-converted/phi-folded and the
3283 // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
3284 // thread this store.
3285 InstructionCost Cost = 0;
3286 InstructionCost Budget =
3287 PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
3288 for (auto &I : BB->instructionsWithoutDebug()) {
3289 // Consider terminator instruction to be free.
3290 if (I.isTerminator())
3291 continue;
3292 // If this is one the stores that we want to speculate out of this BB,
3293 // then don't count it's cost, consider it to be free.
3294 if (auto *S = dyn_cast<StoreInst>(&I))
3295 if (llvm::find(FreeStores, S))
3296 continue;
3297 // Else, we have a white-list of instructions that we are ak speculating.
3298 if (!isa<BinaryOperator>(I) && !isa<GetElementPtrInst>(I))
3299 return false; // Not in white-list - not worthwhile folding.
3300 // And finally, if this is a non-free instruction that we are okay
3301 // speculating, ensure that we consider the speculation budget.
3302 Cost += TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency);
3303 if (Cost > Budget)
3304 return false; // Eagerly refuse to fold as soon as we're out of budget.
3305 }
3306 assert(Cost <= Budget &&((Cost <= Budget && "When we run out of budget we will eagerly return from within the "
"per-instruction loop.") ? static_cast<void> (0) : __assert_fail
("Cost <= Budget && \"When we run out of budget we will eagerly return from within the \" \"per-instruction loop.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3308, __PRETTY_FUNCTION__))
3307 "When we run out of budget we will eagerly return from within the "((Cost <= Budget && "When we run out of budget we will eagerly return from within the "
"per-instruction loop.") ? static_cast<void> (0) : __assert_fail
("Cost <= Budget && \"When we run out of budget we will eagerly return from within the \" \"per-instruction loop.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3308, __PRETTY_FUNCTION__))
3308 "per-instruction loop.")((Cost <= Budget && "When we run out of budget we will eagerly return from within the "
"per-instruction loop.") ? static_cast<void> (0) : __assert_fail
("Cost <= Budget && \"When we run out of budget we will eagerly return from within the \" \"per-instruction loop.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3308, __PRETTY_FUNCTION__))
;
3309 return true;
3310 };
3311
3312 const std::array<StoreInst *, 2> FreeStores = {PStore, QStore};
3313 if (!MergeCondStoresAggressively &&
3314 (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) ||
3315 !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores)))
3316 return false;
3317
3318 // If PostBB has more than two predecessors, we need to split it so we can
3319 // sink the store.
3320 if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) {
3321 // We know that QFB's only successor is PostBB. And QFB has a single
3322 // predecessor. If QTB exists, then its only successor is also PostBB.
3323 // If QTB does not exist, then QFB's only predecessor has a conditional
3324 // branch to QFB and PostBB.
3325 BasicBlock *TruePred = QTB ? QTB : QFB->getSinglePredecessor();
3326 BasicBlock *NewBB =
3327 SplitBlockPredecessors(PostBB, {QFB, TruePred}, "condstore.split", DTU);
3328 if (!NewBB)
3329 return false;
3330 PostBB = NewBB;
3331 }
3332
3333 // OK, we're going to sink the stores to PostBB. The store has to be
3334 // conditional though, so first create the predicate.
3335 Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator())
3336 ->getCondition();
3337 Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator())
3338 ->getCondition();
3339
3340 Value *PPHI = ensureValueAvailableInSuccessor(PStore->getValueOperand(),
3341 PStore->getParent());
3342 Value *QPHI = ensureValueAvailableInSuccessor(QStore->getValueOperand(),
3343 QStore->getParent(), PPHI);
3344
3345 IRBuilder<> QB(&*PostBB->getFirstInsertionPt());
3346
3347 Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond);
3348 Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond);
3349
3350 if (InvertPCond)
3351 PPred = QB.CreateNot(PPred);
3352 if (InvertQCond)
3353 QPred = QB.CreateNot(QPred);
3354 Value *CombinedPred = QB.CreateOr(PPred, QPred);
3355
3356 auto *T = SplitBlockAndInsertIfThen(CombinedPred, &*QB.GetInsertPoint(),
3357 /*Unreachable=*/false,
3358 /*BranchWeights=*/nullptr, DTU);
3359 QB.SetInsertPoint(T);
3360 StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address));
3361 AAMDNodes AAMD;
3362 PStore->getAAMetadata(AAMD, /*Merge=*/false);
3363 PStore->getAAMetadata(AAMD, /*Merge=*/true);
3364 SI->setAAMetadata(AAMD);
3365 // Choose the minimum alignment. If we could prove both stores execute, we
3366 // could use biggest one. In this case, though, we only know that one of the
3367 // stores executes. And we don't know it's safe to take the alignment from a
3368 // store that doesn't execute.
3369 SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign()));
3370
3371 QStore->eraseFromParent();
3372 PStore->eraseFromParent();
3373
3374 return true;
3375}
3376
3377static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI,
3378 DomTreeUpdater *DTU, const DataLayout &DL,
3379 const TargetTransformInfo &TTI) {
3380 // The intention here is to find diamonds or triangles (see below) where each
3381 // conditional block contains a store to the same address. Both of these
3382 // stores are conditional, so they can't be unconditionally sunk. But it may
3383 // be profitable to speculatively sink the stores into one merged store at the
3384 // end, and predicate the merged store on the union of the two conditions of
3385 // PBI and QBI.
3386 //
3387 // This can reduce the number of stores executed if both of the conditions are
3388 // true, and can allow the blocks to become small enough to be if-converted.
3389 // This optimization will also chain, so that ladders of test-and-set
3390 // sequences can be if-converted away.
3391 //
3392 // We only deal with simple diamonds or triangles:
3393 //
3394 // PBI or PBI or a combination of the two
3395 // / \ | \
3396 // PTB PFB | PFB
3397 // \ / | /
3398 // QBI QBI
3399 // / \ | \
3400 // QTB QFB | QFB
3401 // \ / | /
3402 // PostBB PostBB
3403 //
3404 // We model triangles as a type of diamond with a nullptr "true" block.
3405 // Triangles are canonicalized so that the fallthrough edge is represented by
3406 // a true condition, as in the diagram above.
3407 BasicBlock *PTB = PBI->getSuccessor(0);
3408 BasicBlock *PFB = PBI->getSuccessor(1);
3409 BasicBlock *QTB = QBI->getSuccessor(0);
3410 BasicBlock *QFB = QBI->getSuccessor(1);
3411 BasicBlock *PostBB = QFB->getSingleSuccessor();
3412
3413 // Make sure we have a good guess for PostBB. If QTB's only successor is
3414 // QFB, then QFB is a better PostBB.
3415 if (QTB->getSingleSuccessor() == QFB)
3416 PostBB = QFB;
3417
3418 // If we couldn't find a good PostBB, stop.
3419 if (!PostBB)
3420 return false;
3421
3422 bool InvertPCond = false, InvertQCond = false;
3423 // Canonicalize fallthroughs to the true branches.
3424 if (PFB == QBI->getParent()) {
3425 std::swap(PFB, PTB);
3426 InvertPCond = true;
3427 }
3428 if (QFB == PostBB) {
3429 std::swap(QFB, QTB);
3430 InvertQCond = true;
3431 }
3432
3433 // From this point on we can assume PTB or QTB may be fallthroughs but PFB
3434 // and QFB may not. Model fallthroughs as a nullptr block.
3435 if (PTB == QBI->getParent())
3436 PTB = nullptr;
3437 if (QTB == PostBB)
3438 QTB = nullptr;
3439
3440 // Legality bailouts. We must have at least the non-fallthrough blocks and
3441 // the post-dominating block, and the non-fallthroughs must only have one
3442 // predecessor.
3443 auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) {
3444 return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S;
3445 };
3446 if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) ||
3447 !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB))
3448 return false;
3449 if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) ||
3450 (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB)))
3451 return false;
3452 if (!QBI->getParent()->hasNUses(2))
3453 return false;
3454
3455 // OK, this is a sequence of two diamonds or triangles.
3456 // Check if there are stores in PTB or PFB that are repeated in QTB or QFB.
3457 SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses;
3458 for (auto *BB : {PTB, PFB}) {
3459 if (!BB)
3460 continue;
3461 for (auto &I : *BB)
3462 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3463 PStoreAddresses.insert(SI->getPointerOperand());
3464 }
3465 for (auto *BB : {QTB, QFB}) {
3466 if (!BB)
3467 continue;
3468 for (auto &I : *BB)
3469 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3470 QStoreAddresses.insert(SI->getPointerOperand());
3471 }
3472
3473 set_intersect(PStoreAddresses, QStoreAddresses);
3474 // set_intersect mutates PStoreAddresses in place. Rename it here to make it
3475 // clear what it contains.
3476 auto &CommonAddresses = PStoreAddresses;
3477
3478 bool Changed = false;
3479 for (auto *Address : CommonAddresses)
3480 Changed |=
3481 mergeConditionalStoreToAddress(PTB, PFB, QTB, QFB, PostBB, Address,
3482 InvertPCond, InvertQCond, DTU, DL, TTI);
3483 return Changed;
3484}
3485
3486/// If the previous block ended with a widenable branch, determine if reusing
3487/// the target block is profitable and legal. This will have the effect of
3488/// "widening" PBI, but doesn't require us to reason about hosting safety.
3489static bool tryWidenCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
3490 DomTreeUpdater *DTU) {
3491 // TODO: This can be generalized in two important ways:
3492 // 1) We can allow phi nodes in IfFalseBB and simply reuse all the input
3493 // values from the PBI edge.
3494 // 2) We can sink side effecting instructions into BI's fallthrough
3495 // successor provided they doesn't contribute to computation of
3496 // BI's condition.
3497 Value *CondWB, *WC;
3498 BasicBlock *IfTrueBB, *IfFalseBB;
3499 if (!parseWidenableBranch(PBI, CondWB, WC, IfTrueBB, IfFalseBB) ||
3500 IfTrueBB != BI->getParent() || !BI->getParent()->getSinglePredecessor())
3501 return false;
3502 if (!IfFalseBB->phis().empty())
3503 return false; // TODO
3504 // Use lambda to lazily compute expensive condition after cheap ones.
3505 auto NoSideEffects = [](BasicBlock &BB) {
3506 return !llvm::any_of(BB, [](const Instruction &I) {
3507 return I.mayWriteToMemory() || I.mayHaveSideEffects();
3508 });
3509 };
3510 if (BI->getSuccessor(1) != IfFalseBB && // no inf looping
3511 BI->getSuccessor(1)->getTerminatingDeoptimizeCall() && // profitability
3512 NoSideEffects(*BI->getParent())) {
3513 auto *OldSuccessor = BI->getSuccessor(1);
3514 OldSuccessor->removePredecessor(BI->getParent());
3515 BI->setSuccessor(1, IfFalseBB);
3516 if (DTU)
3517 DTU->applyUpdates(
3518 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
3519 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
3520 return true;
3521 }
3522 if (BI->getSuccessor(0) != IfFalseBB && // no inf looping
3523 BI->getSuccessor(0)->getTerminatingDeoptimizeCall() && // profitability
3524 NoSideEffects(*BI->getParent())) {
3525 auto *OldSuccessor = BI->getSuccessor(0);
3526 OldSuccessor->removePredecessor(BI->getParent());
3527 BI->setSuccessor(0, IfFalseBB);
3528 if (DTU)
3529 DTU->applyUpdates(
3530 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
3531 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
3532 return true;
3533 }
3534 return false;
3535}
3536
3537/// If we have a conditional branch as a predecessor of another block,
3538/// this function tries to simplify it. We know
3539/// that PBI and BI are both conditional branches, and BI is in one of the
3540/// successor blocks of PBI - PBI branches to BI.
3541static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
3542 DomTreeUpdater *DTU,
3543 const DataLayout &DL,
3544 const TargetTransformInfo &TTI) {
3545 assert(PBI->isConditional() && BI->isConditional())((PBI->isConditional() && BI->isConditional()) ?
static_cast<void> (0) : __assert_fail ("PBI->isConditional() && BI->isConditional()"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3545, __PRETTY_FUNCTION__))
;
3546 BasicBlock *BB = BI->getParent();
3547
3548 // If this block ends with a branch instruction, and if there is a
3549 // predecessor that ends on a branch of the same condition, make
3550 // this conditional branch redundant.
3551 if (PBI->getCondition() == BI->getCondition() &&
3552 PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3553 // Okay, the outcome of this conditional branch is statically
3554 // knowable. If this block had a single pred, handle specially.
3555 if (BB->getSinglePredecessor()) {
3556 // Turn this into a branch on constant.
3557 bool CondIsTrue = PBI->getSuccessor(0) == BB;
3558 BI->setCondition(
3559 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue));
3560 return true; // Nuke the branch on constant.
3561 }
3562
3563 // Otherwise, if there are multiple predecessors, insert a PHI that merges
3564 // in the constant and simplify the block result. Subsequent passes of
3565 // simplifycfg will thread the block.
3566 if (BlockIsSimpleEnoughToThreadThrough(BB)) {
3567 pred_iterator PB = pred_begin(BB), PE = pred_end(BB);
3568 PHINode *NewPN = PHINode::Create(
3569 Type::getInt1Ty(BB->getContext()), std::distance(PB, PE),
3570 BI->getCondition()->getName() + ".pr", &BB->front());
3571 // Okay, we're going to insert the PHI node. Since PBI is not the only
3572 // predecessor, compute the PHI'd conditional value for all of the preds.
3573 // Any predecessor where the condition is not computable we keep symbolic.
3574 for (pred_iterator PI = PB; PI != PE; ++PI) {
3575 BasicBlock *P = *PI;
3576 if ((PBI = dyn_cast<BranchInst>(P->getTerminator())) && PBI != BI &&
3577 PBI->isConditional() && PBI->getCondition() == BI->getCondition() &&
3578 PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3579 bool CondIsTrue = PBI->getSuccessor(0) == BB;
3580 NewPN->addIncoming(
3581 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue),
3582 P);
3583 } else {
3584 NewPN->addIncoming(BI->getCondition(), P);
3585 }
3586 }
3587
3588 BI->setCondition(NewPN);
3589 return true;
3590 }
3591 }
3592
3593 // If the previous block ended with a widenable branch, determine if reusing
3594 // the target block is profitable and legal. This will have the effect of
3595 // "widening" PBI, but doesn't require us to reason about hosting safety.
3596 if (tryWidenCondBranchToCondBranch(PBI, BI, DTU))
3597 return true;
3598
3599 if (auto *CE = dyn_cast<ConstantExpr>(BI->getCondition()))
3600 if (CE->canTrap())
3601 return false;
3602
3603 // If both branches are conditional and both contain stores to the same
3604 // address, remove the stores from the conditionals and create a conditional
3605 // merged store at the end.
3606 if (MergeCondStores && mergeConditionalStores(PBI, BI, DTU, DL, TTI))
3607 return true;
3608
3609 // If this is a conditional branch in an empty block, and if any
3610 // predecessors are a conditional branch to one of our destinations,
3611 // fold the conditions into logical ops and one cond br.
3612
3613 // Ignore dbg intrinsics.
3614 if (&*BB->instructionsWithoutDebug().begin() != BI)
3615 return false;
3616
3617 int PBIOp, BIOp;
3618 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
3619 PBIOp = 0;
3620 BIOp = 0;
3621 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
3622 PBIOp = 0;
3623 BIOp = 1;
3624 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
3625 PBIOp = 1;
3626 BIOp = 0;
3627 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
3628 PBIOp = 1;
3629 BIOp = 1;
3630 } else {
3631 return false;
3632 }
3633
3634 // Check to make sure that the other destination of this branch
3635 // isn't BB itself. If so, this is an infinite loop that will
3636 // keep getting unwound.
3637 if (PBI->getSuccessor(PBIOp) == BB)
3638 return false;
3639
3640 // Do not perform this transformation if it would require
3641 // insertion of a large number of select instructions. For targets
3642 // without predication/cmovs, this is a big pessimization.
3643
3644 // Also do not perform this transformation if any phi node in the common
3645 // destination block can trap when reached by BB or PBB (PR17073). In that
3646 // case, it would be unsafe to hoist the operation into a select instruction.
3647
3648 BasicBlock *CommonDest = PBI->getSuccessor(PBIOp);
3649 BasicBlock *RemovedDest = PBI->getSuccessor(PBIOp ^ 1);
3650 unsigned NumPhis = 0;
3651 for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II);
3652 ++II, ++NumPhis) {
3653 if (NumPhis > 2) // Disable this xform.
3654 return false;
3655
3656 PHINode *PN = cast<PHINode>(II);
3657 Value *BIV = PN->getIncomingValueForBlock(BB);
3658 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(BIV))
3659 if (CE->canTrap())
3660 return false;
3661
3662 unsigned PBBIdx = PN->getBasicBlockIndex(PBI->getParent());
3663 Value *PBIV = PN->getIncomingValue(PBBIdx);
3664 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(PBIV))
3665 if (CE->canTrap())
3666 return false;
3667 }
3668
3669 // Finally, if everything is ok, fold the branches to logical ops.
3670 BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1);
3671
3672 LLVM_DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING BRs:" << *PBI
->getParent() << "AND: " << *BI->getParent(
); } } while (false)
3673 << "AND: " << *BI->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING BRs:" << *PBI
->getParent() << "AND: " << *BI->getParent(
); } } while (false)
;
3674
3675 SmallVector<DominatorTree::UpdateType, 5> Updates;
3676
3677 // If OtherDest *is* BB, then BB is a basic block with a single conditional
3678 // branch in it, where one edge (OtherDest) goes back to itself but the other
3679 // exits. We don't *know* that the program avoids the infinite loop
3680 // (even though that seems likely). If we do this xform naively, we'll end up
3681 // recursively unpeeling the loop. Since we know that (after the xform is
3682 // done) that the block *is* infinite if reached, we just make it an obviously
3683 // infinite loop with no cond branch.
3684 if (OtherDest == BB) {
3685 // Insert it at the end of the function, because it's either code,
3686 // or it won't matter if it's hot. :)
3687 BasicBlock *InfLoopBlock =
3688 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
3689 BranchInst::Create(InfLoopBlock, InfLoopBlock);
3690 if (DTU)
3691 Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
3692 OtherDest = InfLoopBlock;
3693 }
3694
3695 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << *PBI->getParent()->getParent
(); } } while (false)
;
3696
3697 // BI may have other predecessors. Because of this, we leave
3698 // it alone, but modify PBI.
3699
3700 // Make sure we get to CommonDest on True&True directions.
3701 Value *PBICond = PBI->getCondition();
3702 IRBuilder<NoFolder> Builder(PBI);
3703 if (PBIOp)
3704 PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not");
3705
3706 Value *BICond = BI->getCondition();
3707 if (BIOp)
3708 BICond = Builder.CreateNot(BICond, BICond->getName() + ".not");
3709
3710 // Merge the conditions.
3711 Value *Cond = Builder.CreateOr(PBICond, BICond, "brmerge");
3712
3713 // Modify PBI to branch on the new condition to the new dests.
3714 PBI->setCondition(Cond);
3715 PBI->setSuccessor(0, CommonDest);
3716 PBI->setSuccessor(1, OtherDest);
3717
3718 if (DTU) {
3719 Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest});
3720 Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest});
3721
3722 DTU->applyUpdates(Updates);
3723 }
3724
3725 // Update branch weight for PBI.
3726 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
3727 uint64_t PredCommon, PredOther, SuccCommon, SuccOther;
3728 bool HasWeights =
3729 extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
3730 SuccTrueWeight, SuccFalseWeight);
3731 if (HasWeights) {
3732 PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3733 PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3734 SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3735 SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3736 // The weight to CommonDest should be PredCommon * SuccTotal +
3737 // PredOther * SuccCommon.
3738 // The weight to OtherDest should be PredOther * SuccOther.
3739 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) +
3740 PredOther * SuccCommon,
3741 PredOther * SuccOther};
3742 // Halve the weights if any of them cannot fit in an uint32_t
3743 FitWeights(NewWeights);
3744
3745 setBranchWeights(PBI, NewWeights[0], NewWeights[1]);
3746 }
3747
3748 // OtherDest may have phi nodes. If so, add an entry from PBI's
3749 // block that are identical to the entries for BI's block.
3750 AddPredecessorToBlock(OtherDest, PBI->getParent(), BB);
3751
3752 // We know that the CommonDest already had an edge from PBI to
3753 // it. If it has PHIs though, the PHIs may have different
3754 // entries for BB and PBI's BB. If so, insert a select to make
3755 // them agree.
3756 for (PHINode &PN : CommonDest->phis()) {
3757 Value *BIV = PN.getIncomingValueForBlock(BB);
3758 unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent());
3759 Value *PBIV = PN.getIncomingValue(PBBIdx);
3760 if (BIV != PBIV) {
3761 // Insert a select in PBI to pick the right value.
3762 SelectInst *NV = cast<SelectInst>(
3763 Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux"));
3764 PN.setIncomingValue(PBBIdx, NV);
3765 // Although the select has the same condition as PBI, the original branch
3766 // weights for PBI do not apply to the new select because the select's
3767 // 'logical' edges are incoming edges of the phi that is eliminated, not
3768 // the outgoing edges of PBI.
3769 if (HasWeights) {
3770 uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3771 uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3772 uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3773 uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3774 // The weight to PredCommonDest should be PredCommon * SuccTotal.
3775 // The weight to PredOtherDest should be PredOther * SuccCommon.
3776 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther),
3777 PredOther * SuccCommon};
3778
3779 FitWeights(NewWeights);
3780
3781 setBranchWeights(NV, NewWeights[0], NewWeights[1]);
3782 }
3783 }
3784 }
3785
3786 LLVM_DEBUG(dbgs() << "INTO: " << *PBI->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "INTO: " << *PBI->
getParent(); } } while (false)
;
3787 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << *PBI->getParent()->getParent
(); } } while (false)
;
3788
3789 // This basic block is probably dead. We know it has at least
3790 // one fewer predecessor.
3791 return true;
3792}
3793
3794// Simplifies a terminator by replacing it with a branch to TrueBB if Cond is
3795// true or to FalseBB if Cond is false.
3796// Takes care of updating the successors and removing the old terminator.
3797// Also makes sure not to introduce new successors by assuming that edges to
3798// non-successor TrueBBs and FalseBBs aren't reachable.
3799bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm,
3800 Value *Cond, BasicBlock *TrueBB,
3801 BasicBlock *FalseBB,
3802 uint32_t TrueWeight,
3803 uint32_t FalseWeight) {
3804 auto *BB = OldTerm->getParent();
3805 // Remove any superfluous successor edges from the CFG.
3806 // First, figure out which successors to preserve.
3807 // If TrueBB and FalseBB are equal, only try to preserve one copy of that
3808 // successor.
3809 BasicBlock *KeepEdge1 = TrueBB;
3810 BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr;
3811
3812 SmallPtrSet<BasicBlock *, 2> RemovedSuccessors;
3813
3814 // Then remove the rest.
3815 for (BasicBlock *Succ : successors(OldTerm)) {
3816 // Make sure only to keep exactly one copy of each edge.
3817 if (Succ == KeepEdge1)
3818 KeepEdge1 = nullptr;
3819 else if (Succ == KeepEdge2)
3820 KeepEdge2 = nullptr;
3821 else {
3822 Succ->removePredecessor(BB,
3823 /*KeepOneInputPHIs=*/true);
3824
3825 if (Succ != TrueBB && Succ != FalseBB)
3826 RemovedSuccessors.insert(Succ);
3827 }
3828 }
3829
3830 IRBuilder<> Builder(OldTerm);
3831 Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc());
3832
3833 // Insert an appropriate new terminator.
3834 if (!KeepEdge1 && !KeepEdge2) {
3835 if (TrueBB == FalseBB) {
3836 // We were only looking for one successor, and it was present.
3837 // Create an unconditional branch to it.
3838 Builder.CreateBr(TrueBB);
3839 } else {
3840 // We found both of the successors we were looking for.
3841 // Create a conditional branch sharing the condition of the select.
3842 BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
3843 if (TrueWeight != FalseWeight)
3844 setBranchWeights(NewBI, TrueWeight, FalseWeight);
3845 }
3846 } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
3847 // Neither of the selected blocks were successors, so this
3848 // terminator must be unreachable.
3849 new UnreachableInst(OldTerm->getContext(), OldTerm);
3850 } else {
3851 // One of the selected values was a successor, but the other wasn't.
3852 // Insert an unconditional branch to the one that was found;
3853 // the edge to the one that wasn't must be unreachable.
3854 if (!KeepEdge1) {
3855 // Only TrueBB was found.
3856 Builder.CreateBr(TrueBB);
3857 } else {
3858 // Only FalseBB was found.
3859 Builder.CreateBr(FalseBB);
3860 }
3861 }
3862
3863 EraseTerminatorAndDCECond(OldTerm);
3864
3865 if (DTU) {
3866 SmallVector<DominatorTree::UpdateType, 2> Updates;
3867 Updates.reserve(RemovedSuccessors.size());
3868 for (auto *RemovedSuccessor : RemovedSuccessors)
3869 Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor});
3870 DTU->applyUpdates(Updates);
3871 }
3872
3873 return true;
3874}
3875
3876// Replaces
3877// (switch (select cond, X, Y)) on constant X, Y
3878// with a branch - conditional if X and Y lead to distinct BBs,
3879// unconditional otherwise.
3880bool SimplifyCFGOpt::SimplifySwitchOnSelect(SwitchInst *SI,
3881 SelectInst *Select) {
3882 // Check for constant integer values in the select.
3883 ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue());
3884 ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue());
3885 if (!TrueVal || !FalseVal)
3886 return false;
3887
3888 // Find the relevant condition and destinations.
3889 Value *Condition = Select->getCondition();
3890 BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor();
3891 BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor();
3892
3893 // Get weight for TrueBB and FalseBB.
3894 uint32_t TrueWeight = 0, FalseWeight = 0;
3895 SmallVector<uint64_t, 8> Weights;
3896 bool HasWeights = HasBranchWeights(SI);
3897 if (HasWeights) {
3898 GetBranchWeights(SI, Weights);
3899 if (Weights.size() == 1 + SI->getNumCases()) {
3900 TrueWeight =
3901 (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()];
3902 FalseWeight =
3903 (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()];
3904 }
3905 }
3906
3907 // Perform the actual simplification.
3908 return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight,
3909 FalseWeight);
3910}
3911
3912// Replaces
3913// (indirectbr (select cond, blockaddress(@fn, BlockA),
3914// blockaddress(@fn, BlockB)))
3915// with
3916// (br cond, BlockA, BlockB).
3917bool SimplifyCFGOpt::SimplifyIndirectBrOnSelect(IndirectBrInst *IBI,
3918 SelectInst *SI) {
3919 // Check that both operands of the select are block addresses.
3920 BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue());
3921 BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue());
3922 if (!TBA || !FBA)
3923 return false;
3924
3925 // Extract the actual blocks.
3926 BasicBlock *TrueBB = TBA->getBasicBlock();
3927 BasicBlock *FalseBB = FBA->getBasicBlock();
3928
3929 // Perform the actual simplification.
3930 return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0,
3931 0);
3932}
3933
3934/// This is called when we find an icmp instruction
3935/// (a seteq/setne with a constant) as the only instruction in a
3936/// block that ends with an uncond branch. We are looking for a very specific
3937/// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In
3938/// this case, we merge the first two "or's of icmp" into a switch, but then the
3939/// default value goes to an uncond block with a seteq in it, we get something
3940/// like:
3941///
3942/// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ]
3943/// DEFAULT:
3944/// %tmp = icmp eq i8 %A, 92
3945/// br label %end
3946/// end:
3947/// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ]
3948///
3949/// We prefer to split the edge to 'end' so that there is a true/false entry to
3950/// the PHI, merging the third icmp into the switch.
3951bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt(
3952 ICmpInst *ICI, IRBuilder<> &Builder) {
3953 BasicBlock *BB = ICI->getParent();
3954
3955 // If the block has any PHIs in it or the icmp has multiple uses, it is too
3956 // complex.
3957 if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse())
3958 return false;
3959
3960 Value *V = ICI->getOperand(0);
3961 ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1));
3962
3963 // The pattern we're looking for is where our only predecessor is a switch on
3964 // 'V' and this block is the default case for the switch. In this case we can
3965 // fold the compared value into the switch to simplify things.
3966 BasicBlock *Pred = BB->getSinglePredecessor();
3967 if (!Pred || !isa<SwitchInst>(Pred->getTerminator()))
3968 return false;
3969
3970 SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator());
3971 if (SI->getCondition() != V)
3972 return false;
3973
3974 // If BB is reachable on a non-default case, then we simply know the value of
3975 // V in this block. Substitute it and constant fold the icmp instruction
3976 // away.
3977 if (SI->getDefaultDest() != BB) {
3978 ConstantInt *VVal = SI->findCaseDest(BB);
3979 assert(VVal && "Should have a unique destination value")((VVal && "Should have a unique destination value") ?
static_cast<void> (0) : __assert_fail ("VVal && \"Should have a unique destination value\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 3979, __PRETTY_FUNCTION__))
;
3980 ICI->setOperand(0, VVal);
3981
3982 if (Value *V = SimplifyInstruction(ICI, {DL, ICI})) {
3983 ICI->replaceAllUsesWith(V);
3984 ICI->eraseFromParent();
3985 }
3986 // BB is now empty, so it is likely to simplify away.
3987 return requestResimplify();
3988 }
3989
3990 // Ok, the block is reachable from the default dest. If the constant we're
3991 // comparing exists in one of the other edges, then we can constant fold ICI
3992 // and zap it.
3993 if (SI->findCaseValue(Cst) != SI->case_default()) {
3994 Value *V;
3995 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
3996 V = ConstantInt::getFalse(BB->getContext());
3997 else
3998 V = ConstantInt::getTrue(BB->getContext());
3999
4000 ICI->replaceAllUsesWith(V);
4001 ICI->eraseFromParent();
4002 // BB is now empty, so it is likely to simplify away.
4003 return requestResimplify();
4004 }
4005
4006 // The use of the icmp has to be in the 'end' block, by the only PHI node in
4007 // the block.
4008 BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0);
4009 PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back());
4010 if (PHIUse == nullptr || PHIUse != &SuccBlock->front() ||
4011 isa<PHINode>(++BasicBlock::iterator(PHIUse)))
4012 return false;
4013
4014 // If the icmp is a SETEQ, then the default dest gets false, the new edge gets
4015 // true in the PHI.
4016 Constant *DefaultCst = ConstantInt::getTrue(BB->getContext());
4017 Constant *NewCst = ConstantInt::getFalse(BB->getContext());
4018
4019 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
4020 std::swap(DefaultCst, NewCst);
4021
4022 // Replace ICI (which is used by the PHI for the default value) with true or
4023 // false depending on if it is EQ or NE.
4024 ICI->replaceAllUsesWith(DefaultCst);
4025 ICI->eraseFromParent();
4026
4027 SmallVector<DominatorTree::UpdateType, 2> Updates;
4028
4029 // Okay, the switch goes to this block on a default value. Add an edge from
4030 // the switch to the merge point on the compared value.
4031 BasicBlock *NewBB =
4032 BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB);
4033 {
4034 SwitchInstProfUpdateWrapper SIW(*SI);
4035 auto W0 = SIW.getSuccessorWeight(0);
4036 SwitchInstProfUpdateWrapper::CaseWeightOpt NewW;
4037 if (W0) {
4038 NewW = ((uint64_t(*W0) + 1) >> 1);
4039 SIW.setSuccessorWeight(0, *NewW);
4040 }
4041 SIW.addCase(Cst, NewBB, NewW);
4042 if (DTU)
4043 Updates.push_back({DominatorTree::Insert, Pred, NewBB});
4044 }
4045
4046 // NewBB branches to the phi block, add the uncond branch and the phi entry.
4047 Builder.SetInsertPoint(NewBB);
4048 Builder.SetCurrentDebugLocation(SI->getDebugLoc());
4049 Builder.CreateBr(SuccBlock);
4050 PHIUse->addIncoming(NewCst, NewBB);
4051 if (DTU) {
4052 Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock});
4053 DTU->applyUpdates(Updates);
4054 }
4055 return true;
4056}
4057
4058/// The specified branch is a conditional branch.
4059/// Check to see if it is branching on an or/and chain of icmp instructions, and
4060/// fold it into a switch instruction if so.
4061bool SimplifyCFGOpt::SimplifyBranchOnICmpChain(BranchInst *BI,
4062 IRBuilder<> &Builder,
4063 const DataLayout &DL) {
4064 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
4065 if (!Cond)
4066 return false;
4067
4068 // Change br (X == 0 | X == 1), T, F into a switch instruction.
4069 // If this is a bunch of seteq's or'd together, or if it's a bunch of
4070 // 'setne's and'ed together, collect them.
4071
4072 // Try to gather values from a chain of and/or to be turned into a switch
4073 ConstantComparesGatherer ConstantCompare(Cond, DL);
4074 // Unpack the result
4075 SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals;
4076 Value *CompVal = ConstantCompare.CompValue;
4077 unsigned UsedICmps = ConstantCompare.UsedICmps;
4078 Value *ExtraCase = ConstantCompare.Extra;
4079
4080 // If we didn't have a multiply compared value, fail.
4081 if (!CompVal)
4082 return false;
4083
4084 // Avoid turning single icmps into a switch.
4085 if (UsedICmps <= 1)
4086 return false;
4087
4088 bool TrueWhenEqual = match(Cond, m_LogicalOr(m_Value(), m_Value()));
4089
4090 // There might be duplicate constants in the list, which the switch
4091 // instruction can't handle, remove them now.
4092 array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate);
4093 Values.erase(std::unique(Values.begin(), Values.end()), Values.end());
4094
4095 // If Extra was used, we require at least two switch values to do the
4096 // transformation. A switch with one value is just a conditional branch.
4097 if (ExtraCase && Values.size() < 2)
4098 return false;
4099
4100 // TODO: Preserve branch weight metadata, similarly to how
4101 // FoldValueComparisonIntoPredecessors preserves it.
4102
4103 // Figure out which block is which destination.
4104 BasicBlock *DefaultBB = BI->getSuccessor(1);
4105 BasicBlock *EdgeBB = BI->getSuccessor(0);
4106 if (!TrueWhenEqual)
4107 std::swap(DefaultBB, EdgeBB);
4108
4109 BasicBlock *BB = BI->getParent();
4110
4111 // MSAN does not like undefs as branch condition which can be introduced
4112 // with "explicit branch".
4113 if (ExtraCase && BB->getParent()->hasFnAttribute(Attribute::SanitizeMemory))
4114 return false;
4115
4116 LLVM_DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Converting 'icmp' chain with "
<< Values.size() << " cases into SWITCH. BB is:\n"
<< *BB; } } while (false)
4117 << " cases into SWITCH. BB is:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Converting 'icmp' chain with "
<< Values.size() << " cases into SWITCH. BB is:\n"
<< *BB; } } while (false)
4118 << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "Converting 'icmp' chain with "
<< Values.size() << " cases into SWITCH. BB is:\n"
<< *BB; } } while (false)
;
4119
4120 SmallVector<DominatorTree::UpdateType, 2> Updates;
4121
4122 // If there are any extra values that couldn't be folded into the switch
4123 // then we evaluate them with an explicit branch first. Split the block
4124 // right before the condbr to handle it.
4125 if (ExtraCase) {
4126 BasicBlock *NewBB = SplitBlock(BB, BI, DTU, /*LI=*/nullptr,
4127 /*MSSAU=*/nullptr, "switch.early.test");
4128
4129 // Remove the uncond branch added to the old block.
4130 Instruction *OldTI = BB->getTerminator();
4131 Builder.SetInsertPoint(OldTI);
4132
4133 if (TrueWhenEqual)
4134 Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB);
4135 else
4136 Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB);
4137
4138 OldTI->eraseFromParent();
4139
4140 if (DTU)
4141 Updates.push_back({DominatorTree::Insert, BB, EdgeBB});
4142
4143 // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
4144 // for the edge we just added.
4145 AddPredecessorToBlock(EdgeBB, BB, NewBB);
4146
4147 LLVM_DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCasedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << " ** 'icmp' chain unhandled condition: "
<< *ExtraCase << "\nEXTRABB = " << *BB; } }
while (false)
4148 << "\nEXTRABB = " << *BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << " ** 'icmp' chain unhandled condition: "
<< *ExtraCase << "\nEXTRABB = " << *BB; } }
while (false)
;
4149 BB = NewBB;
4150 }
4151
4152 Builder.SetInsertPoint(BI);
4153 // Convert pointer to int before we switch.
4154 if (CompVal->getType()->isPointerTy()) {
4155 CompVal = Builder.CreatePtrToInt(
4156 CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr");
4157 }
4158
4159 // Create the new switch instruction now.
4160 SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size());
4161
4162 // Add all of the 'cases' to the switch instruction.
4163 for (unsigned i = 0, e = Values.size(); i != e; ++i)
4164 New->addCase(Values[i], EdgeBB);
4165
4166 // We added edges from PI to the EdgeBB. As such, if there were any
4167 // PHI nodes in EdgeBB, they need entries to be added corresponding to
4168 // the number of edges added.
4169 for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) {
4170 PHINode *PN = cast<PHINode>(BBI);
4171 Value *InVal = PN->getIncomingValueForBlock(BB);
4172 for (unsigned i = 0, e = Values.size() - 1; i != e; ++i)
4173 PN->addIncoming(InVal, BB);
4174 }
4175
4176 // Erase the old branch instruction.
4177 EraseTerminatorAndDCECond(BI);
4178 if (DTU)
4179 DTU->applyUpdates(Updates);
4180
4181 LLVM_DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << " ** 'icmp' chain result is:\n"
<< *BB << '\n'; } } while (false)
;
4182 return true;
4183}
4184
4185bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
4186 if (isa<PHINode>(RI->getValue()))
19
Assuming the object is a 'PHINode'
20
Taking true branch
4187 return simplifyCommonResume(RI);
21
Calling 'SimplifyCFGOpt::simplifyCommonResume'
4188 else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) &&
4189 RI->getValue() == RI->getParent()->getFirstNonPHI())
4190 // The resume must unwind the exception that caused control to branch here.
4191 return simplifySingleResume(RI);
4192
4193 return false;
4194}
4195
4196// Check if cleanup block is empty
4197static bool isCleanupBlockEmpty(iterator_range<BasicBlock::iterator> R) {
4198 for (Instruction &I : R) {
4199 auto *II = dyn_cast<IntrinsicInst>(&I);
4200 if (!II)
4201 return false;
4202
4203 Intrinsic::ID IntrinsicID = II->getIntrinsicID();
4204 switch (IntrinsicID) {
4205 case Intrinsic::dbg_declare:
4206 case Intrinsic::dbg_value:
4207 case Intrinsic::dbg_label:
4208 case Intrinsic::lifetime_end:
4209 break;
4210 default:
4211 return false;
4212 }
4213 }
4214 return true;
23
Returning the value 1, which participates in a condition later
4215}
4216
4217// Simplify resume that is shared by several landing pads (phi of landing pad).
4218bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
4219 BasicBlock *BB = RI->getParent();
4220
4221 // Check that there are no other instructions except for debug and lifetime
4222 // intrinsics between the phi's and resume instruction.
4223 if (!isCleanupBlockEmpty(
22
Calling 'isCleanupBlockEmpty'
24
Returning from 'isCleanupBlockEmpty'
25
Taking false branch
4224 make_range(RI->getParent()->getFirstNonPHI(), BB->getTerminator())))
4225 return false;
4226
4227 SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks;
4228 auto *PhiLPInst = cast<PHINode>(RI->getValue());
26
The object is a 'PHINode'
4229
4230 // Check incoming blocks to see if any of them are trivial.
4231 for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End;
27
Assuming 'Idx' is not equal to 'End'
28
Loop condition is true. Entering loop body
4232 Idx++) {
4233 auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx);
4234 auto *IncomingValue = PhiLPInst->getIncomingValue(Idx);
4235
4236 // If the block has other successors, we can not delete it because
4237 // it has other dependents.
4238 if (IncomingBB->getUniqueSuccessor() != BB)
29
Assuming the condition is false
30
Taking false branch
4239 continue;
4240
4241 auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI());
31
Assuming the object is not a 'LandingPadInst'
32
'LandingPad' initialized to a null pointer value
4242 // Not the landing pad that caused the control to branch here.
4243 if (IncomingValue
32.1
'IncomingValue' is equal to 'LandingPad'
!= LandingPad)
33
Taking false branch
4244 continue;
4245
4246 if (isCleanupBlockEmpty(
4247 make_range(LandingPad->getNextNode(), IncomingBB->getTerminator())))
34
Called C++ object pointer is null
4248 TrivialUnwindBlocks.insert(IncomingBB);
4249 }
4250
4251 // If no trivial unwind blocks, don't do any simplifications.
4252 if (TrivialUnwindBlocks.empty())
4253 return false;
4254
4255 // Turn all invokes that unwind here into calls.
4256 for (auto *TrivialBB : TrivialUnwindBlocks) {
4257 // Blocks that will be simplified should be removed from the phi node.
4258 // Note there could be multiple edges to the resume block, and we need
4259 // to remove them all.
4260 while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1)
4261 BB->removePredecessor(TrivialBB, true);
4262
4263 for (BasicBlock *Pred :
4264 llvm::make_early_inc_range(predecessors(TrivialBB))) {
4265 removeUnwindEdge(Pred, DTU);
4266 ++NumInvokes;
4267 }
4268
4269 // In each SimplifyCFG run, only the current processed block can be erased.
4270 // Otherwise, it will break the iteration of SimplifyCFG pass. So instead
4271 // of erasing TrivialBB, we only remove the branch to the common resume
4272 // block so that we can later erase the resume block since it has no
4273 // predecessors.
4274 TrivialBB->getTerminator()->eraseFromParent();
4275 new UnreachableInst(RI->getContext(), TrivialBB);
4276 if (DTU)
4277 DTU->applyUpdates({{DominatorTree::Delete, TrivialBB, BB}});
4278 }
4279
4280 // Delete the resume block if all its predecessors have been removed.
4281 if (pred_empty(BB)) {
4282 if (DTU)
4283 DTU->deleteBB(BB);
4284 else
4285 BB->eraseFromParent();
4286 }
4287
4288 return !TrivialUnwindBlocks.empty();
4289}
4290
4291// Simplify resume that is only used by a single (non-phi) landing pad.
4292bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) {
4293 BasicBlock *BB = RI->getParent();
4294 auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI());
4295 assert(RI->getValue() == LPInst &&((RI->getValue() == LPInst && "Resume must unwind the exception that caused control to here"
) ? static_cast<void> (0) : __assert_fail ("RI->getValue() == LPInst && \"Resume must unwind the exception that caused control to here\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4296, __PRETTY_FUNCTION__))
4296 "Resume must unwind the exception that caused control to here")((RI->getValue() == LPInst && "Resume must unwind the exception that caused control to here"
) ? static_cast<void> (0) : __assert_fail ("RI->getValue() == LPInst && \"Resume must unwind the exception that caused control to here\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4296, __PRETTY_FUNCTION__))
;
4297
4298 // Check that there are no other instructions except for debug intrinsics.
4299 if (!isCleanupBlockEmpty(
4300 make_range<Instruction *>(LPInst->getNextNode(), RI)))
4301 return false;
4302
4303 // Turn all invokes that unwind here into calls and delete the basic block.
4304 for (BasicBlock *Pred : llvm::make_early_inc_range(predecessors(BB))) {
4305 removeUnwindEdge(Pred, DTU);
4306 ++NumInvokes;
4307 }
4308
4309 // The landingpad is now unreachable. Zap it.
4310 if (DTU)
4311 DTU->deleteBB(BB);
4312 else
4313 BB->eraseFromParent();
4314 return true;
4315}
4316
4317static bool removeEmptyCleanup(CleanupReturnInst *RI, DomTreeUpdater *DTU) {
4318 // If this is a trivial cleanup pad that executes no instructions, it can be
4319 // eliminated. If the cleanup pad continues to the caller, any predecessor
4320 // that is an EH pad will be updated to continue to the caller and any
4321 // predecessor that terminates with an invoke instruction will have its invoke
4322 // instruction converted to a call instruction. If the cleanup pad being
4323 // simplified does not continue to the caller, each predecessor will be
4324 // updated to continue to the unwind destination of the cleanup pad being
4325 // simplified.
4326 BasicBlock *BB = RI->getParent();
4327 CleanupPadInst *CPInst = RI->getCleanupPad();
4328 if (CPInst->getParent() != BB)
4329 // This isn't an empty cleanup.
4330 return false;
4331
4332 // We cannot kill the pad if it has multiple uses. This typically arises
4333 // from unreachable basic blocks.
4334 if (!CPInst->hasOneUse())
4335 return false;
4336
4337 // Check that there are no other instructions except for benign intrinsics.
4338 if (!isCleanupBlockEmpty(
4339 make_range<Instruction *>(CPInst->getNextNode(), RI)))
4340 return false;
4341
4342 // If the cleanup return we are simplifying unwinds to the caller, this will
4343 // set UnwindDest to nullptr.
4344 BasicBlock *UnwindDest = RI->getUnwindDest();
4345 Instruction *DestEHPad = UnwindDest ? UnwindDest->getFirstNonPHI() : nullptr;
4346
4347 // We're about to remove BB from the control flow. Before we do, sink any
4348 // PHINodes into the unwind destination. Doing this before changing the
4349 // control flow avoids some potentially slow checks, since we can currently
4350 // be certain that UnwindDest and BB have no common predecessors (since they
4351 // are both EH pads).
4352 if (UnwindDest) {
4353 // First, go through the PHI nodes in UnwindDest and update any nodes that
4354 // reference the block we are removing
4355 for (BasicBlock::iterator I = UnwindDest->begin(),
4356 IE = DestEHPad->getIterator();
4357 I != IE; ++I) {
4358 PHINode *DestPN = cast<PHINode>(I);
4359
4360 int Idx = DestPN->getBasicBlockIndex(BB);
4361 // Since BB unwinds to UnwindDest, it has to be in the PHI node.
4362 assert(Idx != -1)((Idx != -1) ? static_cast<void> (0) : __assert_fail ("Idx != -1"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4362, __PRETTY_FUNCTION__))
;
4363 // This PHI node has an incoming value that corresponds to a control
4364 // path through the cleanup pad we are removing. If the incoming
4365 // value is in the cleanup pad, it must be a PHINode (because we
4366 // verified above that the block is otherwise empty). Otherwise, the
4367 // value is either a constant or a value that dominates the cleanup
4368 // pad being removed.
4369 //
4370 // Because BB and UnwindDest are both EH pads, all of their
4371 // predecessors must unwind to these blocks, and since no instruction
4372 // can have multiple unwind destinations, there will be no overlap in
4373 // incoming blocks between SrcPN and DestPN.
4374 Value *SrcVal = DestPN->getIncomingValue(Idx);
4375 PHINode *SrcPN = dyn_cast<PHINode>(SrcVal);
4376
4377 // Remove the entry for the block we are deleting.
4378 DestPN->removeIncomingValue(Idx, false);
4379
4380 if (SrcPN && SrcPN->getParent() == BB) {
4381 // If the incoming value was a PHI node in the cleanup pad we are
4382 // removing, we need to merge that PHI node's incoming values into
4383 // DestPN.
4384 for (unsigned SrcIdx = 0, SrcE = SrcPN->getNumIncomingValues();
4385 SrcIdx != SrcE; ++SrcIdx) {
4386 DestPN->addIncoming(SrcPN->getIncomingValue(SrcIdx),
4387 SrcPN->getIncomingBlock(SrcIdx));
4388 }
4389 } else {
4390 // Otherwise, the incoming value came from above BB and
4391 // so we can just reuse it. We must associate all of BB's
4392 // predecessors with this value.
4393 for (auto *pred : predecessors(BB)) {
4394 DestPN->addIncoming(SrcVal, pred);
4395 }
4396 }
4397 }
4398
4399 // Sink any remaining PHI nodes directly into UnwindDest.
4400 Instruction *InsertPt = DestEHPad;
4401 for (BasicBlock::iterator I = BB->begin(),
4402 IE = BB->getFirstNonPHI()->getIterator();
4403 I != IE;) {
4404 // The iterator must be incremented here because the instructions are
4405 // being moved to another block.
4406 PHINode *PN = cast<PHINode>(I++);
4407 if (PN->use_empty() || !PN->isUsedOutsideOfBlock(BB))
4408 // If the PHI node has no uses or all of its uses are in this basic
4409 // block (meaning they are debug or lifetime intrinsics), just leave
4410 // it. It will be erased when we erase BB below.
4411 continue;
4412
4413 // Otherwise, sink this PHI node into UnwindDest.
4414 // Any predecessors to UnwindDest which are not already represented
4415 // must be back edges which inherit the value from the path through
4416 // BB. In this case, the PHI value must reference itself.
4417 for (auto *pred : predecessors(UnwindDest))
4418 if (pred != BB)
4419 PN->addIncoming(PN, pred);
4420 PN->moveBefore(InsertPt);
4421 }
4422 }
4423
4424 std::vector<DominatorTree::UpdateType> Updates;
4425
4426 // We use make_early_inc_range here because we may remove some predecessors.
4427 for (BasicBlock *PredBB : llvm::make_early_inc_range(predecessors(BB))) {
4428 if (UnwindDest == nullptr) {
4429 if (DTU) {
4430 DTU->applyUpdates(Updates);
4431 Updates.clear();
4432 }
4433 removeUnwindEdge(PredBB, DTU);
4434 ++NumInvokes;
4435 } else {
4436 Instruction *TI = PredBB->getTerminator();
4437 TI->replaceUsesOfWith(BB, UnwindDest);
4438 if (DTU) {
4439 Updates.push_back({DominatorTree::Insert, PredBB, UnwindDest});
4440 Updates.push_back({DominatorTree::Delete, PredBB, BB});
4441 }
4442 }
4443 }
4444
4445 if (DTU) {
4446 DTU->applyUpdates(Updates);
4447 DTU->deleteBB(BB);
4448 } else
4449 // The cleanup pad is now unreachable. Zap it.
4450 BB->eraseFromParent();
4451
4452 return true;
4453}
4454
4455// Try to merge two cleanuppads together.
4456static bool mergeCleanupPad(CleanupReturnInst *RI) {
4457 // Skip any cleanuprets which unwind to caller, there is nothing to merge
4458 // with.
4459 BasicBlock *UnwindDest = RI->getUnwindDest();
4460 if (!UnwindDest)
4461 return false;
4462
4463 // This cleanupret isn't the only predecessor of this cleanuppad, it wouldn't
4464 // be safe to merge without code duplication.
4465 if (UnwindDest->getSinglePredecessor() != RI->getParent())
4466 return false;
4467
4468 // Verify that our cleanuppad's unwind destination is another cleanuppad.
4469 auto *SuccessorCleanupPad = dyn_cast<CleanupPadInst>(&UnwindDest->front());
4470 if (!SuccessorCleanupPad)
4471 return false;
4472
4473 CleanupPadInst *PredecessorCleanupPad = RI->getCleanupPad();
4474 // Replace any uses of the successor cleanupad with the predecessor pad
4475 // The only cleanuppad uses should be this cleanupret, it's cleanupret and
4476 // funclet bundle operands.
4477 SuccessorCleanupPad->replaceAllUsesWith(PredecessorCleanupPad);
4478 // Remove the old cleanuppad.
4479 SuccessorCleanupPad->eraseFromParent();
4480 // Now, we simply replace the cleanupret with a branch to the unwind
4481 // destination.
4482 BranchInst::Create(UnwindDest, RI->getParent());
4483 RI->eraseFromParent();
4484
4485 return true;
4486}
4487
4488bool SimplifyCFGOpt::simplifyCleanupReturn(CleanupReturnInst *RI) {
4489 // It is possible to transiantly have an undef cleanuppad operand because we
4490 // have deleted some, but not all, dead blocks.
4491 // Eventually, this block will be deleted.
4492 if (isa<UndefValue>(RI->getOperand(0)))
4493 return false;
4494
4495 if (mergeCleanupPad(RI))
4496 return true;
4497
4498 if (removeEmptyCleanup(RI, DTU))
4499 return true;
4500
4501 return false;
4502}
4503
4504bool SimplifyCFGOpt::simplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
4505 BasicBlock *BB = RI->getParent();
4506 if (!BB->getFirstNonPHIOrDbg()->isTerminator())
4507 return false;
4508
4509 // Find predecessors that end with branches.
4510 SmallVector<BasicBlock *, 8> UncondBranchPreds;
4511 SmallVector<BranchInst *, 8> CondBranchPreds;
4512 for (BasicBlock *P : predecessors(BB)) {
4513 Instruction *PTI = P->getTerminator();
4514 if (BranchInst *BI = dyn_cast<BranchInst>(PTI)) {
4515 if (BI->isUnconditional())
4516 UncondBranchPreds.push_back(P);
4517 else
4518 CondBranchPreds.push_back(BI);
4519 }
4520 }
4521
4522 // If we found some, do the transformation!
4523 if (!UncondBranchPreds.empty() && DupRet) {
4524 while (!UncondBranchPreds.empty()) {
4525 BasicBlock *Pred = UncondBranchPreds.pop_back_val();
4526 LLVM_DEBUG(dbgs() << "FOLDING: " << *BBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING: " << *BB <<
"INTO UNCOND BRANCH PRED: " << *Pred; } } while (false
)
4527 << "INTO UNCOND BRANCH PRED: " << *Pred)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "FOLDING: " << *BB <<
"INTO UNCOND BRANCH PRED: " << *Pred; } } while (false
)
;
4528 (void)FoldReturnIntoUncondBranch(RI, BB, Pred, DTU);
4529 }
4530
4531 // If we eliminated all predecessors of the block, delete the block now.
4532 if (pred_empty(BB)) {
4533 // We know there are no successors, so just nuke the block.
4534 if (DTU)
4535 DTU->deleteBB(BB);
4536 else
4537 BB->eraseFromParent();
4538 }
4539
4540 return true;
4541 }
4542
4543 // Check out all of the conditional branches going to this return
4544 // instruction. If any of them just select between returns, change the
4545 // branch itself into a select/return pair.
4546 while (!CondBranchPreds.empty()) {
4547 BranchInst *BI = CondBranchPreds.pop_back_val();
4548
4549 // Check to see if the non-BB successor is also a return block.
4550 if (isa<ReturnInst>(BI->getSuccessor(0)->getTerminator()) &&
4551 isa<ReturnInst>(BI->getSuccessor(1)->getTerminator()) &&
4552 SimplifyCondBranchToTwoReturns(BI, Builder))
4553 return true;
4554 }
4555 return false;
4556}
4557
4558bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
4559 BasicBlock *BB = UI->getParent();
4560
4561 bool Changed = false;
4562
4563 // If there are any instructions immediately before the unreachable that can
4564 // be removed, do so.
4565 while (UI->getIterator() != BB->begin()) {
4566 BasicBlock::iterator BBI = UI->getIterator();
4567 --BBI;
4568 // Do not delete instructions that can have side effects which might cause
4569 // the unreachable to not be reachable; specifically, calls and volatile
4570 // operations may have this effect.
4571 if (isa<CallInst>(BBI) && !isa<DbgInfoIntrinsic>(BBI))
4572 break;
4573
4574 if (BBI->mayHaveSideEffects()) {
4575 if (auto *SI = dyn_cast<StoreInst>(BBI)) {
4576 if (SI->isVolatile())
4577 break;
4578 } else if (auto *LI = dyn_cast<LoadInst>(BBI)) {
4579 if (LI->isVolatile())
4580 break;
4581 } else if (auto *RMWI = dyn_cast<AtomicRMWInst>(BBI)) {
4582 if (RMWI->isVolatile())
4583 break;
4584 } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(BBI)) {
4585 if (CXI->isVolatile())
4586 break;
4587 } else if (isa<CatchPadInst>(BBI)) {
4588 // A catchpad may invoke exception object constructors and such, which
4589 // in some languages can be arbitrary code, so be conservative by
4590 // default.
4591 // For CoreCLR, it just involves a type test, so can be removed.
4592 if (classifyEHPersonality(BB->getParent()->getPersonalityFn()) !=
4593 EHPersonality::CoreCLR)
4594 break;
4595 } else if (!isa<FenceInst>(BBI) && !isa<VAArgInst>(BBI) &&
4596 !isa<LandingPadInst>(BBI)) {
4597 break;
4598 }
4599 // Note that deleting LandingPad's here is in fact okay, although it
4600 // involves a bit of subtle reasoning. If this inst is a LandingPad,
4601 // all the predecessors of this block will be the unwind edges of Invokes,
4602 // and we can therefore guarantee this block will be erased.
4603 }
4604
4605 // Delete this instruction (any uses are guaranteed to be dead)
4606 if (!BBI->use_empty())
4607 BBI->replaceAllUsesWith(UndefValue::get(BBI->getType()));
4608 BBI->eraseFromParent();
4609 Changed = true;
4610 }
4611
4612 // If the unreachable instruction is the first in the block, take a gander
4613 // at all of the predecessors of this instruction, and simplify them.
4614 if (&BB->front() != UI)
4615 return Changed;
4616
4617 std::vector<DominatorTree::UpdateType> Updates;
4618
4619 SmallSetVector<BasicBlock *, 8> Preds(pred_begin(BB), pred_end(BB));
4620 for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
4621 auto *Predecessor = Preds[i];
4622 Instruction *TI = Predecessor->getTerminator();
4623 IRBuilder<> Builder(TI);
4624 if (auto *BI = dyn_cast<BranchInst>(TI)) {
4625 // We could either have a proper unconditional branch,
4626 // or a degenerate conditional branch with matching destinations.
4627 if (all_of(BI->successors(),
4628 [BB](auto *Successor) { return Successor == BB; })) {
4629 new UnreachableInst(TI->getContext(), TI);
4630 TI->eraseFromParent();
4631 Changed = true;
4632 } else {
4633 assert(BI->isConditional() && "Can't get here with an uncond branch.")((BI->isConditional() && "Can't get here with an uncond branch."
) ? static_cast<void> (0) : __assert_fail ("BI->isConditional() && \"Can't get here with an uncond branch.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4633, __PRETTY_FUNCTION__))
;
4634 Value* Cond = BI->getCondition();
4635 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&((BI->getSuccessor(0) != BI->getSuccessor(1) &&
"The destinations are guaranteed to be different here.") ? static_cast
<void> (0) : __assert_fail ("BI->getSuccessor(0) != BI->getSuccessor(1) && \"The destinations are guaranteed to be different here.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4636, __PRETTY_FUNCTION__))
4636 "The destinations are guaranteed to be different here.")((BI->getSuccessor(0) != BI->getSuccessor(1) &&
"The destinations are guaranteed to be different here.") ? static_cast
<void> (0) : __assert_fail ("BI->getSuccessor(0) != BI->getSuccessor(1) && \"The destinations are guaranteed to be different here.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4636, __PRETTY_FUNCTION__))
;
4637 if (BI->getSuccessor(0) == BB) {
4638 Builder.CreateAssumption(Builder.CreateNot(Cond));
4639 Builder.CreateBr(BI->getSuccessor(1));
4640 } else {
4641 assert(BI->getSuccessor(1) == BB && "Incorrect CFG")((BI->getSuccessor(1) == BB && "Incorrect CFG") ? static_cast
<void> (0) : __assert_fail ("BI->getSuccessor(1) == BB && \"Incorrect CFG\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4641, __PRETTY_FUNCTION__))
;
4642 Builder.CreateAssumption(Cond);
4643 Builder.CreateBr(BI->getSuccessor(0));
4644 }
4645 EraseTerminatorAndDCECond(BI);
4646 Changed = true;
4647 }
4648 if (DTU)
4649 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4650 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
4651 SwitchInstProfUpdateWrapper SU(*SI);
4652 for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) {
4653 if (i->getCaseSuccessor() != BB) {
4654 ++i;
4655 continue;
4656 }
4657 BB->removePredecessor(SU->getParent());
4658 i = SU.removeCase(i);
4659 e = SU->case_end();
4660 Changed = true;
4661 }
4662 // Note that the default destination can't be removed!
4663 if (DTU && SI->getDefaultDest() != BB)
4664 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4665 } else if (auto *II = dyn_cast<InvokeInst>(TI)) {
4666 if (II->getUnwindDest() == BB) {
4667 if (DTU) {
4668 DTU->applyUpdates(Updates);
4669 Updates.clear();
4670 }
4671 removeUnwindEdge(TI->getParent(), DTU);
4672 Changed = true;
4673 }
4674 } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) {
4675 if (CSI->getUnwindDest() == BB) {
4676 if (DTU) {
4677 DTU->applyUpdates(Updates);
4678 Updates.clear();
4679 }
4680 removeUnwindEdge(TI->getParent(), DTU);
4681 Changed = true;
4682 continue;
4683 }
4684
4685 for (CatchSwitchInst::handler_iterator I = CSI->handler_begin(),
4686 E = CSI->handler_end();
4687 I != E; ++I) {
4688 if (*I == BB) {
4689 CSI->removeHandler(I);
4690 --I;
4691 --E;
4692 Changed = true;
4693 }
4694 }
4695 if (DTU)
4696 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4697 if (CSI->getNumHandlers() == 0) {
4698 if (CSI->hasUnwindDest()) {
4699 // Redirect all predecessors of the block containing CatchSwitchInst
4700 // to instead branch to the CatchSwitchInst's unwind destination.
4701 if (DTU) {
4702 for (auto *PredecessorOfPredecessor : predecessors(Predecessor)) {
4703 Updates.push_back({DominatorTree::Insert,
4704 PredecessorOfPredecessor,
4705 CSI->getUnwindDest()});
4706 Updates.push_back({DominatorTree::Delete,
4707 PredecessorOfPredecessor, Predecessor});
4708 }
4709 }
4710 Predecessor->replaceAllUsesWith(CSI->getUnwindDest());
4711 } else {
4712 // Rewrite all preds to unwind to caller (or from invoke to call).
4713 if (DTU) {
4714 DTU->applyUpdates(Updates);
4715 Updates.clear();
4716 }
4717 SmallVector<BasicBlock *, 8> EHPreds(predecessors(Predecessor));
4718 for (BasicBlock *EHPred : EHPreds)
4719 removeUnwindEdge(EHPred, DTU);
4720 }
4721 // The catchswitch is no longer reachable.
4722 new UnreachableInst(CSI->getContext(), CSI);
4723 CSI->eraseFromParent();
4724 Changed = true;
4725 }
4726 } else if (auto *CRI = dyn_cast<CleanupReturnInst>(TI)) {
4727 (void)CRI;
4728 assert(CRI->hasUnwindDest() && CRI->getUnwindDest() == BB &&((CRI->hasUnwindDest() && CRI->getUnwindDest() ==
BB && "Expected to always have an unwind to BB.") ? static_cast
<void> (0) : __assert_fail ("CRI->hasUnwindDest() && CRI->getUnwindDest() == BB && \"Expected to always have an unwind to BB.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4729, __PRETTY_FUNCTION__))
4729 "Expected to always have an unwind to BB.")((CRI->hasUnwindDest() && CRI->getUnwindDest() ==
BB && "Expected to always have an unwind to BB.") ? static_cast
<void> (0) : __assert_fail ("CRI->hasUnwindDest() && CRI->getUnwindDest() == BB && \"Expected to always have an unwind to BB.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4729, __PRETTY_FUNCTION__))
;
4730 if (DTU)
4731 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4732 new UnreachableInst(TI->getContext(), TI);
4733 TI->eraseFromParent();
4734 Changed = true;
4735 }
4736 }
4737
4738 if (DTU)
4739 DTU->applyUpdates(Updates);
4740
4741 // If this block is now dead, remove it.
4742 if (pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) {
4743 // We know there are no successors, so just nuke the block.
4744 if (DTU)
4745 DTU->deleteBB(BB);
4746 else
4747 BB->eraseFromParent();
4748 return true;
4749 }
4750
4751 return Changed;
4752}
4753
4754static bool CasesAreContiguous(SmallVectorImpl<ConstantInt *> &Cases) {
4755 assert(Cases.size() >= 1)((Cases.size() >= 1) ? static_cast<void> (0) : __assert_fail
("Cases.size() >= 1", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4755, __PRETTY_FUNCTION__))
;
4756
4757 array_pod_sort(Cases.begin(), Cases.end(), ConstantIntSortPredicate);
4758 for (size_t I = 1, E = Cases.size(); I != E; ++I) {
4759 if (Cases[I - 1]->getValue() != Cases[I]->getValue() + 1)
4760 return false;
4761 }
4762 return true;
4763}
4764
4765static void createUnreachableSwitchDefault(SwitchInst *Switch,
4766 DomTreeUpdater *DTU) {
4767 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch default is dead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SimplifyCFG: switch default is dead.\n"
; } } while (false)
;
4768 auto *BB = Switch->getParent();
4769 BasicBlock *NewDefaultBlock = SplitBlockPredecessors(
4770 Switch->getDefaultDest(), Switch->getParent(), "", DTU);
4771 auto *OrigDefaultBlock = Switch->getDefaultDest();
4772 Switch->setDefaultDest(&*NewDefaultBlock);
4773 if (DTU)
4774 DTU->applyUpdates({{DominatorTree::Insert, BB, &*NewDefaultBlock},
4775 {DominatorTree::Delete, BB, OrigDefaultBlock}});
4776 SplitBlock(&*NewDefaultBlock, &NewDefaultBlock->front(), DTU);
4777 SmallVector<DominatorTree::UpdateType, 2> Updates;
4778 if (DTU)
4779 for (auto *Successor : successors(NewDefaultBlock))
4780 Updates.push_back({DominatorTree::Delete, NewDefaultBlock, Successor});
4781 auto *NewTerminator = NewDefaultBlock->getTerminator();
4782 new UnreachableInst(Switch->getContext(), NewTerminator);
4783 EraseTerminatorAndDCECond(NewTerminator);
4784 if (DTU)
4785 DTU->applyUpdates(Updates);
4786}
4787
4788/// Turn a switch with two reachable destinations into an integer range
4789/// comparison and branch.
4790bool SimplifyCFGOpt::TurnSwitchRangeIntoICmp(SwitchInst *SI,
4791 IRBuilder<> &Builder) {
4792 assert(SI->getNumCases() > 1 && "Degenerate switch?")((SI->getNumCases() > 1 && "Degenerate switch?"
) ? static_cast<void> (0) : __assert_fail ("SI->getNumCases() > 1 && \"Degenerate switch?\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4792, __PRETTY_FUNCTION__))
;
4793
4794 bool HasDefault =
4795 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
4796
4797 auto *BB = SI->getParent();
4798
4799 // Partition the cases into two sets with different destinations.
4800 BasicBlock *DestA = HasDefault ? SI->getDefaultDest() : nullptr;
4801 BasicBlock *DestB = nullptr;
4802 SmallVector<ConstantInt *, 16> CasesA;
4803 SmallVector<ConstantInt *, 16> CasesB;
4804
4805 for (auto Case : SI->cases()) {
4806 BasicBlock *Dest = Case.getCaseSuccessor();
4807 if (!DestA)
4808 DestA = Dest;
4809 if (Dest == DestA) {
4810 CasesA.push_back(Case.getCaseValue());
4811 continue;
4812 }
4813 if (!DestB)
4814 DestB = Dest;
4815 if (Dest == DestB) {
4816 CasesB.push_back(Case.getCaseValue());
4817 continue;
4818 }
4819 return false; // More than two destinations.
4820 }
4821
4822 assert(DestA && DestB &&((DestA && DestB && "Single-destination switch should have been folded."
) ? static_cast<void> (0) : __assert_fail ("DestA && DestB && \"Single-destination switch should have been folded.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4823, __PRETTY_FUNCTION__))
4823 "Single-destination switch should have been folded.")((DestA && DestB && "Single-destination switch should have been folded."
) ? static_cast<void> (0) : __assert_fail ("DestA && DestB && \"Single-destination switch should have been folded.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4823, __PRETTY_FUNCTION__))
;
4824 assert(DestA != DestB)((DestA != DestB) ? static_cast<void> (0) : __assert_fail
("DestA != DestB", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4824, __PRETTY_FUNCTION__))
;
4825 assert(DestB != SI->getDefaultDest())((DestB != SI->getDefaultDest()) ? static_cast<void>
(0) : __assert_fail ("DestB != SI->getDefaultDest()", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4825, __PRETTY_FUNCTION__))
;
4826 assert(!CasesB.empty() && "There must be non-default cases.")((!CasesB.empty() && "There must be non-default cases."
) ? static_cast<void> (0) : __assert_fail ("!CasesB.empty() && \"There must be non-default cases.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4826, __PRETTY_FUNCTION__))
;
4827 assert(!CasesA.empty() || HasDefault)((!CasesA.empty() || HasDefault) ? static_cast<void> (0
) : __assert_fail ("!CasesA.empty() || HasDefault", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4827, __PRETTY_FUNCTION__))
;
4828
4829 // Figure out if one of the sets of cases form a contiguous range.
4830 SmallVectorImpl<ConstantInt *> *ContiguousCases = nullptr;
4831 BasicBlock *ContiguousDest = nullptr;
4832 BasicBlock *OtherDest = nullptr;
4833 if (!CasesA.empty() && CasesAreContiguous(CasesA)) {
4834 ContiguousCases = &CasesA;
4835 ContiguousDest = DestA;
4836 OtherDest = DestB;
4837 } else if (CasesAreContiguous(CasesB)) {
4838 ContiguousCases = &CasesB;
4839 ContiguousDest = DestB;
4840 OtherDest = DestA;
4841 } else
4842 return false;
4843
4844 // Start building the compare and branch.
4845
4846 Constant *Offset = ConstantExpr::getNeg(ContiguousCases->back());
4847 Constant *NumCases =
4848 ConstantInt::get(Offset->getType(), ContiguousCases->size());
4849
4850 Value *Sub = SI->getCondition();
4851 if (!Offset->isNullValue())
4852 Sub = Builder.CreateAdd(Sub, Offset, Sub->getName() + ".off");
4853
4854 Value *Cmp;
4855 // If NumCases overflowed, then all possible values jump to the successor.
4856 if (NumCases->isNullValue() && !ContiguousCases->empty())
4857 Cmp = ConstantInt::getTrue(SI->getContext());
4858 else
4859 Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch");
4860 BranchInst *NewBI = Builder.CreateCondBr(Cmp, ContiguousDest, OtherDest);
4861
4862 // Update weight for the newly-created conditional branch.
4863 if (HasBranchWeights(SI)) {
4864 SmallVector<uint64_t, 8> Weights;
4865 GetBranchWeights(SI, Weights);
4866 if (Weights.size() == 1 + SI->getNumCases()) {
4867 uint64_t TrueWeight = 0;
4868 uint64_t FalseWeight = 0;
4869 for (size_t I = 0, E = Weights.size(); I != E; ++I) {
4870 if (SI->getSuccessor(I) == ContiguousDest)
4871 TrueWeight += Weights[I];
4872 else
4873 FalseWeight += Weights[I];
4874 }
4875 while (TrueWeight > UINT32_MAX(4294967295U) || FalseWeight > UINT32_MAX(4294967295U)) {
4876 TrueWeight /= 2;
4877 FalseWeight /= 2;
4878 }
4879 setBranchWeights(NewBI, TrueWeight, FalseWeight);
4880 }
4881 }
4882
4883 // Prune obsolete incoming values off the successors' PHI nodes.
4884 for (auto BBI = ContiguousDest->begin(); isa<PHINode>(BBI); ++BBI) {
4885 unsigned PreviousEdges = ContiguousCases->size();
4886 if (ContiguousDest == SI->getDefaultDest())
4887 ++PreviousEdges;
4888 for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4889 cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4890 }
4891 for (auto BBI = OtherDest->begin(); isa<PHINode>(BBI); ++BBI) {
4892 unsigned PreviousEdges = SI->getNumCases() - ContiguousCases->size();
4893 if (OtherDest == SI->getDefaultDest())
4894 ++PreviousEdges;
4895 for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4896 cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4897 }
4898
4899 // Clean up the default block - it may have phis or other instructions before
4900 // the unreachable terminator.
4901 if (!HasDefault)
4902 createUnreachableSwitchDefault(SI, DTU);
4903
4904 auto *UnreachableDefault = SI->getDefaultDest();
4905
4906 // Drop the switch.
4907 SI->eraseFromParent();
4908
4909 if (!HasDefault && DTU)
4910 DTU->applyUpdates({{DominatorTree::Delete, BB, UnreachableDefault}});
4911
4912 return true;
4913}
4914
4915/// Compute masked bits for the condition of a switch
4916/// and use it to remove dead cases.
4917static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU,
4918 AssumptionCache *AC,
4919 const DataLayout &DL) {
4920 Value *Cond = SI->getCondition();
4921 unsigned Bits = Cond->getType()->getIntegerBitWidth();
4922 KnownBits Known = computeKnownBits(Cond, DL, 0, AC, SI);
4923
4924 // We can also eliminate cases by determining that their values are outside of
4925 // the limited range of the condition based on how many significant (non-sign)
4926 // bits are in the condition value.
4927 unsigned ExtraSignBits = ComputeNumSignBits(Cond, DL, 0, AC, SI) - 1;
4928 unsigned MaxSignificantBitsInCond = Bits - ExtraSignBits;
4929
4930 // Gather dead cases.
4931 SmallVector<ConstantInt *, 8> DeadCases;
4932 SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
4933 for (auto &Case : SI->cases()) {
4934 auto *Successor = Case.getCaseSuccessor();
4935 if (DTU)
4936 ++NumPerSuccessorCases[Successor];
4937 const APInt &CaseVal = Case.getCaseValue()->getValue();
4938 if (Known.Zero.intersects(CaseVal) || !Known.One.isSubsetOf(CaseVal) ||
4939 (CaseVal.getMinSignedBits() > MaxSignificantBitsInCond)) {
4940 DeadCases.push_back(Case.getCaseValue());
4941 if (DTU)
4942 --NumPerSuccessorCases[Successor];
4943 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseValdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SimplifyCFG: switch case "
<< CaseVal << " is dead.\n"; } } while (false)
4944 << " is dead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("simplifycfg")) { dbgs() << "SimplifyCFG: switch case "
<< CaseVal << " is dead.\n"; } } while (false)
;
4945 }
4946 }
4947
4948 // If we can prove that the cases must cover all possible values, the
4949 // default destination becomes dead and we can remove it. If we know some
4950 // of the bits in the value, we can use that to more precisely compute the
4951 // number of possible unique case values.
4952 bool HasDefault =
4953 !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
4954 const unsigned NumUnknownBits =
4955 Bits - (Known.Zero | Known.One).countPopulation();
4956 assert(NumUnknownBits <= Bits)((NumUnknownBits <= Bits) ? static_cast<void> (0) : __assert_fail
("NumUnknownBits <= Bits", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4956, __PRETTY_FUNCTION__))
;
4957 if (HasDefault && DeadCases.empty() &&
4958 NumUnknownBits < 64 /* avoid overflow */ &&
4959 SI->getNumCases() == (1ULL << NumUnknownBits)) {
4960 createUnreachableSwitchDefault(SI, DTU);
4961 return true;
4962 }
4963
4964 if (DeadCases.empty())
4965 return false;
4966
4967 SwitchInstProfUpdateWrapper SIW(*SI);
4968 for (ConstantInt *DeadCase : DeadCases) {
4969 SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase);
4970 assert(CaseI != SI->case_default() &&((CaseI != SI->case_default() && "Case was not found. Probably mistake in DeadCases forming."
) ? static_cast<void> (0) : __assert_fail ("CaseI != SI->case_default() && \"Case was not found. Probably mistake in DeadCases forming.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4971, __PRETTY_FUNCTION__))
4971 "Case was not found. Probably mistake in DeadCases forming.")((CaseI != SI->case_default() && "Case was not found. Probably mistake in DeadCases forming."
) ? static_cast<void> (0) : __assert_fail ("CaseI != SI->case_default() && \"Case was not found. Probably mistake in DeadCases forming.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 4971, __PRETTY_FUNCTION__))
;
4972 // Prune unused values from PHI nodes.
4973 CaseI->getCaseSuccessor()->removePredecessor(SI->getParent());
4974 SIW.removeCase(CaseI);
4975 }
4976
4977 if (DTU) {
4978 std::vector<DominatorTree::UpdateType> Updates;
4979 for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
4980 if (I.second == 0)
4981 Updates.push_back({DominatorTree::Delete, SI->getParent(), I.first});
4982 DTU->applyUpdates(Updates);
4983 }
4984
4985 return true;
4986}
4987
4988/// If BB would be eligible for simplification by
4989/// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated
4990/// by an unconditional branch), look at the phi node for BB in the successor
4991/// block and see if the incoming value is equal to CaseValue. If so, return
4992/// the phi node, and set PhiIndex to BB's index in the phi node.
4993static PHINode *FindPHIForConditionForwarding(ConstantInt *CaseValue,
4994 BasicBlock *BB, int *PhiIndex) {
4995 if (BB->getFirstNonPHIOrDbg() != BB->getTerminator())
4996 return nullptr; // BB must be empty to be a candidate for simplification.
4997 if (!BB->getSinglePredecessor())
4998 return nullptr; // BB must be dominated by the switch.
4999
5000 BranchInst *Branch = dyn_cast<BranchInst>(BB->getTerminator());
5001 if (!Branch || !Branch->isUnconditional())
5002 return nullptr; // Terminator must be unconditional branch.
5003
5004 BasicBlock *Succ = Branch->getSuccessor(0);
5005
5006 for (PHINode &PHI : Succ->phis()) {
5007 int Idx = PHI.getBasicBlockIndex(BB);
5008 assert(Idx >= 0 && "PHI has no entry for predecessor?")((Idx >= 0 && "PHI has no entry for predecessor?")
? static_cast<void> (0) : __assert_fail ("Idx >= 0 && \"PHI has no entry for predecessor?\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5008, __PRETTY_FUNCTION__))
;
5009
5010 Value *InValue = PHI.getIncomingValue(Idx);
5011 if (InValue != CaseValue)
5012 continue;
5013
5014 *PhiIndex = Idx;
5015 return &PHI;
5016 }
5017
5018 return nullptr;
5019}
5020
5021/// Try to forward the condition of a switch instruction to a phi node
5022/// dominated by the switch, if that would mean that some of the destination
5023/// blocks of the switch can be folded away. Return true if a change is made.
5024static bool ForwardSwitchConditionToPHI(SwitchInst *SI) {
5025 using ForwardingNodesMap = DenseMap<PHINode *, SmallVector<int, 4>>;
5026
5027 ForwardingNodesMap ForwardingNodes;
5028 BasicBlock *SwitchBlock = SI->getParent();
5029 bool Changed = false;
5030 for (auto &Case : SI->cases()) {
5031 ConstantInt *CaseValue = Case.getCaseValue();
5032 BasicBlock *CaseDest = Case.getCaseSuccessor();
5033
5034 // Replace phi operands in successor blocks that are using the constant case
5035 // value rather than the switch condition variable:
5036 // switchbb:
5037 // switch i32 %x, label %default [
5038 // i32 17, label %succ
5039 // ...
5040 // succ:
5041 // %r = phi i32 ... [ 17, %switchbb ] ...
5042 // -->
5043 // %r = phi i32 ... [ %x, %switchbb ] ...
5044
5045 for (PHINode &Phi : CaseDest->phis()) {
5046 // This only works if there is exactly 1 incoming edge from the switch to
5047 // a phi. If there is >1, that means multiple cases of the switch map to 1
5048 // value in the phi, and that phi value is not the switch condition. Thus,
5049 // this transform would not make sense (the phi would be invalid because
5050 // a phi can't have different incoming values from the same block).
5051 int SwitchBBIdx = Phi.getBasicBlockIndex(SwitchBlock);
5052 if (Phi.getIncomingValue(SwitchBBIdx) == CaseValue &&
5053 count(Phi.blocks(), SwitchBlock) == 1) {
5054 Phi.setIncomingValue(SwitchBBIdx, SI->getCondition());
5055 Changed = true;
5056 }
5057 }
5058
5059 // Collect phi nodes that are indirectly using this switch's case constants.
5060 int PhiIdx;
5061 if (auto *Phi = FindPHIForConditionForwarding(CaseValue, CaseDest, &PhiIdx))
5062 ForwardingNodes[Phi].push_back(PhiIdx);
5063 }
5064
5065 for (auto &ForwardingNode : ForwardingNodes) {
5066 PHINode *Phi = ForwardingNode.first;
5067 SmallVectorImpl<int> &Indexes = ForwardingNode.second;
5068 if (Indexes.size() < 2)
5069 continue;
5070
5071 for (int Index : Indexes)
5072 Phi->setIncomingValue(Index, SI->getCondition());
5073 Changed = true;
5074 }
5075
5076 return Changed;
5077}
5078
5079/// Return true if the backend will be able to handle
5080/// initializing an array of constants like C.
5081static bool ValidLookupTableConstant(Constant *C, const TargetTransformInfo &TTI) {
5082 if (C->isThreadDependent())
5083 return false;
5084 if (C->isDLLImportDependent())
5085 return false;
5086
5087 if (!isa<ConstantFP>(C) && !isa<ConstantInt>(C) &&
5088 !isa<ConstantPointerNull>(C) && !isa<GlobalValue>(C) &&
5089 !isa<UndefValue>(C) && !isa<ConstantExpr>(C))
5090 return false;
5091
5092 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
5093 if (!CE->isGEPWithNoNotionalOverIndexing())
5094 return false;
5095 if (!ValidLookupTableConstant(CE->getOperand(0), TTI))
5096 return false;
5097 }
5098
5099 if (!TTI.shouldBuildLookupTablesForConstant(C))
5100 return false;
5101
5102 return true;
5103}
5104
5105/// If V is a Constant, return it. Otherwise, try to look up
5106/// its constant value in ConstantPool, returning 0 if it's not there.
5107static Constant *
5108LookupConstant(Value *V,
5109 const SmallDenseMap<Value *, Constant *> &ConstantPool) {
5110 if (Constant *C = dyn_cast<Constant>(V))
5111 return C;
5112 return ConstantPool.lookup(V);
5113}
5114
5115/// Try to fold instruction I into a constant. This works for
5116/// simple instructions such as binary operations where both operands are
5117/// constant or can be replaced by constants from the ConstantPool. Returns the
5118/// resulting constant on success, 0 otherwise.
5119static Constant *
5120ConstantFold(Instruction *I, const DataLayout &DL,
5121 const SmallDenseMap<Value *, Constant *> &ConstantPool) {
5122 if (SelectInst *Select = dyn_cast<SelectInst>(I)) {
5123 Constant *A = LookupConstant(Select->getCondition(), ConstantPool);
5124 if (!A)
5125 return nullptr;
5126 if (A->isAllOnesValue())
5127 return LookupConstant(Select->getTrueValue(), ConstantPool);
5128 if (A->isNullValue())
5129 return LookupConstant(Select->getFalseValue(), ConstantPool);
5130 return nullptr;
5131 }
5132
5133 SmallVector<Constant *, 4> COps;
5134 for (unsigned N = 0, E = I->getNumOperands(); N != E; ++N) {
5135 if (Constant *A = LookupConstant(I->getOperand(N), ConstantPool))
5136 COps.push_back(A);
5137 else
5138 return nullptr;
5139 }
5140
5141 if (CmpInst *Cmp = dyn_cast<CmpInst>(I)) {
5142 return ConstantFoldCompareInstOperands(Cmp->getPredicate(), COps[0],
5143 COps[1], DL);
5144 }
5145
5146 return ConstantFoldInstOperands(I, COps, DL);
5147}
5148
5149/// Try to determine the resulting constant values in phi nodes
5150/// at the common destination basic block, *CommonDest, for one of the case
5151/// destionations CaseDest corresponding to value CaseVal (0 for the default
5152/// case), of a switch instruction SI.
5153static bool
5154GetCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest,
5155 BasicBlock **CommonDest,
5156 SmallVectorImpl<std::pair<PHINode *, Constant *>> &Res,
5157 const DataLayout &DL, const TargetTransformInfo &TTI) {
5158 // The block from which we enter the common destination.
5159 BasicBlock *Pred = SI->getParent();
5160
5161 // If CaseDest is empty except for some side-effect free instructions through
5162 // which we can constant-propagate the CaseVal, continue to its successor.
5163 SmallDenseMap<Value *, Constant *> ConstantPool;
5164 ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal));
5165 for (Instruction &I :CaseDest->instructionsWithoutDebug()) {
5166 if (I.isTerminator()) {
5167 // If the terminator is a simple branch, continue to the next block.
5168 if (I.getNumSuccessors() != 1 || I.isExceptionalTerminator())
5169 return false;
5170 Pred = CaseDest;
5171 CaseDest = I.getSuccessor(0);
5172 } else if (Constant *C = ConstantFold(&I, DL, ConstantPool)) {
5173 // Instruction is side-effect free and constant.
5174
5175 // If the instruction has uses outside this block or a phi node slot for
5176 // the block, it is not safe to bypass the instruction since it would then
5177 // no longer dominate all its uses.
5178 for (auto &Use : I.uses()) {
5179 User *User = Use.getUser();
5180 if (Instruction *I = dyn_cast<Instruction>(User))
5181 if (I->getParent() == CaseDest)
5182 continue;
5183 if (PHINode *Phi = dyn_cast<PHINode>(User))
5184 if (Phi->getIncomingBlock(Use) == CaseDest)
5185 continue;
5186 return false;
5187 }
5188
5189 ConstantPool.insert(std::make_pair(&I, C));
5190 } else {
5191 break;
5192 }
5193 }
5194
5195 // If we did not have a CommonDest before, use the current one.
5196 if (!*CommonDest)
5197 *CommonDest = CaseDest;
5198 // If the destination isn't the common one, abort.
5199 if (CaseDest != *CommonDest)
5200 return false;
5201
5202 // Get the values for this case from phi nodes in the destination block.
5203 for (PHINode &PHI : (*CommonDest)->phis()) {
5204 int Idx = PHI.getBasicBlockIndex(Pred);
5205 if (Idx == -1)
5206 continue;
5207
5208 Constant *ConstVal =
5209 LookupConstant(PHI.getIncomingValue(Idx), ConstantPool);
5210 if (!ConstVal)
5211 return false;
5212
5213 // Be conservative about which kinds of constants we support.
5214 if (!ValidLookupTableConstant(ConstVal, TTI))
5215 return false;
5216
5217 Res.push_back(std::make_pair(&PHI, ConstVal));
5218 }
5219
5220 return Res.size() > 0;
5221}
5222
5223// Helper function used to add CaseVal to the list of cases that generate
5224// Result. Returns the updated number of cases that generate this result.
5225static uintptr_t MapCaseToResult(ConstantInt *CaseVal,
5226 SwitchCaseResultVectorTy &UniqueResults,
5227 Constant *Result) {
5228 for (auto &I : UniqueResults) {
5229 if (I.first == Result) {
5230 I.second.push_back(CaseVal);
5231 return I.second.size();
5232 }
5233 }
5234 UniqueResults.push_back(
5235 std::make_pair(Result, SmallVector<ConstantInt *, 4>(1, CaseVal)));
5236 return 1;
5237}
5238
5239// Helper function that initializes a map containing
5240// results for the PHI node of the common destination block for a switch
5241// instruction. Returns false if multiple PHI nodes have been found or if
5242// there is not a common destination block for the switch.
5243static bool
5244InitializeUniqueCases(SwitchInst *SI, PHINode *&PHI, BasicBlock *&CommonDest,
5245 SwitchCaseResultVectorTy &UniqueResults,
5246 Constant *&DefaultResult, const DataLayout &DL,
5247 const TargetTransformInfo &TTI,
5248 uintptr_t MaxUniqueResults, uintptr_t MaxCasesPerResult) {
5249 for (auto &I : SI->cases()) {
5250 ConstantInt *CaseVal = I.getCaseValue();
5251
5252 // Resulting value at phi nodes for this case value.
5253 SwitchCaseResultsTy Results;
5254 if (!GetCaseResults(SI, CaseVal, I.getCaseSuccessor(), &CommonDest, Results,
5255 DL, TTI))
5256 return false;
5257
5258 // Only one value per case is permitted.
5259 if (Results.size() > 1)
5260 return false;
5261
5262 // Add the case->result mapping to UniqueResults.
5263 const uintptr_t NumCasesForResult =
5264 MapCaseToResult(CaseVal, UniqueResults, Results.begin()->second);
5265
5266 // Early out if there are too many cases for this result.
5267 if (NumCasesForResult > MaxCasesPerResult)
5268 return false;
5269
5270 // Early out if there are too many unique results.
5271 if (UniqueResults.size() > MaxUniqueResults)
5272 return false;
5273
5274 // Check the PHI consistency.
5275 if (!PHI)
5276 PHI = Results[0].first;
5277 else if (PHI != Results[0].first)
5278 return false;
5279 }
5280 // Find the default result value.
5281 SmallVector<std::pair<PHINode *, Constant *>, 1> DefaultResults;
5282 BasicBlock *DefaultDest = SI->getDefaultDest();
5283 GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, DefaultResults,
5284 DL, TTI);
5285 // If the default value is not found abort unless the default destination
5286 // is unreachable.
5287 DefaultResult =
5288 DefaultResults.size() == 1 ? DefaultResults.begin()->second : nullptr;
5289 if ((!DefaultResult &&
5290 !isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg())))
5291 return false;
5292
5293 return true;
5294}
5295
5296// Helper function that checks if it is possible to transform a switch with only
5297// two cases (or two cases + default) that produces a result into a select.
5298// Example:
5299// switch (a) {
5300// case 10: %0 = icmp eq i32 %a, 10
5301// return 10; %1 = select i1 %0, i32 10, i32 4
5302// case 20: ----> %2 = icmp eq i32 %a, 20
5303// return 2; %3 = select i1 %2, i32 2, i32 %1
5304// default:
5305// return 4;
5306// }
5307static Value *ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector,
5308 Constant *DefaultResult, Value *Condition,
5309 IRBuilder<> &Builder) {
5310 // If we are selecting between only two cases transform into a simple
5311 // select or a two-way select if default is possible.
5312 if (ResultVector.size() == 2 && ResultVector[0].second.size() == 1 &&
5313 ResultVector[1].second.size() == 1) {
5314 ConstantInt *const FirstCase = ResultVector[0].second[0];
5315 ConstantInt *const SecondCase = ResultVector[1].second[0];
5316
5317 bool DefaultCanTrigger = DefaultResult;
5318 Value *SelectValue = ResultVector[1].first;
5319 if (DefaultCanTrigger) {
5320 Value *const ValueCompare =
5321 Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp");
5322 SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first,
5323 DefaultResult, "switch.select");
5324 }
5325 Value *const ValueCompare =
5326 Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp");
5327 return Builder.CreateSelect(ValueCompare, ResultVector[0].first,
5328 SelectValue, "switch.select");
5329 }
5330
5331 // Handle the degenerate case where two cases have the same value.
5332 if (ResultVector.size() == 1 && ResultVector[0].second.size() == 2 &&
5333 DefaultResult) {
5334 Value *Cmp1 = Builder.CreateICmpEQ(
5335 Condition, ResultVector[0].second[0], "switch.selectcmp.case1");
5336 Value *Cmp2 = Builder.CreateICmpEQ(
5337 Condition, ResultVector[0].second[1], "switch.selectcmp.case2");
5338 Value *Cmp = Builder.CreateOr(Cmp1, Cmp2, "switch.selectcmp");
5339 return Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult);
5340 }
5341
5342 return nullptr;
5343}
5344
5345// Helper function to cleanup a switch instruction that has been converted into
5346// a select, fixing up PHI nodes and basic blocks.
5347static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI,
5348 Value *SelectValue,
5349 IRBuilder<> &Builder,
5350 DomTreeUpdater *DTU) {
5351 std::vector<DominatorTree::UpdateType> Updates;
5352
5353 BasicBlock *SelectBB = SI->getParent();
5354 BasicBlock *DestBB = PHI->getParent();
5355
5356 if (DTU && !is_contained(predecessors(DestBB), SelectBB))
5357 Updates.push_back({DominatorTree::Insert, SelectBB, DestBB});
5358 Builder.CreateBr(DestBB);
5359
5360 // Remove the switch.
5361
5362 while (PHI->getBasicBlockIndex(SelectBB) >= 0)
5363 PHI->removeIncomingValue(SelectBB);
5364 PHI->addIncoming(SelectValue, SelectBB);
5365
5366 SmallPtrSet<BasicBlock *, 4> RemovedSuccessors;
5367 for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
5368 BasicBlock *Succ = SI->getSuccessor(i);
5369
5370 if (Succ == DestBB)
5371 continue;
5372 Succ->removePredecessor(SelectBB);
5373 if (DTU && RemovedSuccessors.insert(Succ).second)
5374 Updates.push_back({DominatorTree::Delete, SelectBB, Succ});
5375 }
5376 SI->eraseFromParent();
5377 if (DTU)
5378 DTU->applyUpdates(Updates);
5379}
5380
5381/// If the switch is only used to initialize one or more
5382/// phi nodes in a common successor block with only two different
5383/// constant values, replace the switch with select.
5384static bool switchToSelect(SwitchInst *SI, IRBuilder<> &Builder,
5385 DomTreeUpdater *DTU, const DataLayout &DL,
5386 const TargetTransformInfo &TTI) {
5387 Value *const Cond = SI->getCondition();
5388 PHINode *PHI = nullptr;
5389 BasicBlock *CommonDest = nullptr;
5390 Constant *DefaultResult;
5391 SwitchCaseResultVectorTy UniqueResults;
5392 // Collect all the cases that will deliver the same value from the switch.
5393 if (!InitializeUniqueCases(SI, PHI, CommonDest, UniqueResults, DefaultResult,
5394 DL, TTI, /*MaxUniqueResults*/2,
5395 /*MaxCasesPerResult*/2))
5396 return false;
5397 assert(PHI != nullptr && "PHI for value select not found")((PHI != nullptr && "PHI for value select not found")
? static_cast<void> (0) : __assert_fail ("PHI != nullptr && \"PHI for value select not found\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5397, __PRETTY_FUNCTION__))
;
5398
5399 Builder.SetInsertPoint(SI);
5400 Value *SelectValue =
5401 ConvertTwoCaseSwitch(UniqueResults, DefaultResult, Cond, Builder);
5402 if (SelectValue) {
5403 RemoveSwitchAfterSelectConversion(SI, PHI, SelectValue, Builder, DTU);
5404 return true;
5405 }
5406 // The switch couldn't be converted into a select.
5407 return false;
5408}
5409
5410namespace {
5411
5412/// This class represents a lookup table that can be used to replace a switch.
5413class SwitchLookupTable {
5414public:
5415 /// Create a lookup table to use as a switch replacement with the contents
5416 /// of Values, using DefaultValue to fill any holes in the table.
5417 SwitchLookupTable(
5418 Module &M, uint64_t TableSize, ConstantInt *Offset,
5419 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
5420 Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName);
5421
5422 /// Build instructions with Builder to retrieve the value at
5423 /// the position given by Index in the lookup table.
5424 Value *BuildLookup(Value *Index, IRBuilder<> &Builder);
5425
5426 /// Return true if a table with TableSize elements of
5427 /// type ElementType would fit in a target-legal register.
5428 static bool WouldFitInRegister(const DataLayout &DL, uint64_t TableSize,
5429 Type *ElementType);
5430
5431private:
5432 // Depending on the contents of the table, it can be represented in
5433 // different ways.
5434 enum {
5435 // For tables where each element contains the same value, we just have to
5436 // store that single value and return it for each lookup.
5437 SingleValueKind,
5438
5439 // For tables where there is a linear relationship between table index
5440 // and values. We calculate the result with a simple multiplication
5441 // and addition instead of a table lookup.
5442 LinearMapKind,
5443
5444 // For small tables with integer elements, we can pack them into a bitmap
5445 // that fits into a target-legal register. Values are retrieved by
5446 // shift and mask operations.
5447 BitMapKind,
5448
5449 // The table is stored as an array of values. Values are retrieved by load
5450 // instructions from the table.
5451 ArrayKind
5452 } Kind;
5453
5454 // For SingleValueKind, this is the single value.
5455 Constant *SingleValue = nullptr;
5456
5457 // For BitMapKind, this is the bitmap.
5458 ConstantInt *BitMap = nullptr;
5459 IntegerType *BitMapElementTy = nullptr;
5460
5461 // For LinearMapKind, these are the constants used to derive the value.
5462 ConstantInt *LinearOffset = nullptr;
5463 ConstantInt *LinearMultiplier = nullptr;
5464
5465 // For ArrayKind, this is the array.
5466 GlobalVariable *Array = nullptr;
5467};
5468
5469} // end anonymous namespace
5470
5471SwitchLookupTable::SwitchLookupTable(
5472 Module &M, uint64_t TableSize, ConstantInt *Offset,
5473 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
5474 Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName) {
5475 assert(Values.size() && "Can't build lookup table without values!")((Values.size() && "Can't build lookup table without values!"
) ? static_cast<void> (0) : __assert_fail ("Values.size() && \"Can't build lookup table without values!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5475, __PRETTY_FUNCTION__))
;
5476 assert(TableSize >= Values.size() && "Can't fit values in table!")((TableSize >= Values.size() && "Can't fit values in table!"
) ? static_cast<void> (0) : __assert_fail ("TableSize >= Values.size() && \"Can't fit values in table!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5476, __PRETTY_FUNCTION__))
;
5477
5478 // If all values in the table are equal, this is that value.
5479 SingleValue = Values.begin()->second;
5480
5481 Type *ValueType = Values.begin()->second->getType();
5482
5483 // Build up the table contents.
5484 SmallVector<Constant *, 64> TableContents(TableSize);
5485 for (size_t I = 0, E = Values.size(); I != E; ++I) {
5486 ConstantInt *CaseVal = Values[I].first;
5487 Constant *CaseRes = Values[I].second;
5488 assert(CaseRes->getType() == ValueType)((CaseRes->getType() == ValueType) ? static_cast<void>
(0) : __assert_fail ("CaseRes->getType() == ValueType", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5488, __PRETTY_FUNCTION__))
;
5489
5490 uint64_t Idx = (CaseVal->getValue() - Offset->getValue()).getLimitedValue();
5491 TableContents[Idx] = CaseRes;
5492
5493 if (CaseRes != SingleValue)
5494 SingleValue = nullptr;
5495 }
5496
5497 // Fill in any holes in the table with the default result.
5498 if (Values.size() < TableSize) {
5499 assert(DefaultValue &&((DefaultValue && "Need a default value to fill the lookup table holes."
) ? static_cast<void> (0) : __assert_fail ("DefaultValue && \"Need a default value to fill the lookup table holes.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5500, __PRETTY_FUNCTION__))
5500 "Need a default value to fill the lookup table holes.")((DefaultValue && "Need a default value to fill the lookup table holes."
) ? static_cast<void> (0) : __assert_fail ("DefaultValue && \"Need a default value to fill the lookup table holes.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5500, __PRETTY_FUNCTION__))
;
5501 assert(DefaultValue->getType() == ValueType)((DefaultValue->getType() == ValueType) ? static_cast<void
> (0) : __assert_fail ("DefaultValue->getType() == ValueType"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5501, __PRETTY_FUNCTION__))
;
5502 for (uint64_t I = 0; I < TableSize; ++I) {
5503 if (!TableContents[I])
5504 TableContents[I] = DefaultValue;
5505 }
5506
5507 if (DefaultValue != SingleValue)
5508 SingleValue = nullptr;
5509 }
5510
5511 // If each element in the table contains the same value, we only need to store
5512 // that single value.
5513 if (SingleValue) {
5514 Kind = SingleValueKind;
5515 return;
5516 }
5517
5518 // Check if we can derive the value with a linear transformation from the
5519 // table index.
5520 if (isa<IntegerType>(ValueType)) {
5521 bool LinearMappingPossible = true;
5522 APInt PrevVal;
5523 APInt DistToPrev;
5524 assert(TableSize >= 2 && "Should be a SingleValue table.")((TableSize >= 2 && "Should be a SingleValue table."
) ? static_cast<void> (0) : __assert_fail ("TableSize >= 2 && \"Should be a SingleValue table.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5524, __PRETTY_FUNCTION__))
;
5525 // Check if there is the same distance between two consecutive values.
5526 for (uint64_t I = 0; I < TableSize; ++I) {
5527 ConstantInt *ConstVal = dyn_cast<ConstantInt>(TableContents[I]);
5528 if (!ConstVal) {
5529 // This is an undef. We could deal with it, but undefs in lookup tables
5530 // are very seldom. It's probably not worth the additional complexity.
5531 LinearMappingPossible = false;
5532 break;
5533 }
5534 const APInt &Val = ConstVal->getValue();
5535 if (I != 0) {
5536 APInt Dist = Val - PrevVal;
5537 if (I == 1) {
5538 DistToPrev = Dist;
5539 } else if (Dist != DistToPrev) {
5540 LinearMappingPossible = false;
5541 break;
5542 }
5543 }
5544 PrevVal = Val;
5545 }
5546 if (LinearMappingPossible) {
5547 LinearOffset = cast<ConstantInt>(TableContents[0]);
5548 LinearMultiplier = ConstantInt::get(M.getContext(), DistToPrev);
5549 Kind = LinearMapKind;
5550 ++NumLinearMaps;
5551 return;
5552 }
5553 }
5554
5555 // If the type is integer and the table fits in a register, build a bitmap.
5556 if (WouldFitInRegister(DL, TableSize, ValueType)) {
5557 IntegerType *IT = cast<IntegerType>(ValueType);
5558 APInt TableInt(TableSize * IT->getBitWidth(), 0);
5559 for (uint64_t I = TableSize; I > 0; --I) {
5560 TableInt <<= IT->getBitWidth();
5561 // Insert values into the bitmap. Undef values are set to zero.
5562 if (!isa<UndefValue>(TableContents[I - 1])) {
5563 ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]);
5564 TableInt |= Val->getValue().zext(TableInt.getBitWidth());
5565 }
5566 }
5567 BitMap = ConstantInt::get(M.getContext(), TableInt);
5568 BitMapElementTy = IT;
5569 Kind = BitMapKind;
5570 ++NumBitMaps;
5571 return;
5572 }
5573
5574 // Store the table in an array.
5575 ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize);
5576 Constant *Initializer = ConstantArray::get(ArrayTy, TableContents);
5577
5578 Array = new GlobalVariable(M, ArrayTy, /*isConstant=*/true,
5579 GlobalVariable::PrivateLinkage, Initializer,
5580 "switch.table." + FuncName);
5581 Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
5582 // Set the alignment to that of an array items. We will be only loading one
5583 // value out of it.
5584 Array->setAlignment(Align(DL.getPrefTypeAlignment(ValueType)));
5585 Kind = ArrayKind;
5586}
5587
5588Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) {
5589 switch (Kind) {
5590 case SingleValueKind:
5591 return SingleValue;
5592 case LinearMapKind: {
5593 // Derive the result value from the input value.
5594 Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(),
5595 false, "switch.idx.cast");
5596 if (!LinearMultiplier->isOne())
5597 Result = Builder.CreateMul(Result, LinearMultiplier, "switch.idx.mult");
5598 if (!LinearOffset->isZero())
5599 Result = Builder.CreateAdd(Result, LinearOffset, "switch.offset");
5600 return Result;
5601 }
5602 case BitMapKind: {
5603 // Type of the bitmap (e.g. i59).
5604 IntegerType *MapTy = BitMap->getType();
5605
5606 // Cast Index to the same type as the bitmap.
5607 // Note: The Index is <= the number of elements in the table, so
5608 // truncating it to the width of the bitmask is safe.
5609 Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast");
5610
5611 // Multiply the shift amount by the element width.
5612 ShiftAmt = Builder.CreateMul(
5613 ShiftAmt, ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()),
5614 "switch.shiftamt");
5615
5616 // Shift down.
5617 Value *DownShifted =
5618 Builder.CreateLShr(BitMap, ShiftAmt, "switch.downshift");
5619 // Mask off.
5620 return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked");
5621 }
5622 case ArrayKind: {
5623 // Make sure the table index will not overflow when treated as signed.
5624 IntegerType *IT = cast<IntegerType>(Index->getType());
5625 uint64_t TableSize =
5626 Array->getInitializer()->getType()->getArrayNumElements();
5627 if (TableSize > (1ULL << (IT->getBitWidth() - 1)))
5628 Index = Builder.CreateZExt(
5629 Index, IntegerType::get(IT->getContext(), IT->getBitWidth() + 1),
5630 "switch.tableidx.zext");
5631
5632 Value *GEPIndices[] = {Builder.getInt32(0), Index};
5633 Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array,
5634 GEPIndices, "switch.gep");
5635 return Builder.CreateLoad(
5636 cast<ArrayType>(Array->getValueType())->getElementType(), GEP,
5637 "switch.load");
5638 }
5639 }
5640 llvm_unreachable("Unknown lookup table kind!")::llvm::llvm_unreachable_internal("Unknown lookup table kind!"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5640)
;
5641}
5642
5643bool SwitchLookupTable::WouldFitInRegister(const DataLayout &DL,
5644 uint64_t TableSize,
5645 Type *ElementType) {
5646 auto *IT = dyn_cast<IntegerType>(ElementType);
5647 if (!IT)
5648 return false;
5649 // FIXME: If the type is wider than it needs to be, e.g. i8 but all values
5650 // are <= 15, we could try to narrow the type.
5651
5652 // Avoid overflow, fitsInLegalInteger uses unsigned int for the width.
5653 if (TableSize >= UINT_MAX(2147483647 *2U +1U) / IT->getBitWidth())
5654 return false;
5655 return DL.fitsInLegalInteger(TableSize * IT->getBitWidth());
5656}
5657
5658/// Determine whether a lookup table should be built for this switch, based on
5659/// the number of cases, size of the table, and the types of the results.
5660static bool
5661ShouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize,
5662 const TargetTransformInfo &TTI, const DataLayout &DL,
5663 const SmallDenseMap<PHINode *, Type *> &ResultTypes) {
5664 if (SI->getNumCases() > TableSize || TableSize >= UINT64_MAX(18446744073709551615UL) / 10)
5665 return false; // TableSize overflowed, or mul below might overflow.
5666
5667 bool AllTablesFitInRegister = true;
5668 bool HasIllegalType = false;
5669 for (const auto &I : ResultTypes) {
5670 Type *Ty = I.second;
5671
5672 // Saturate this flag to true.
5673 HasIllegalType = HasIllegalType || !TTI.isTypeLegal(Ty);
5674
5675 // Saturate this flag to false.
5676 AllTablesFitInRegister =
5677 AllTablesFitInRegister &&
5678 SwitchLookupTable::WouldFitInRegister(DL, TableSize, Ty);
5679
5680 // If both flags saturate, we're done. NOTE: This *only* works with
5681 // saturating flags, and all flags have to saturate first due to the
5682 // non-deterministic behavior of iterating over a dense map.
5683 if (HasIllegalType && !AllTablesFitInRegister)
5684 break;
5685 }
5686
5687 // If each table would fit in a register, we should build it anyway.
5688 if (AllTablesFitInRegister)
5689 return true;
5690
5691 // Don't build a table that doesn't fit in-register if it has illegal types.
5692 if (HasIllegalType)
5693 return false;
5694
5695 // The table density should be at least 40%. This is the same criterion as for
5696 // jump tables, see SelectionDAGBuilder::handleJTSwitchCase.
5697 // FIXME: Find the best cut-off.
5698 return SI->getNumCases() * 10 >= TableSize * 4;
5699}
5700
5701/// Try to reuse the switch table index compare. Following pattern:
5702/// \code
5703/// if (idx < tablesize)
5704/// r = table[idx]; // table does not contain default_value
5705/// else
5706/// r = default_value;
5707/// if (r != default_value)
5708/// ...
5709/// \endcode
5710/// Is optimized to:
5711/// \code
5712/// cond = idx < tablesize;
5713/// if (cond)
5714/// r = table[idx];
5715/// else
5716/// r = default_value;
5717/// if (cond)
5718/// ...
5719/// \endcode
5720/// Jump threading will then eliminate the second if(cond).
5721static void reuseTableCompare(
5722 User *PhiUser, BasicBlock *PhiBlock, BranchInst *RangeCheckBranch,
5723 Constant *DefaultValue,
5724 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values) {
5725 ICmpInst *CmpInst = dyn_cast<ICmpInst>(PhiUser);
5726 if (!CmpInst)
5727 return;
5728
5729 // We require that the compare is in the same block as the phi so that jump
5730 // threading can do its work afterwards.
5731 if (CmpInst->getParent() != PhiBlock)
5732 return;
5733
5734 Constant *CmpOp1 = dyn_cast<Constant>(CmpInst->getOperand(1));
5735 if (!CmpOp1)
5736 return;
5737
5738 Value *RangeCmp = RangeCheckBranch->getCondition();
5739 Constant *TrueConst = ConstantInt::getTrue(RangeCmp->getType());
5740 Constant *FalseConst = ConstantInt::getFalse(RangeCmp->getType());
5741
5742 // Check if the compare with the default value is constant true or false.
5743 Constant *DefaultConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
5744 DefaultValue, CmpOp1, true);
5745 if (DefaultConst != TrueConst && DefaultConst != FalseConst)
5746 return;
5747
5748 // Check if the compare with the case values is distinct from the default
5749 // compare result.
5750 for (auto ValuePair : Values) {
5751 Constant *CaseConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
5752 ValuePair.second, CmpOp1, true);
5753 if (!CaseConst || CaseConst == DefaultConst || isa<UndefValue>(CaseConst))
5754 return;
5755 assert((CaseConst == TrueConst || CaseConst == FalseConst) &&(((CaseConst == TrueConst || CaseConst == FalseConst) &&
"Expect true or false as compare result.") ? static_cast<
void> (0) : __assert_fail ("(CaseConst == TrueConst || CaseConst == FalseConst) && \"Expect true or false as compare result.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5756, __PRETTY_FUNCTION__))
5756 "Expect true or false as compare result.")(((CaseConst == TrueConst || CaseConst == FalseConst) &&
"Expect true or false as compare result.") ? static_cast<
void> (0) : __assert_fail ("(CaseConst == TrueConst || CaseConst == FalseConst) && \"Expect true or false as compare result.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5756, __PRETTY_FUNCTION__))
;
5757 }
5758
5759 // Check if the branch instruction dominates the phi node. It's a simple
5760 // dominance check, but sufficient for our needs.
5761 // Although this check is invariant in the calling loops, it's better to do it
5762 // at this late stage. Practically we do it at most once for a switch.
5763 BasicBlock *BranchBlock = RangeCheckBranch->getParent();
5764 for (BasicBlock *Pred : predecessors(PhiBlock)) {
5765 if (Pred != BranchBlock && Pred->getUniquePredecessor() != BranchBlock)
5766 return;
5767 }
5768
5769 if (DefaultConst == FalseConst) {
5770 // The compare yields the same result. We can replace it.
5771 CmpInst->replaceAllUsesWith(RangeCmp);
5772 ++NumTableCmpReuses;
5773 } else {
5774 // The compare yields the same result, just inverted. We can replace it.
5775 Value *InvertedTableCmp = BinaryOperator::CreateXor(
5776 RangeCmp, ConstantInt::get(RangeCmp->getType(), 1), "inverted.cmp",
5777 RangeCheckBranch);
5778 CmpInst->replaceAllUsesWith(InvertedTableCmp);
5779 ++NumTableCmpReuses;
5780 }
5781}
5782
5783/// If the switch is only used to initialize one or more phi nodes in a common
5784/// successor block with different constant values, replace the switch with
5785/// lookup tables.
5786static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
5787 DomTreeUpdater *DTU, const DataLayout &DL,
5788 const TargetTransformInfo &TTI) {
5789 assert(SI->getNumCases() > 1 && "Degenerate switch?")((SI->getNumCases() > 1 && "Degenerate switch?"
) ? static_cast<void> (0) : __assert_fail ("SI->getNumCases() > 1 && \"Degenerate switch?\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5789, __PRETTY_FUNCTION__))
;
5790
5791 BasicBlock *BB = SI->getParent();
5792 Function *Fn = BB->getParent();
5793 // Only build lookup table when we have a target that supports it or the
5794 // attribute is not set.
5795 if (!TTI.shouldBuildLookupTables() ||
5796 (Fn->getFnAttribute("no-jump-tables").getValueAsString() == "true"))
5797 return false;
5798
5799 // FIXME: If the switch is too sparse for a lookup table, perhaps we could
5800 // split off a dense part and build a lookup table for that.
5801
5802 // FIXME: This creates arrays of GEPs to constant strings, which means each
5803 // GEP needs a runtime relocation in PIC code. We should just build one big
5804 // string and lookup indices into that.
5805
5806 // Ignore switches with less than three cases. Lookup tables will not make
5807 // them faster, so we don't analyze them.
5808 if (SI->getNumCases() < 3)
5809 return false;
5810
5811 // Figure out the corresponding result for each case value and phi node in the
5812 // common destination, as well as the min and max case values.
5813 assert(!SI->cases().empty())((!SI->cases().empty()) ? static_cast<void> (0) : __assert_fail
("!SI->cases().empty()", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Utils/SimplifyCFG.cpp"
, 5813, __PRETTY_FUNCTION__))
;
5814 SwitchInst::CaseIt CI = SI->case_begin();
5815 ConstantInt *MinCaseVal = CI->getCaseValue();
5816 ConstantInt *MaxCaseVal = CI->getCaseValue();
5817
5818 BasicBlock *CommonDest = nullptr;
5819
5820 using ResultListTy = SmallVector<std::pair<ConstantInt *, Constant *>, 4>;
5821 SmallDenseMap<PHINode *, ResultListTy> ResultLists;
5822
5823 SmallDenseMap<PHINode *, Constant *> DefaultResults;
5824 SmallDenseMap<PHINode *, Type *> ResultTypes;
5825 SmallVector<PHINode *, 4> PHIs;
5826
5827 for (SwitchInst::CaseIt E = SI->case_end(); CI != E; ++CI) {
5828 ConstantInt *CaseVal = CI->getCaseValue();
5829 if (CaseVal->getValue().slt(MinCaseVal->getValue()))
5830 MinCaseVal = CaseVal;
5831 if (CaseVal->getValue().sgt(MaxCaseVal->getValue()))
5832 MaxCaseVal = CaseVal;
5833
5834 // Resulting value at phi nodes for this case value.
5835 using ResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
5836 ResultsTy Results;
5837 if (!GetCaseResults(SI, CaseVal, CI->getCaseSuccessor(), &CommonDest,
5838 Results, DL, TTI))
5839 return false;
5840
5841 // Append the result from this case to the list for each phi.
5842 for (const auto &I : Results) {
5843 PHINode *PHI = I.first;
5844 Constant *Value = I.second;
5845 if (!ResultLists.count(PHI))
5846 PHIs.push_back(PHI);
5847 ResultLists[PHI].push_back(std::make_pair(CaseVal, Value));
5848 }
5849 }
5850
5851 // Keep track of the result types.
5852 for (PHINode *PHI : PHIs) {
5853 ResultTypes[PHI] = ResultLists[PHI][0].second->getType();
5854 }
5855