LLVM 24.0.0git
SimplifyCFG.cpp
Go to the documentation of this file.
1//===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Peephole optimize the CFG.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APInt.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/Sequence.h"
20#include "llvm/ADT/SetVector.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/ADT/StringRef.h"
31#include "llvm/Analysis/Loads.h"
36#include "llvm/IR/Attributes.h"
37#include "llvm/IR/BasicBlock.h"
38#include "llvm/IR/CFG.h"
39#include "llvm/IR/Constant.h"
41#include "llvm/IR/Constants.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/DebugInfo.h"
45#include "llvm/IR/Function.h"
46#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/IRBuilder.h"
49#include "llvm/IR/InstrTypes.h"
50#include "llvm/IR/Instruction.h"
53#include "llvm/IR/LLVMContext.h"
54#include "llvm/IR/MDBuilder.h"
56#include "llvm/IR/Metadata.h"
57#include "llvm/IR/Module.h"
58#include "llvm/IR/NoFolder.h"
59#include "llvm/IR/Operator.h"
62#include "llvm/IR/Type.h"
63#include "llvm/IR/Use.h"
64#include "llvm/IR/User.h"
65#include "llvm/IR/Value.h"
66#include "llvm/IR/ValueHandle.h"
70#include "llvm/Support/Debug.h"
80#include <algorithm>
81#include <cassert>
82#include <climits>
83#include <cstddef>
84#include <cstdint>
85#include <iterator>
86#include <map>
87#include <optional>
88#include <set>
89#include <tuple>
90#include <utility>
91#include <vector>
92
93using namespace llvm;
94using namespace PatternMatch;
95
96#define DEBUG_TYPE "simplifycfg"
97
98namespace llvm {
99
101 "simplifycfg-require-and-preserve-domtree", cl::Hidden,
102
103 cl::desc(
104 "Temporary development switch used to gradually uplift SimplifyCFG "
105 "into preserving DomTree,"));
106
107// Chosen as 2 so as to be cheap, but still to have enough power to fold
108// a select, so the "clamp" idiom (of a min followed by a max) will be caught.
109// To catch this, we need to fold a compare and a select, hence '2' being the
110// minimum reasonable default.
112 "phi-node-folding-threshold", cl::Hidden, cl::init(2),
113 cl::desc(
114 "Control the amount of phi node folding to perform (default = 2)"));
115
117 "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4),
118 cl::desc("Control the maximal total instruction cost that we are willing "
119 "to speculatively execute to fold a 2-entry PHI node into a "
120 "select (default = 4)"));
121
122static cl::opt<bool>
123 HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true),
124 cl::desc("Hoist common instructions up to the parent block"));
125
127 "simplifycfg-hoist-loads-with-cond-faulting", cl::Hidden, cl::init(true),
128 cl::desc("Hoist loads if the target supports conditional faulting"));
129
131 "simplifycfg-hoist-stores-with-cond-faulting", cl::Hidden, cl::init(true),
132 cl::desc("Hoist stores if the target supports conditional faulting"));
133
135 "hoist-loads-stores-with-cond-faulting-threshold", cl::Hidden, cl::init(6),
136 cl::desc("Control the maximal conditional load/store that we are willing "
137 "to speculatively execute to eliminate conditional branch "
138 "(default = 6)"));
139
141 HoistCommonSkipLimit("simplifycfg-hoist-common-skip-limit", cl::Hidden,
142 cl::init(20),
143 cl::desc("Allow reordering across at most this many "
144 "instructions when hoisting"));
145
146static cl::opt<bool>
147 SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
148 cl::desc("Sink common instructions down to the end block"));
149
151 "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true),
152 cl::desc("Hoist conditional stores if an unconditional store precedes"));
153
155 "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true),
156 cl::desc("Hoist conditional stores even if an unconditional store does not "
157 "precede - hoist multiple conditional stores into a single "
158 "predicated store"));
159
161 "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false),
162 cl::desc("When merging conditional stores, do so even if the resultant "
163 "basic blocks are unlikely to be if-converted as a result"));
164
166 "speculate-one-expensive-inst", cl::Hidden, cl::init(true),
167 cl::desc("Allow exactly one expensive instruction to be speculatively "
168 "executed"));
169
171 "max-speculation-depth", cl::Hidden, cl::init(10),
172 cl::desc("Limit maximum recursion depth when calculating costs of "
173 "speculatively executed instructions"));
174
175static cl::opt<int>
176 MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden,
177 cl::init(10),
178 cl::desc("Max size of a block which is still considered "
179 "small enough to thread through"));
180
181// Two is chosen to allow one negation and a logical combine.
183 BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden,
184 cl::init(2),
185 cl::desc("Maximum cost of combining conditions when "
186 "folding branches"));
187
189 "simplifycfg-branch-fold-common-dest-vector-multiplier", cl::Hidden,
190 cl::init(2),
191 cl::desc("Multiplier to apply to threshold when determining whether or not "
192 "to fold branch to common destination when vector operations are "
193 "present"));
194
196 "simplifycfg-merge-compatible-invokes", cl::Hidden, cl::init(true),
197 cl::desc("Allow SimplifyCFG to merge invokes together when appropriate"));
198
200 "max-switch-cases-per-result", cl::Hidden, cl::init(16),
201 cl::desc("Limit cases to analyze when converting a switch to select"));
202
204 "simplifycfg-max-loop-header-threading-search", cl::Hidden, cl::init(1024),
205 cl::desc("Limit number of blocks searched to prove that threading through "
206 "a loop header keeps the loop canonical"));
207
209 "max-jump-threading-live-blocks", cl::Hidden, cl::init(24),
210 cl::desc("Limit number of blocks a define in a threaded block is allowed "
211 "to be live in"));
212
214
215} // end namespace llvm
216
217STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps");
218STATISTIC(NumLinearMaps,
219 "Number of switch instructions turned into linear mapping");
220STATISTIC(NumLookupTables,
221 "Number of switch instructions turned into lookup tables");
223 NumLookupTablesHoles,
224 "Number of switch instructions turned into lookup tables (holes checked)");
225STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares");
226STATISTIC(NumFoldValueComparisonIntoPredecessors,
227 "Number of value comparisons folded into predecessor basic blocks");
228STATISTIC(NumFoldBranchToCommonDest,
229 "Number of branches folded into predecessor basic block");
231 NumHoistCommonCode,
232 "Number of common instruction 'blocks' hoisted up to the begin block");
233STATISTIC(NumHoistCommonInstrs,
234 "Number of common instructions hoisted up to the begin block");
235STATISTIC(NumSinkCommonCode,
236 "Number of common instruction 'blocks' sunk down to the end block");
237STATISTIC(NumSinkCommonInstrs,
238 "Number of common instructions sunk down to the end block");
239STATISTIC(NumSpeculations, "Number of speculative executed instructions");
240STATISTIC(NumInvokes,
241 "Number of invokes with empty resume blocks simplified into calls");
242STATISTIC(NumInvokesMerged, "Number of invokes that were merged together");
243STATISTIC(NumInvokeSetsFormed, "Number of invoke sets that were formed");
244
245namespace {
246
247// The first field contains the value that the switch produces when a certain
248// case group is selected, and the second field is a vector containing the
249// cases composing the case group.
250using SwitchCaseResultVectorTy =
252
253// The first field contains the phi node that generates a result of the switch
254// and the second field contains the value generated for a certain case in the
255// switch for that PHI.
256using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
257
258/// Represents a case of a switch.
259struct ValueEqualityComparisonCase {
261 BasicBlock *Dest;
262
263 ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest)
264 : Value(Value), Dest(Dest) {}
265
266 bool operator<(ValueEqualityComparisonCase RHS) const {
267 // Comparing pointers is ok as we only rely on the order for uniquing.
268 return Value < RHS.Value;
269 }
270
271 bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; }
272};
273
274class SimplifyCFGOpt {
275 const TargetTransformInfo &TTI;
276 DomTreeUpdater *DTU;
277 const DataLayout &DL;
278 ArrayRef<WeakVH> LoopHeaders;
279 const SimplifyCFGOptions &Options;
280 bool Resimplify;
281
282 Value *isValueEqualityComparison(Instruction *TI);
283 BasicBlock *getValueEqualityComparisonCases(
284 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases);
285 bool simplifyEqualityComparisonWithOnlyPredecessor(Instruction *TI,
286 BasicBlock *Pred,
287 IRBuilder<> &Builder);
288 bool performValueComparisonIntoPredecessorFolding(Instruction *TI, Value *&CV,
289 Instruction *PTI,
290 IRBuilder<> &Builder);
291 bool foldValueComparisonIntoPredecessors(Instruction *TI,
292 IRBuilder<> &Builder);
293
294 bool simplifyResume(ResumeInst *RI, IRBuilder<> &Builder);
295 bool simplifySingleResume(ResumeInst *RI);
296 bool simplifyCommonResume(ResumeInst *RI);
297 bool simplifyCleanupReturn(CleanupReturnInst *RI);
298 bool simplifyUnreachable(UnreachableInst *UI);
299 bool simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder);
300 bool simplifyDuplicateSwitchArms(SwitchInst *SI, DomTreeUpdater *DTU);
301 bool simplifyIndirectBr(IndirectBrInst *IBI);
302 bool simplifyUncondBranch(UncondBrInst *BI, IRBuilder<> &Builder);
303 bool simplifyCondBranch(CondBrInst *BI, IRBuilder<> &Builder);
304 bool foldCondBranchOnValueKnownInPredecessor(CondBrInst *BI);
305
306 bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
307 IRBuilder<> &Builder);
308 bool tryToSimplifyUncondBranchWithICmpSelectInIt(ICmpInst *ICI,
309 SelectInst *Select,
310 IRBuilder<> &Builder);
311 bool hoistCommonCodeFromSuccessors(Instruction *TI, bool AllInstsEqOnly);
312 bool hoistSuccIdenticalTerminatorToSwitchOrIf(
313 Instruction *TI, Instruction *I1,
314 SmallVectorImpl<Instruction *> &OtherSuccTIs,
315 ArrayRef<BasicBlock *> UniqueSuccessors);
316 bool speculativelyExecuteBB(CondBrInst *BI, BasicBlock *ThenBB);
317 bool simplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
318 BasicBlock *TrueBB, BasicBlock *FalseBB,
319 uint32_t TrueWeight, uint32_t FalseWeight);
320 bool simplifyBranchOnICmpChain(CondBrInst *BI, IRBuilder<> &Builder,
321 const DataLayout &DL);
322 bool simplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select);
323 bool simplifySwitchOnSelectRemap(SwitchInst *SI, SelectInst *Select, Value *X,
324 ConstantInt *C, bool Negate);
325 bool simplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI);
326 bool turnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder);
327 bool simplifyDuplicatePredecessors(BasicBlock *Succ, DomTreeUpdater *DTU);
328
329public:
330 SimplifyCFGOpt(const TargetTransformInfo &TTI, DomTreeUpdater *DTU,
331 const DataLayout &DL, ArrayRef<WeakVH> LoopHeaders,
332 const SimplifyCFGOptions &Opts)
333 : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) {
334 assert((!DTU || !DTU->hasPostDomTree()) &&
335 "SimplifyCFG is not yet capable of maintaining validity of a "
336 "PostDomTree, so don't ask for it.");
337 }
338
339 bool simplifyOnce(BasicBlock *BB);
340 bool run(BasicBlock *BB);
341
342 // Helper to set Resimplify and return change indication.
343 bool requestResimplify() {
344 Resimplify = true;
345 return true;
346 }
347};
348
349// we synthesize a || b as select a, true, b
350// we synthesize a && b as select a, b, false
351// this function determines if SI is playing one of those roles.
352[[maybe_unused]] bool
353isSelectInRoleOfConjunctionOrDisjunction(const SelectInst *SI) {
354 return ((isa<ConstantInt>(SI->getTrueValue()) &&
355 (dyn_cast<ConstantInt>(SI->getTrueValue())->isOne())) ||
356 (isa<ConstantInt>(SI->getFalseValue()) &&
357 (dyn_cast<ConstantInt>(SI->getFalseValue())->isNullValue())));
358}
359
360} // end anonymous namespace
361
362/// Return true if all the PHI nodes in the basic block \p BB
363/// receive compatible (identical) incoming values when coming from
364/// all of the predecessor blocks that are specified in \p IncomingBlocks.
365///
366/// Note that if the values aren't exactly identical, but \p EquivalenceSet
367/// is provided, and *both* of the values are present in the set,
368/// then they are considered equal.
370 BasicBlock *BB, ArrayRef<BasicBlock *> IncomingBlocks,
371 SmallPtrSetImpl<Value *> *EquivalenceSet = nullptr) {
372 assert(IncomingBlocks.size() == 2 &&
373 "Only for a pair of incoming blocks at the time!");
374
375 // FIXME: it is okay if one of the incoming values is an `undef` value,
376 // iff the other incoming value is guaranteed to be a non-poison value.
377 // FIXME: it is okay if one of the incoming values is a `poison` value.
378 return all_of(BB->phis(), [IncomingBlocks, EquivalenceSet](PHINode &PN) {
379 Value *IV0 = PN.getIncomingValueForBlock(IncomingBlocks[0]);
380 Value *IV1 = PN.getIncomingValueForBlock(IncomingBlocks[1]);
381 if (IV0 == IV1)
382 return true;
383 if (EquivalenceSet && EquivalenceSet->contains(IV0) &&
384 EquivalenceSet->contains(IV1))
385 return true;
386 return false;
387 });
388}
389
390/// Return true if it is safe to merge these two
391/// terminator instructions together.
392static bool
394 SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) {
395 if (SI1 == SI2)
396 return false; // Can't merge with self!
397
398 // It is not safe to merge these two switch instructions if they have a common
399 // successor, and if that successor has a PHI node, and if *that* PHI node has
400 // conflicting incoming values from the two switch blocks.
401 BasicBlock *SI1BB = SI1->getParent();
402 BasicBlock *SI2BB = SI2->getParent();
403
405 bool Fail = false;
406 for (BasicBlock *Succ : successors(SI2BB)) {
407 if (!SI1Succs.count(Succ))
408 continue;
409 if (incomingValuesAreCompatible(Succ, {SI1BB, SI2BB}))
410 continue;
411 Fail = true;
412 if (FailBlocks)
413 FailBlocks->insert(Succ);
414 else
415 break;
416 }
417
418 return !Fail;
419}
420
421/// Update PHI nodes in Succ to indicate that there will now be entries in it
422/// from the 'NewPred' block. The values that will be flowing into the PHI nodes
423/// will be the same as those coming in from ExistPred, an existing predecessor
424/// of Succ.
425static void addPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
426 BasicBlock *ExistPred,
427 MemorySSAUpdater *MSSAU = nullptr) {
428 for (PHINode &PN : Succ->phis())
429 PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred);
430 if (MSSAU)
431 if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ))
432 MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred);
433}
434
435/// Compute an abstract "cost" of speculating the given instruction,
436/// which is assumed to be safe to speculate. TCC_Free means cheap,
437/// TCC_Basic means less cheap, and TCC_Expensive means prohibitively
438/// expensive.
440 const TargetTransformInfo &TTI) {
441 return TTI.getInstructionCost(I, TargetTransformInfo::TCK_SizeAndLatency);
442}
443
444/// If we have a merge point of an "if condition" as accepted above,
445/// return true if the specified value dominates the block. We don't handle
446/// the true generality of domination here, just a special case which works
447/// well enough for us.
448///
449/// If AggressiveInsts is non-null, and if V does not dominate BB, we check to
450/// see if V (which must be an instruction) and its recursive operands
451/// that do not dominate BB have a combined cost lower than Budget and
452/// are non-trapping. If both are true, the instruction is inserted into the
453/// set and true is returned.
454///
455/// The cost for most non-trapping instructions is defined as 1 except for
456/// Select whose cost is 2.
457///
458/// After this function returns, Cost is increased by the cost of
459/// V plus its non-dominating operands. If that cost is greater than
460/// Budget, false is returned and Cost is undefined.
462 Value *V, BasicBlock *BB, Instruction *InsertPt,
463 SmallPtrSetImpl<Instruction *> &AggressiveInsts, InstructionCost &Cost,
465 SmallPtrSetImpl<Instruction *> &ZeroCostInstructions, unsigned Depth = 0) {
466 // It is possible to hit a zero-cost cycle (phi/gep instructions for example),
467 // so limit the recursion depth.
468 // TODO: While this recursion limit does prevent pathological behavior, it
469 // would be better to track visited instructions to avoid cycles.
471 return false;
472
474 if (!I) {
475 // Non-instructions dominate all instructions and can be executed
476 // unconditionally.
477 return true;
478 }
479 BasicBlock *PBB = I->getParent();
480
481 // We don't want to allow weird loops that might have the "if condition" in
482 // the bottom of this block.
483 if (PBB == BB)
484 return false;
485
486 // If this instruction is defined in a block that contains an unconditional
487 // branch to BB, then it must be in the 'conditional' part of the "if
488 // statement". If not, it definitely dominates the region.
490 if (!BI || BI->getSuccessor() != BB)
491 return true;
492
493 // If we have seen this instruction before, don't count it again.
494 if (AggressiveInsts.count(I))
495 return true;
496
497 // Okay, it looks like the instruction IS in the "condition". Check to
498 // see if it's a cheap instruction to unconditionally compute, and if it
499 // only uses stuff defined outside of the condition. If so, hoist it out.
500 if (!isSafeToSpeculativelyExecute(I, InsertPt, AC))
501 return false;
502
503 // Overflow arithmetic instruction plus extract value are usually generated
504 // when a division is being replaced. But, in this case, the zero check may
505 // still be kept in the code. In that case it would be worth to hoist these
506 // two instruction out of the basic block. Let's treat this pattern as one
507 // single cheap instruction here!
508 WithOverflowInst *OverflowInst;
509 if (match(I, m_ExtractValue<1>(m_OneUse(m_WithOverflowInst(OverflowInst))))) {
510 ZeroCostInstructions.insert(OverflowInst);
511 Cost += 1;
512 } else if (!ZeroCostInstructions.contains(I))
513 Cost += computeSpeculationCost(I, TTI);
514
515 // Allow exactly one instruction to be speculated regardless of its cost
516 // (as long as it is safe to do so).
517 // This is intended to flatten the CFG even if the instruction is a division
518 // or other expensive operation. The speculation of an expensive instruction
519 // is expected to be undone in CodeGenPrepare if the speculation has not
520 // enabled further IR optimizations.
521 if (Cost > Budget &&
522 (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0 ||
523 !Cost.isValid()))
524 return false;
525
526 // Okay, we can only really hoist these out if their operands do
527 // not take us over the cost threshold.
528 for (Use &Op : I->operands())
529 if (!dominatesMergePoint(Op, BB, InsertPt, AggressiveInsts, Cost, Budget,
530 TTI, AC, ZeroCostInstructions, Depth + 1))
531 return false;
532 // Okay, it's safe to do this! Remember this instruction.
533 AggressiveInsts.insert(I);
534 return true;
535}
536
537/// Extract ConstantInt from value, looking through IntToPtr
538/// and PointerNullValue. Return NULL if value is not a constant int.
540 // Normal constant int.
542 if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy())
543 return CI;
544
545 // It is not safe to look through inttoptr or ptrtoint when using unstable
546 // pointer types.
547 if (DL.hasUnstableRepresentation(V->getType()))
548 return nullptr;
549
550 // This is some kind of pointer constant. Turn it into a pointer-sized
551 // ConstantInt if possible.
552 IntegerType *IntPtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType()));
553
554 // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
556 return ConstantInt::get(IntPtrTy, 0);
557
558 // IntToPtr const int, we can look through this if the semantics of
559 // inttoptr for this address space are a simple (truncating) bitcast.
561 if (CE->getOpcode() == Instruction::IntToPtr)
562 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) {
563 // The constant is very likely to have the right type already.
564 if (CI->getType() == IntPtrTy)
565 return CI;
566 else
567 return cast<ConstantInt>(
568 ConstantFoldIntegerCast(CI, IntPtrTy, /*isSigned=*/false, DL));
569 }
570 return nullptr;
571}
572
573namespace {
574
575/// Given a chain of or (||) or and (&&) comparison of a value against a
576/// constant, this will try to recover the information required for a switch
577/// structure.
578/// It will depth-first traverse the chain of comparison, seeking for patterns
579/// like %a == 12 or %a < 4 and combine them to produce a set of integer
580/// representing the different cases for the switch.
581/// Note that if the chain is composed of '||' it will build the set of elements
582/// that matches the comparisons (i.e. any of this value validate the chain)
583/// while for a chain of '&&' it will build the set elements that make the test
584/// fail.
585struct ConstantComparesGatherer {
586 const DataLayout &DL;
587
588 /// Value found for the switch comparison
589 Value *CompValue = nullptr;
590
591 /// Extra clause to be checked before the switch
592 Value *Extra = nullptr;
593
594 /// Set of integers to match in switch
596
597 /// Number of comparisons matched in the and/or chain
598 unsigned UsedICmps = 0;
599
600 /// If the elements in Vals matches the comparisons
601 bool IsEq = false;
602
603 // Used to check if the first matched CompValue shall be the Extra check.
604 bool IgnoreFirstMatch = false;
605 bool MultipleMatches = false;
606
607 /// Construct and compute the result for the comparison instruction Cond
608 ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) {
609 gather(Cond);
610 if (CompValue || !MultipleMatches)
611 return;
612 Extra = nullptr;
613 Vals.clear();
614 UsedICmps = 0;
615 IgnoreFirstMatch = true;
616 gather(Cond);
617 }
618
619 ConstantComparesGatherer(const ConstantComparesGatherer &) = delete;
620 ConstantComparesGatherer &
621 operator=(const ConstantComparesGatherer &) = delete;
622
623private:
624 /// Try to set the current value used for the comparison, it succeeds only if
625 /// it wasn't set before or if the new value is the same as the old one
626 bool setValueOnce(Value *NewVal) {
627 if (IgnoreFirstMatch) {
628 IgnoreFirstMatch = false;
629 return false;
630 }
631 if (CompValue && CompValue != NewVal) {
632 MultipleMatches = true;
633 return false;
634 }
635 CompValue = NewVal;
636 return true;
637 }
638
639 /// Try to match Instruction "I" as a comparison against a constant and
640 /// populates the array Vals with the set of values that match (or do not
641 /// match depending on isEQ).
642 /// Return false on failure. On success, the Value the comparison matched
643 /// against is placed in CompValue.
644 /// If CompValue is already set, the function is expected to fail if a match
645 /// is found but the value compared to is different.
646 bool matchInstruction(Instruction *I, bool isEQ) {
647 if (match(I, m_Not(m_Instruction(I))))
648 isEQ = !isEQ;
649
650 Value *Val;
651 if (match(I, m_NUWTrunc(m_Value(Val)))) {
652 // If we already have a value for the switch, it has to match!
653 if (!setValueOnce(Val))
654 return false;
655 UsedICmps++;
656 Vals.push_back(ConstantInt::get(cast<IntegerType>(Val->getType()), isEQ));
657 return true;
658 }
659 // If this is an icmp against a constant, handle this as one of the cases.
660 ICmpInst *ICI;
661 ConstantInt *C;
662 if (!((ICI = dyn_cast<ICmpInst>(I)) &&
663 (C = getConstantInt(I->getOperand(1), DL)))) {
664 return false;
665 }
666
667 Value *RHSVal;
668 const APInt *RHSC;
669
670 // Pattern match a special case
671 // (x & ~2^z) == y --> x == y || x == y|2^z
672 // This undoes a transformation done by instcombine to fuse 2 compares.
673 if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
674 // It's a little bit hard to see why the following transformations are
675 // correct. Here is a CVC3 program to verify them for 64-bit values:
676
677 /*
678 ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63);
679 x : BITVECTOR(64);
680 y : BITVECTOR(64);
681 z : BITVECTOR(64);
682 mask : BITVECTOR(64) = BVSHL(ONE, z);
683 QUERY( (y & ~mask = y) =>
684 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
685 );
686 QUERY( (y | mask = y) =>
687 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
688 );
689 */
690
691 // Please note that each pattern must be a dual implication (<--> or
692 // iff). One directional implication can create spurious matches. If the
693 // implication is only one-way, an unsatisfiable condition on the left
694 // side can imply a satisfiable condition on the right side. Dual
695 // implication ensures that satisfiable conditions are transformed to
696 // other satisfiable conditions and unsatisfiable conditions are
697 // transformed to other unsatisfiable conditions.
698
699 // Here is a concrete example of a unsatisfiable condition on the left
700 // implying a satisfiable condition on the right:
701 //
702 // mask = (1 << z)
703 // (x & ~mask) == y --> (x == y || x == (y | mask))
704 //
705 // Substituting y = 3, z = 0 yields:
706 // (x & -2) == 3 --> (x == 3 || x == 2)
707
708 // Pattern match a special case:
709 /*
710 QUERY( (y & ~mask = y) =>
711 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
712 );
713 */
714 if (match(ICI->getOperand(0),
715 m_And(m_Value(RHSVal), m_APInt(RHSC)))) {
716 APInt Mask = ~*RHSC;
717 if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) {
718 // If we already have a value for the switch, it has to match!
719 if (!setValueOnce(RHSVal))
720 return false;
721
722 Vals.push_back(C);
723 Vals.push_back(
724 ConstantInt::get(C->getContext(),
725 C->getValue() | Mask));
726 UsedICmps++;
727 return true;
728 }
729 }
730
731 // Pattern match a special case:
732 /*
733 QUERY( (y | mask = y) =>
734 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
735 );
736 */
737 if (match(ICI->getOperand(0),
738 m_Or(m_Value(RHSVal), m_APInt(RHSC)))) {
739 APInt Mask = *RHSC;
740 if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) {
741 // If we already have a value for the switch, it has to match!
742 if (!setValueOnce(RHSVal))
743 return false;
744
745 Vals.push_back(C);
746 Vals.push_back(ConstantInt::get(C->getContext(),
747 C->getValue() & ~Mask));
748 UsedICmps++;
749 return true;
750 }
751 }
752
753 // If we already have a value for the switch, it has to match!
754 if (!setValueOnce(ICI->getOperand(0)))
755 return false;
756
757 UsedICmps++;
758 Vals.push_back(C);
759 return true;
760 }
761
762 // If we have "x ult 3", for example, then we can add 0,1,2 to the set.
763 ConstantRange Span =
765
766 // Shift the range if the compare is fed by an add. This is the range
767 // compare idiom as emitted by instcombine.
768 Value *CandidateVal = I->getOperand(0);
769 if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) {
770 Span = Span.subtract(*RHSC);
771 CandidateVal = RHSVal;
772 }
773
774 // If this is an and/!= check, then we are looking to build the set of
775 // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into
776 // x != 0 && x != 1.
777 if (!isEQ)
778 Span = Span.inverse();
779
780 // If there are a ton of values, we don't want to make a ginormous switch.
781 if (Span.isSizeLargerThan(8) || Span.isEmptySet()) {
782 return false;
783 }
784
785 // If we already have a value for the switch, it has to match!
786 if (!setValueOnce(CandidateVal))
787 return false;
788
789 // Add all values from the range to the set
790 APInt Tmp = Span.getLower();
791 do
792 Vals.push_back(ConstantInt::get(I->getContext(), Tmp));
793 while (++Tmp != Span.getUpper());
794
795 UsedICmps++;
796 return true;
797 }
798
799 /// Given a potentially 'or'd or 'and'd together collection of icmp
800 /// eq/ne/lt/gt instructions that compare a value against a constant, extract
801 /// the value being compared, and stick the list constants into the Vals
802 /// vector.
803 /// One "Extra" case is allowed to differ from the other.
804 void gather(Value *V) {
805 Value *Op0, *Op1;
806 if (match(V, m_LogicalOr(m_Value(Op0), m_Value(Op1))))
807 IsEq = true;
808 else if (match(V, m_LogicalAnd(m_Value(Op0), m_Value(Op1))))
809 IsEq = false;
810 else
811 return;
812 // Keep a stack (SmallVector for efficiency) for depth-first traversal
813 SmallVector<Value *, 8> DFT{Op0, Op1};
814 SmallPtrSet<Value *, 8> Visited{V, Op0, Op1};
815
816 while (!DFT.empty()) {
817 V = DFT.pop_back_val();
818
819 if (Instruction *I = dyn_cast<Instruction>(V)) {
820 // If it is a || (or && depending on isEQ), process the operands.
821 if (IsEq ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1)))
822 : match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
823 if (Visited.insert(Op1).second)
824 DFT.push_back(Op1);
825 if (Visited.insert(Op0).second)
826 DFT.push_back(Op0);
827
828 continue;
829 }
830
831 // Try to match the current instruction
832 if (matchInstruction(I, IsEq))
833 // Match succeed, continue the loop
834 continue;
835 }
836
837 // One element of the sequence of || (or &&) could not be match as a
838 // comparison against the same value as the others.
839 // We allow only one "Extra" case to be checked before the switch
840 if (!Extra) {
841 Extra = V;
842 continue;
843 }
844 // Failed to parse a proper sequence, abort now
845 CompValue = nullptr;
846 break;
847 }
848 }
849};
850
851} // end anonymous namespace
852
854 MemorySSAUpdater *MSSAU = nullptr) {
855 Instruction *Cond = nullptr;
857 Cond = dyn_cast<Instruction>(SI->getCondition());
858 } else if (CondBrInst *BI = dyn_cast<CondBrInst>(TI)) {
859 Cond = dyn_cast<Instruction>(BI->getCondition());
860 } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) {
861 Cond = dyn_cast<Instruction>(IBI->getAddress());
862 }
863
864 TI->eraseFromParent();
865 if (Cond)
867}
868
869/// If the terminator is an integer value check, return the value being checked.
870Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) {
871 Value *CV = nullptr;
872 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI))
873 CV = SI->getCondition();
874 else if (CondBrInst *BI = dyn_cast<CondBrInst>(TI))
875 if (BI->getCondition()->hasOneUse()) {
876 if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) {
877 if (ICI->isEquality() && getConstantInt(ICI->getOperand(1), DL))
878 CV = ICI->getOperand(0);
879 } else if (auto *Trunc = dyn_cast<TruncInst>(BI->getCondition())) {
880 if (Trunc->hasNoUnsignedWrap())
881 CV = Trunc->getOperand(0);
882 }
883 }
884
885 // Unwrap any lossless ptrtoint cast (except for unstable pointers).
886 if (CV) {
887 if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) {
888 Value *Ptr = PTII->getPointerOperand();
889 if (DL.hasUnstableRepresentation(Ptr->getType()))
890 return CV;
891 if (PTII->getType() == DL.getIntPtrType(Ptr->getType()))
892 CV = Ptr;
893 }
894 }
895 return CV;
896}
897
898/// Given a value comparison instruction,
899/// decode all of the 'cases' that it represents and return the 'default' block.
900BasicBlock *SimplifyCFGOpt::getValueEqualityComparisonCases(
901 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases) {
902 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
903 Cases.reserve(SI->getNumCases());
904 for (auto Case : SI->cases())
905 Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(),
906 Case.getCaseSuccessor()));
907 return SI->getDefaultDest();
908 }
909
910 CondBrInst *BI = cast<CondBrInst>(TI);
911 Value *Cond = BI->getCondition();
912 ICmpInst::Predicate Pred;
913 ConstantInt *C;
914 if (auto *ICI = dyn_cast<ICmpInst>(Cond)) {
915 Pred = ICI->getPredicate();
916 C = getConstantInt(ICI->getOperand(1), DL);
917 } else {
918 Pred = ICmpInst::ICMP_NE;
919 auto *Trunc = cast<TruncInst>(Cond);
920 C = ConstantInt::get(cast<IntegerType>(Trunc->getOperand(0)->getType()), 0);
921 }
922 BasicBlock *Succ = BI->getSuccessor(Pred == ICmpInst::ICMP_NE);
923 Cases.push_back(ValueEqualityComparisonCase(C, Succ));
924 return BI->getSuccessor(Pred == ICmpInst::ICMP_EQ);
925}
926
927/// Given a vector of bb/value pairs, remove any entries
928/// in the list that match the specified block.
929static void
931 std::vector<ValueEqualityComparisonCase> &Cases) {
932 llvm::erase(Cases, BB);
933}
934
935/// Return true if there are any keys in C1 that exist in C2 as well.
936static bool valuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
937 std::vector<ValueEqualityComparisonCase> &C2) {
938 std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2;
939
940 // Make V1 be smaller than V2.
941 if (V1->size() > V2->size())
942 std::swap(V1, V2);
943
944 if (V1->empty())
945 return false;
946 if (V1->size() == 1) {
947 // Just scan V2.
948 ConstantInt *TheVal = (*V1)[0].Value;
949 for (const ValueEqualityComparisonCase &VECC : *V2)
950 if (TheVal == VECC.Value)
951 return true;
952 }
953
954 // Otherwise, just sort both lists and compare element by element.
955 array_pod_sort(V1->begin(), V1->end());
956 array_pod_sort(V2->begin(), V2->end());
957 unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size();
958 while (i1 != e1 && i2 != e2) {
959 if ((*V1)[i1].Value == (*V2)[i2].Value)
960 return true;
961 if ((*V1)[i1].Value < (*V2)[i2].Value)
962 ++i1;
963 else
964 ++i2;
965 }
966 return false;
967}
968
969/// If TI is a terminator instruction and its block has a single predecessor,
970/// and that predecessor also compares the same value as TI, simplify the TI or
971/// its block based on the conditions that jump to TI's block:
972/// - Prune branches of TI based on the values handled in the predecessor.
973/// - If the predecessor jumps to TI's block on one value, or if TI is a
974/// conditional branch whose only case is ruled out by the predecessor,
975/// replace it with a jump to its only possible destination.
976bool SimplifyCFGOpt::simplifyEqualityComparisonWithOnlyPredecessor(
977 Instruction *TI, BasicBlock *Pred, IRBuilder<> &Builder) {
978 Value *PredVal = isValueEqualityComparison(Pred->getTerminator());
979 if (!PredVal)
980 return false; // Not a value comparison in predecessor.
981
982 Value *ThisVal = isValueEqualityComparison(TI);
983 assert(ThisVal && "This isn't a value comparison!!");
984 if (ThisVal != PredVal)
985 return false; // Different predicates.
986
987 // TODO: Preserve branch weight metadata, similarly to how
988 // foldValueComparisonIntoPredecessors preserves it.
989
990 // Find out information about when control will move from Pred to TI's block.
991 std::vector<ValueEqualityComparisonCase> PredCases;
992 BasicBlock *PredDef =
993 getValueEqualityComparisonCases(Pred->getTerminator(), PredCases);
994 // Remove cases that branch to the default target.
995 eliminateBlockCases(PredDef, PredCases);
996
997 // Find information about how control leaves this block.
998 std::vector<ValueEqualityComparisonCase> ThisCases;
999 BasicBlock *ThisDef = getValueEqualityComparisonCases(TI, ThisCases);
1000 // Remove cases that branch to the default target.
1001 eliminateBlockCases(ThisDef, ThisCases);
1002
1003 BasicBlock *TIBB = TI->getParent();
1004
1005 // If TI's block is the default block from Pred's comparison, prune branches
1006 // of TI based on the values handled in the Pred's comparison.
1007 if (PredDef == TIBB) {
1008 // If we are here, we know that the value is none of those cases listed in
1009 // PredCases. If there are any cases in ThisCases that are in PredCases, we
1010 // can simplify TI.
1011 if (!valuesOverlap(PredCases, ThisCases))
1012 return false;
1013
1014 if (isa<CondBrInst>(TI)) {
1015 // TI checks for one value, which is already checked in the predecessor.
1016 // So we know which branch will be taken, convert it to an unconditional
1017 // br.
1018 assert(ThisCases.size() == 1 && "Branch can only have one case!");
1019 // Insert the new branch.
1020 Instruction *NI = Builder.CreateBr(ThisDef);
1021 (void)NI;
1022
1023 // Remove PHI node entries for the dead edge.
1024 ThisCases[0].Dest->removePredecessor(TIBB);
1025
1026 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
1027 << "Through successor TI: " << *TI << "Leaving: " << *NI
1028 << "\n");
1029
1031
1032 if (DTU)
1033 DTU->applyUpdates({{DominatorTree::Delete, TIBB, ThisCases[0].Dest}});
1034
1035 return true;
1036 }
1037
1038 SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI);
1039 // Okay, TI has cases that are statically dead, prune them away.
1040 SmallPtrSet<Constant *, 16> DeadCases;
1041 for (const ValueEqualityComparisonCase &Case : PredCases)
1042 DeadCases.insert(Case.Value);
1043
1044 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
1045 << "Through successor TI: " << *TI);
1046
1047 SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
1048 for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
1049 --i;
1050 auto *Successor = i->getCaseSuccessor();
1051 if (DTU)
1052 ++NumPerSuccessorCases[Successor];
1053 if (DeadCases.count(i->getCaseValue())) {
1054 Successor->removePredecessor(PredDef);
1055 SI.removeCase(i);
1056 if (DTU)
1057 --NumPerSuccessorCases[Successor];
1058 }
1059 }
1060
1061 if (DTU) {
1062 std::vector<DominatorTree::UpdateType> Updates;
1063 for (const auto &I : NumPerSuccessorCases)
1064 if (I.second == 0)
1065 Updates.push_back({DominatorTree::Delete, PredDef, I.first});
1066 DTU->applyUpdates(Updates);
1067 }
1068
1069 LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n");
1070 return true;
1071 }
1072
1073 // Otherwise, TI's block must correspond to some matched value. Find out
1074 // which value (or set of values) this is.
1075 ConstantInt *TIV = nullptr;
1076 for (const auto &[Value, Dest] : PredCases)
1077 if (Dest == TIBB) {
1078 if (TIV)
1079 return false; // Cannot handle multiple values coming to this block.
1080 TIV = Value;
1081 }
1082 assert(TIV && "No edge from pred to succ?");
1083
1084 // Find out which successor will unconditionally be branched to.
1085 BasicBlock *TheRealDest = nullptr;
1086 for (const auto &[Value, Dest] : ThisCases)
1087 if (Value == TIV) {
1088 TheRealDest = Dest;
1089 break;
1090 }
1091
1092 // If not handled by any explicit cases, it is handled by the default case.
1093 if (!TheRealDest)
1094 TheRealDest = ThisDef;
1095
1096 SmallPtrSet<BasicBlock *, 2> RemovedSuccs;
1097
1098 // Remove PHI node entries for dead edges.
1099 BasicBlock *CheckEdge = TheRealDest;
1100 for (BasicBlock *Succ : successors(TIBB))
1101 if (Succ != CheckEdge) {
1102 if (Succ != TheRealDest)
1103 RemovedSuccs.insert(Succ);
1104 Succ->removePredecessor(TIBB);
1105 } else
1106 CheckEdge = nullptr;
1107
1108 // Insert the new branch.
1109 Instruction *NI = Builder.CreateBr(TheRealDest);
1110 (void)NI;
1111
1112 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
1113 << "Through successor TI: " << *TI << "Leaving: " << *NI
1114 << "\n");
1115
1117 if (DTU) {
1118 SmallVector<DominatorTree::UpdateType, 2> Updates;
1119 Updates.reserve(RemovedSuccs.size());
1120 for (auto *RemovedSucc : RemovedSuccs)
1121 Updates.push_back({DominatorTree::Delete, TIBB, RemovedSucc});
1122 DTU->applyUpdates(Updates);
1123 }
1124 return true;
1125}
1126
1127namespace {
1128
1129/// This class implements a stable ordering of constant
1130/// integers that does not depend on their address. This is important for
1131/// applications that sort ConstantInt's to ensure uniqueness.
1132struct ConstantIntOrdering {
1133 bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const {
1134 return LHS->getValue().ult(RHS->getValue());
1135 }
1136};
1137
1138} // end anonymous namespace
1139
1141 ConstantInt *const *P2) {
1142 const ConstantInt *LHS = *P1;
1143 const ConstantInt *RHS = *P2;
1144 if (LHS == RHS)
1145 return 0;
1146 return LHS->getValue().ult(RHS->getValue()) ? 1 : -1;
1147}
1148
1149/// Get Weights of a given terminator, the default weight is at the front
1150/// of the vector. If TI is a conditional eq, we need to swap the branch-weight
1151/// metadata.
1153 SmallVectorImpl<uint64_t> &Weights) {
1154 MDNode *MD = TI->getMetadata(LLVMContext::MD_prof);
1155 assert(MD && "Invalid branch-weight metadata");
1156 extractFromBranchWeightMD64(MD, Weights);
1157
1158 // If TI is a conditional eq, the default case is the false case,
1159 // and the corresponding branch-weight data is at index 2. We swap the
1160 // default weight to be the first entry.
1161 if (CondBrInst *BI = dyn_cast<CondBrInst>(TI)) {
1162 assert(Weights.size() == 2);
1163 auto *ICI = dyn_cast<ICmpInst>(BI->getCondition());
1164 if (!ICI)
1165 return;
1166
1167 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
1168 std::swap(Weights.front(), Weights.back());
1169 }
1170}
1171
1173 BasicBlock *BB, BasicBlock *PredBlock, ValueToValueMapTy &VMap) {
1174 Instruction *PTI = PredBlock->getTerminator();
1175
1176 // If we have bonus instructions, clone them into the predecessor block.
1177 // Note that there may be multiple predecessor blocks, so we cannot move
1178 // bonus instructions to a predecessor block.
1179 for (Instruction &BonusInst : *BB) {
1180 if (BonusInst.isTerminator())
1181 continue;
1182
1183 // Skip cloning pseudo probes into the predecessor, as it would overcount
1184 // otherwise.
1185 if (isa<PseudoProbeInst>(BonusInst))
1186 continue;
1187
1188 Instruction *NewBonusInst = BonusInst.clone();
1189 NewBonusInst->insertInto(PredBlock, PTI->getIterator());
1190
1191 if (!NewBonusInst->getDebugLoc().isSameSourceLocation(PTI->getDebugLoc())) {
1192 // Unless the instruction has the same !dbg location as the original
1193 // branch, drop it. When we fold the bonus instructions we want to make
1194 // sure we reset their debug locations in order to avoid stepping on
1195 // dead code caused by folding dead branches.
1196 NewBonusInst->dropLocation();
1197 } else if (const DebugLoc &DL = NewBonusInst->getDebugLoc()) {
1198 mapAtomInstance(DL, VMap);
1199 }
1200
1201 RemapInstruction(NewBonusInst, VMap,
1203
1204 // If we speculated an instruction, we need to drop any metadata that may
1205 // result in undefined behavior, as the metadata might have been valid
1206 // only given the branch precondition.
1207 // Similarly strip attributes on call parameters that may cause UB in
1208 // location the call is moved to.
1209 NewBonusInst->dropUBImplyingAttrsAndMetadata();
1210
1211 auto Range = NewBonusInst->cloneDebugInfoFrom(&BonusInst);
1212 RemapDbgRecordRange(NewBonusInst->getModule(), Range, VMap,
1214
1215 NewBonusInst->takeName(&BonusInst);
1216 BonusInst.setName(NewBonusInst->getName() + ".old");
1217 VMap[&BonusInst] = NewBonusInst;
1218
1219 // Update (liveout) uses of bonus instructions,
1220 // now that the bonus instruction has been cloned into predecessor.
1221 // Note that we expect to be in a block-closed SSA form for this to work!
1222 for (Use &U : make_early_inc_range(BonusInst.uses())) {
1223 auto *UI = cast<Instruction>(U.getUser());
1224 auto *PN = dyn_cast<PHINode>(UI);
1225 if (!PN) {
1226 assert(UI->getParent() == BB && BonusInst.comesBefore(UI) &&
1227 "If the user is not a PHI node, then it should be in the same "
1228 "block as, and come after, the original bonus instruction.");
1229 continue; // Keep using the original bonus instruction.
1230 }
1231 // Is this the block-closed SSA form PHI node?
1232 if (PN->getIncomingBlock(U) == BB)
1233 continue; // Great, keep using the original bonus instruction.
1234 // The only other alternative is an "use" when coming from
1235 // the predecessor block - here we should refer to the cloned bonus instr.
1236 assert(PN->getIncomingBlock(U) == PredBlock &&
1237 "Not in block-closed SSA form?");
1238 U.set(NewBonusInst);
1239 }
1240 }
1241
1242 // Key Instructions: We may have propagated atom info into the pred. If the
1243 // pred's terminator already has atom info do nothing as merging would drop
1244 // one atom group anyway. If it doesn't, propagte the remapped atom group
1245 // from BB's terminator.
1246 if (auto &PredDL = PTI->getDebugLoc()) {
1247 auto &DL = BB->getTerminator()->getDebugLoc();
1248 if (!PredDL->getAtomGroup() && DL && DL->getAtomGroup() &&
1249 PredDL.isSameSourceLocation(DL)) {
1250 PTI->setDebugLoc(DL);
1251 RemapSourceAtom(PTI, VMap);
1252 }
1253 }
1254}
1255
1256bool SimplifyCFGOpt::performValueComparisonIntoPredecessorFolding(
1257 Instruction *TI, Value *&CV, Instruction *PTI, IRBuilder<> &Builder) {
1258 BasicBlock *BB = TI->getParent();
1259 BasicBlock *Pred = PTI->getParent();
1260
1262
1263 // Figure out which 'cases' to copy from SI to PSI.
1264 std::vector<ValueEqualityComparisonCase> BBCases;
1265 BasicBlock *BBDefault = getValueEqualityComparisonCases(TI, BBCases);
1266
1267 std::vector<ValueEqualityComparisonCase> PredCases;
1268 BasicBlock *PredDefault = getValueEqualityComparisonCases(PTI, PredCases);
1269
1270 // Based on whether the default edge from PTI goes to BB or not, fill in
1271 // PredCases and PredDefault with the new switch cases we would like to
1272 // build.
1273 SmallMapVector<BasicBlock *, int, 8> NewSuccessors;
1274
1275 // Update the branch weight metadata along the way
1276 SmallVector<uint64_t, 8> Weights;
1277 bool PredHasWeights = hasBranchWeightMD(*PTI);
1278 bool SuccHasWeights = hasBranchWeightMD(*TI);
1279
1280 if (PredHasWeights) {
1281 getBranchWeights(PTI, Weights);
1282 // branch-weight metadata is inconsistent here.
1283 if (Weights.size() != 1 + PredCases.size())
1284 PredHasWeights = SuccHasWeights = false;
1285 } else if (SuccHasWeights)
1286 // If there are no predecessor weights but there are successor weights,
1287 // populate Weights with 1, which will later be scaled to the sum of
1288 // successor's weights
1289 Weights.assign(1 + PredCases.size(), 1);
1290
1291 SmallVector<uint64_t, 8> SuccWeights;
1292 if (SuccHasWeights) {
1293 getBranchWeights(TI, SuccWeights);
1294 // branch-weight metadata is inconsistent here.
1295 if (SuccWeights.size() != 1 + BBCases.size())
1296 PredHasWeights = SuccHasWeights = false;
1297 } else if (PredHasWeights)
1298 SuccWeights.assign(1 + BBCases.size(), 1);
1299
1300 if (PredDefault == BB) {
1301 // If this is the default destination from PTI, only the edges in TI
1302 // that don't occur in PTI, or that branch to BB will be activated.
1303 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1304 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1305 if (PredCases[i].Dest != BB)
1306 PTIHandled.insert(PredCases[i].Value);
1307 else {
1308 // The default destination is BB, we don't need explicit targets.
1309 std::swap(PredCases[i], PredCases.back());
1310
1311 if (PredHasWeights || SuccHasWeights) {
1312 // Increase weight for the default case.
1313 Weights[0] += Weights[i + 1];
1314 std::swap(Weights[i + 1], Weights.back());
1315 Weights.pop_back();
1316 }
1317
1318 PredCases.pop_back();
1319 --i;
1320 --e;
1321 }
1322
1323 // Reconstruct the new switch statement we will be building.
1324 if (PredDefault != BBDefault) {
1325 PredDefault->removePredecessor(Pred);
1326 if (DTU && PredDefault != BB)
1327 Updates.push_back({DominatorTree::Delete, Pred, PredDefault});
1328 PredDefault = BBDefault;
1329 ++NewSuccessors[BBDefault];
1330 }
1331
1332 unsigned CasesFromPred = Weights.size();
1333 uint64_t ValidTotalSuccWeight = 0;
1334 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1335 if (!PTIHandled.count(BBCases[i].Value) && BBCases[i].Dest != BBDefault) {
1336 PredCases.push_back(BBCases[i]);
1337 ++NewSuccessors[BBCases[i].Dest];
1338 if (SuccHasWeights || PredHasWeights) {
1339 // The default weight is at index 0, so weight for the ith case
1340 // should be at index i+1. Scale the cases from successor by
1341 // PredDefaultWeight (Weights[0]).
1342 Weights.push_back(Weights[0] * SuccWeights[i + 1]);
1343 ValidTotalSuccWeight += SuccWeights[i + 1];
1344 }
1345 }
1346
1347 if (SuccHasWeights || PredHasWeights) {
1348 ValidTotalSuccWeight += SuccWeights[0];
1349 // Scale the cases from predecessor by ValidTotalSuccWeight.
1350 for (unsigned i = 1; i < CasesFromPred; ++i)
1351 Weights[i] *= ValidTotalSuccWeight;
1352 // Scale the default weight by SuccDefaultWeight (SuccWeights[0]).
1353 Weights[0] *= SuccWeights[0];
1354 }
1355 } else {
1356 // If this is not the default destination from PSI, only the edges
1357 // in SI that occur in PSI with a destination of BB will be
1358 // activated.
1359 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1360 std::map<ConstantInt *, uint64_t> WeightsForHandled;
1361 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1362 if (PredCases[i].Dest == BB) {
1363 PTIHandled.insert(PredCases[i].Value);
1364
1365 if (PredHasWeights || SuccHasWeights) {
1366 WeightsForHandled[PredCases[i].Value] = Weights[i + 1];
1367 std::swap(Weights[i + 1], Weights.back());
1368 Weights.pop_back();
1369 }
1370
1371 std::swap(PredCases[i], PredCases.back());
1372 PredCases.pop_back();
1373 --i;
1374 --e;
1375 }
1376
1377 // Okay, now we know which constants were sent to BB from the
1378 // predecessor. Figure out where they will all go now.
1379 for (const ValueEqualityComparisonCase &Case : BBCases)
1380 if (PTIHandled.count(Case.Value)) {
1381 // If this is one we are capable of getting...
1382 if (PredHasWeights || SuccHasWeights)
1383 Weights.push_back(WeightsForHandled[Case.Value]);
1384 PredCases.push_back(Case);
1385 ++NewSuccessors[Case.Dest];
1386 PTIHandled.erase(Case.Value); // This constant is taken care of
1387 }
1388
1389 // If there are any constants vectored to BB that TI doesn't handle,
1390 // they must go to the default destination of TI.
1391 for (ConstantInt *I : PTIHandled) {
1392 if (PredHasWeights || SuccHasWeights)
1393 Weights.push_back(WeightsForHandled[I]);
1394 PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault));
1395 ++NewSuccessors[BBDefault];
1396 }
1397 }
1398
1399 // Okay, at this point, we know which new successor Pred will get. Make
1400 // sure we update the number of entries in the PHI nodes for these
1401 // successors.
1402 SmallPtrSet<BasicBlock *, 2> SuccsOfPred;
1403 if (DTU) {
1404 SuccsOfPred = {llvm::from_range, successors(Pred)};
1405 Updates.reserve(Updates.size() + NewSuccessors.size());
1406 }
1407 for (const std::pair<BasicBlock *, int /*Num*/> &NewSuccessor :
1408 NewSuccessors) {
1409 for (auto I : seq(NewSuccessor.second)) {
1410 (void)I;
1411 addPredecessorToBlock(NewSuccessor.first, Pred, BB);
1412 }
1413 if (DTU && !SuccsOfPred.contains(NewSuccessor.first))
1414 Updates.push_back({DominatorTree::Insert, Pred, NewSuccessor.first});
1415 }
1416
1417 Builder.SetInsertPoint(PTI);
1418 // Convert pointer to int before we switch.
1419 if (CV->getType()->isPointerTy()) {
1420 assert(!DL.hasUnstableRepresentation(CV->getType()) &&
1421 "Should not end up here with unstable pointers");
1422 CV =
1423 Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()), "magicptr");
1424 }
1425
1426 // Now that the successors are updated, create the new Switch instruction.
1427 SwitchInst *NewSI = Builder.CreateSwitch(CV, PredDefault, PredCases.size());
1428 NewSI->setDebugLoc(PTI->getDebugLoc());
1429 for (ValueEqualityComparisonCase &V : PredCases)
1430 NewSI->addCase(V.Value, V.Dest);
1431
1432 if (PredHasWeights || SuccHasWeights)
1433 setFittedBranchWeights(*NewSI, Weights, /*IsExpected=*/false,
1434 /*ElideAllZero=*/true);
1435 else
1437
1438 // The new switch is only known to be unpredictable if both of the comparisons
1439 // it was built from were unpredictable.
1440 if (MDNode *Unpredictable = PTI->getMetadata(LLVMContext::MD_unpredictable))
1441 if (TI->hasMetadata(LLVMContext::MD_unpredictable))
1442 NewSI->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
1443
1445
1446 // Okay, last check. If BB is still a successor of PSI, then we must
1447 // have an infinite loop case. If so, add an infinitely looping block
1448 // to handle the case to preserve the behavior of the code.
1449 BasicBlock *InfLoopBlock = nullptr;
1450 for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i)
1451 if (NewSI->getSuccessor(i) == BB) {
1452 if (!InfLoopBlock) {
1453 // Insert it at the end of the function, because it's either code,
1454 // or it won't matter if it's hot. :)
1455 InfLoopBlock =
1456 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
1457 UncondBrInst::Create(InfLoopBlock, InfLoopBlock);
1458 if (DTU)
1459 Updates.push_back(
1460 {DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
1461 }
1462 NewSI->setSuccessor(i, InfLoopBlock);
1463 }
1464
1465 if (DTU) {
1466 if (InfLoopBlock)
1467 Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock});
1468
1469 Updates.push_back({DominatorTree::Delete, Pred, BB});
1470
1471 DTU->applyUpdates(Updates);
1472 }
1473
1474 ++NumFoldValueComparisonIntoPredecessors;
1475 return true;
1476}
1477
1478/// The specified terminator is a value equality comparison instruction
1479/// (either a switch or a branch on "X == c").
1480/// See if any of the predecessors of the terminator block are value comparisons
1481/// on the same value. If so, and if safe to do so, fold them together.
1482bool SimplifyCFGOpt::foldValueComparisonIntoPredecessors(Instruction *TI,
1483 IRBuilder<> &Builder) {
1484 BasicBlock *BB = TI->getParent();
1485 Value *CV = isValueEqualityComparison(TI); // CondVal
1486 assert(CV && "Not a comparison?");
1487
1488 // Limit `incoming_edges * outgoing_edges < 128` to avoid code size explosion
1489 // when we duplicate branches in multiple predecessors.
1490 if (!BB->getSinglePredecessor() &&
1492 return false;
1493
1494 bool Changed = false;
1495
1496 SmallSetVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB));
1497 while (!Preds.empty()) {
1498 BasicBlock *Pred = Preds.pop_back_val();
1499 Instruction *PTI = Pred->getTerminator();
1500
1501 // Don't try to fold into itself.
1502 if (Pred == BB)
1503 continue;
1504
1505 // See if the predecessor is a comparison with the same value.
1506 Value *PCV = isValueEqualityComparison(PTI); // PredCondVal
1507 if (PCV != CV)
1508 continue;
1509
1510 SmallSetVector<BasicBlock *, 4> FailBlocks;
1511 if (!safeToMergeTerminators(TI, PTI, &FailBlocks)) {
1512 for (auto *Succ : FailBlocks) {
1513 if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split", DTU))
1514 return false;
1515 }
1516 }
1517
1518 performValueComparisonIntoPredecessorFolding(TI, CV, PTI, Builder);
1519 Changed = true;
1520 }
1521 return Changed;
1522}
1523
1524// If we would need to insert a select that uses the value of this invoke
1525// (comments in hoistSuccIdenticalTerminatorToSwitchOrIf explain why we would
1526// need to do this), we can't hoist the invoke, as there is nowhere to put the
1527// select in this case.
1529 Instruction *I1, Instruction *I2) {
1530 for (BasicBlock *Succ : successors(BB1)) {
1531 for (const PHINode &PN : Succ->phis()) {
1532 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1533 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1534 if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) {
1535 return false;
1536 }
1537 }
1538 }
1539 return true;
1540}
1541
1542// Get interesting characteristics of instructions that
1543// `hoistCommonCodeFromSuccessors` didn't hoist. They restrict what kind of
1544// instructions can be reordered across.
1550
1552 // Pseudo probes don't constrain reordering of other instructions.
1554 return 0;
1555 unsigned Flags = 0;
1556 if (I->mayReadFromMemory())
1557 Flags |= SkipReadMem;
1558 // We can't arbitrarily move around allocas, e.g. moving allocas (especially
1559 // inalloca) across stacksave/stackrestore boundaries.
1560 if (I->mayHaveSideEffects() || isa<AllocaInst>(I))
1561 Flags |= SkipSideEffect;
1563 Flags |= SkipImplicitControlFlow;
1564 return Flags;
1565}
1566
1567// Returns true if it is safe to reorder an instruction across preceding
1568// instructions in a basic block.
1569static bool isSafeToHoistInstr(Instruction *I, unsigned Flags) {
1570 // Don't reorder a store over a load.
1571 if ((Flags & SkipReadMem) && I->mayWriteToMemory())
1572 return false;
1573
1574 // If we have seen an instruction with side effects, it's unsafe to reorder an
1575 // instruction which reads memory or itself has side effects.
1576 if ((Flags & SkipSideEffect) &&
1577 (I->mayReadFromMemory() || I->mayHaveSideEffects() || isa<AllocaInst>(I)))
1578 return false;
1579
1580 // Reordering across an instruction which does not necessarily transfer
1581 // control to the next instruction is speculation.
1583 return false;
1584
1585 // Hoisting of llvm.deoptimize is only legal together with the next return
1586 // instruction, which this pass is not always able to do.
1587 if (auto *CB = dyn_cast<CallBase>(I))
1588 if (CB->getIntrinsicID() == Intrinsic::experimental_deoptimize)
1589 return false;
1590
1591 // It's also unsafe/illegal to hoist an instruction above its instruction
1592 // operands
1593 BasicBlock *BB = I->getParent();
1594 for (Value *Op : I->operands()) {
1595 if (auto *J = dyn_cast<Instruction>(Op))
1596 if (J->getParent() == BB)
1597 return false;
1598 }
1599
1600 return true;
1601}
1602
1603static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified = false);
1604
1605/// Helper function for hoistCommonCodeFromSuccessors. Return true if identical
1606/// instructions \p I1 and \p I2 can and should be hoisted.
1608 const TargetTransformInfo &TTI) {
1609 // If we're going to hoist a call, make sure that the two instructions
1610 // we're commoning/hoisting are both marked with musttail, or neither of
1611 // them is marked as such. Otherwise, we might end up in a situation where
1612 // we hoist from a block where the terminator is a `ret` to a block where
1613 // the terminator is a `br`, and `musttail` calls expect to be followed by
1614 // a return.
1615 auto *C1 = dyn_cast<CallInst>(I1);
1616 auto *C2 = dyn_cast<CallInst>(I2);
1617 if (C1 && C2)
1618 if (C1->isMustTailCall() != C2->isMustTailCall())
1619 return false;
1620
1621 if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2))
1622 return false;
1623
1624 // If any of the two call sites has nomerge or convergent attribute, stop
1625 // hoisting.
1626 if (const auto *CB1 = dyn_cast<CallBase>(I1))
1627 if (CB1->cannotMerge() || CB1->isConvergent())
1628 return false;
1629 if (const auto *CB2 = dyn_cast<CallBase>(I2))
1630 if (CB2->cannotMerge() || CB2->isConvergent())
1631 return false;
1632
1633 return true;
1634}
1635
1636/// Hoists DbgVariableRecords from \p I1 and \p OtherInstrs that are identical
1637/// in lock-step to \p TI. This matches how dbg.* intrinsics are hoisting in
1638/// hoistCommonCodeFromSuccessors. e.g. The input:
1639/// I1 DVRs: { x, z },
1640/// OtherInsts: { I2 DVRs: { x, y, z } }
1641/// would result in hoisting only DbgVariableRecord x.
1643 Instruction *TI, Instruction *I1,
1644 SmallVectorImpl<Instruction *> &OtherInsts) {
1645 if (!I1->hasDbgRecords())
1646 return;
1647 using CurrentAndEndIt =
1648 std::pair<DbgRecord::self_iterator, DbgRecord::self_iterator>;
1649 // Vector of {Current, End} iterators.
1651 Itrs.reserve(OtherInsts.size() + 1);
1652 // Helper lambdas for lock-step checks:
1653 // Return true if this Current == End.
1654 auto atEnd = [](const CurrentAndEndIt &Pair) {
1655 return Pair.first == Pair.second;
1656 };
1657 // Return true if all Current are identical.
1658 auto allIdentical = [](const SmallVector<CurrentAndEndIt> &Itrs) {
1659 return all_of(make_first_range(ArrayRef(Itrs).drop_front()),
1661 return Itrs[0].first->isIdenticalToWhenDefined(*I);
1662 });
1663 };
1664
1665 // Collect the iterators.
1666 Itrs.push_back(
1667 {I1->getDbgRecordRange().begin(), I1->getDbgRecordRange().end()});
1668 for (Instruction *Other : OtherInsts) {
1669 if (!Other->hasDbgRecords())
1670 return;
1671 Itrs.push_back(
1672 {Other->getDbgRecordRange().begin(), Other->getDbgRecordRange().end()});
1673 }
1674
1675 // Iterate in lock-step until any of the DbgRecord lists are exausted. If
1676 // the lock-step DbgRecord are identical, hoist all of them to TI.
1677 // This replicates the dbg.* intrinsic behaviour in
1678 // hoistCommonCodeFromSuccessors.
1679 while (none_of(Itrs, atEnd)) {
1680 bool HoistDVRs = allIdentical(Itrs);
1681 for (CurrentAndEndIt &Pair : Itrs) {
1682 // Increment Current iterator now as we may be about to move the
1683 // DbgRecord.
1684 DbgRecord &DR = *Pair.first++;
1685 if (HoistDVRs) {
1686 DR.removeFromParent();
1687 TI->getParent()->insertDbgRecordBefore(&DR, TI->getIterator());
1688 }
1689 }
1690 }
1691}
1692
1694 const Instruction *I2) {
1695 if (I1->isIdenticalToWhenDefined(I2, /*IntersectAttrs=*/true))
1696 return true;
1697
1698 if (auto *Cmp1 = dyn_cast<CmpInst>(I1))
1699 if (auto *Cmp2 = dyn_cast<CmpInst>(I2))
1700 return Cmp1->getPredicate() == Cmp2->getSwappedPredicate() &&
1701 Cmp1->getOperand(0) == Cmp2->getOperand(1) &&
1702 Cmp1->getOperand(1) == Cmp2->getOperand(0);
1703
1704 if (I1->isCommutative() && I1->isSameOperationAs(I2)) {
1705 return I1->getOperand(0) == I2->getOperand(1) &&
1706 I1->getOperand(1) == I2->getOperand(0) &&
1707 equal(drop_begin(I1->operands(), 2), drop_begin(I2->operands(), 2));
1708 }
1709
1710 return false;
1711}
1712
1713/// If the target supports conditional faulting,
1714/// we look for the following pattern:
1715/// \code
1716/// BB:
1717/// ...
1718/// %cond = icmp ult %x, %y
1719/// br i1 %cond, label %TrueBB, label %FalseBB
1720/// FalseBB:
1721/// store i32 1, ptr %q, align 4
1722/// ...
1723/// TrueBB:
1724/// %maskedloadstore = load i32, ptr %b, align 4
1725/// store i32 %maskedloadstore, ptr %p, align 4
1726/// ...
1727/// \endcode
1728///
1729/// and transform it into:
1730///
1731/// \code
1732/// BB:
1733/// ...
1734/// %cond = icmp ult %x, %y
1735/// %maskedloadstore = cload i32, ptr %b, %cond
1736/// cstore i32 %maskedloadstore, ptr %p, %cond
1737/// cstore i32 1, ptr %q, ~%cond
1738/// br i1 %cond, label %TrueBB, label %FalseBB
1739/// FalseBB:
1740/// ...
1741/// TrueBB:
1742/// ...
1743/// \endcode
1744///
1745/// where cload/cstore are represented by llvm.masked.load/store intrinsics,
1746/// e.g.
1747///
1748/// \code
1749/// %vcond = bitcast i1 %cond to <1 x i1>
1750/// %v0 = call <1 x i32> @llvm.masked.load.v1i32.p0
1751/// (ptr %b, i32 4, <1 x i1> %vcond, <1 x i32> poison)
1752/// %maskedloadstore = bitcast <1 x i32> %v0 to i32
1753/// call void @llvm.masked.store.v1i32.p0
1754/// (<1 x i32> %v0, ptr %p, i32 4, <1 x i1> %vcond)
1755/// %cond.not = xor i1 %cond, true
1756/// %vcond.not = bitcast i1 %cond.not to <1 x i>
1757/// call void @llvm.masked.store.v1i32.p0
1758/// (<1 x i32> <i32 1>, ptr %q, i32 4, <1x i1> %vcond.not)
1759/// \endcode
1760///
1761/// So we need to turn hoisted load/store into cload/cstore.
1762///
1763/// \param BI The branch instruction.
1764/// \param SpeculatedConditionalLoadsStores The load/store instructions that
1765/// will be speculated.
1766/// \param Invert indicates if speculates FalseBB. Only used in triangle CFG.
1768 CondBrInst *BI,
1769 SmallVectorImpl<Instruction *> &SpeculatedConditionalLoadsStores,
1770 std::optional<bool> Invert, Instruction *Sel) {
1771 auto &Context = BI->getParent()->getContext();
1772 auto *VCondTy = FixedVectorType::get(Type::getInt1Ty(Context), 1);
1773 auto *Cond = BI->getCondition();
1774 // Construct the condition if needed.
1775 BasicBlock *BB = BI->getParent();
1776 Value *Mask = nullptr;
1777 Value *MaskFalse = nullptr;
1778 Value *MaskTrue = nullptr;
1779 if (Invert.has_value()) {
1780 IRBuilder<> Builder(Sel ? Sel : SpeculatedConditionalLoadsStores.back());
1781 Mask = Builder.CreateBitCast(
1782 *Invert ? Builder.CreateXor(Cond, ConstantInt::getTrue(Context)) : Cond,
1783 VCondTy);
1784 } else {
1785 IRBuilder<> Builder(BI);
1786 MaskFalse = Builder.CreateBitCast(
1787 Builder.CreateXor(Cond, ConstantInt::getTrue(Context)), VCondTy);
1788 MaskTrue = Builder.CreateBitCast(Cond, VCondTy);
1789 }
1790 auto PeekThroughBitcasts = [](Value *V) {
1791 while (auto *BitCast = dyn_cast<BitCastInst>(V))
1792 V = BitCast->getOperand(0);
1793 return V;
1794 };
1795 for (auto *I : SpeculatedConditionalLoadsStores) {
1796 IRBuilder<> Builder(Invert.has_value() ? I : BI);
1797 if (!Invert.has_value())
1798 Mask = I->getParent() == BI->getSuccessor(0) ? MaskTrue : MaskFalse;
1799 // We currently assume conditional faulting load/store is supported for
1800 // scalar types only when creating new instructions. This can be easily
1801 // extended for vector types in the future.
1802 assert(!getLoadStoreType(I)->isVectorTy() && "not implemented");
1803 auto *Op0 = I->getOperand(0);
1804 CallInst *MaskedLoadStore = nullptr;
1805 if (auto *LI = dyn_cast<LoadInst>(I)) {
1806 // Handle Load.
1807 auto *Ty = I->getType();
1808 PHINode *PN = nullptr;
1809 Value *PassThru = nullptr;
1810 if (Invert.has_value())
1811 for (User *U : I->users()) {
1812 if ((PN = dyn_cast<PHINode>(U))) {
1813 PassThru = Builder.CreateBitCast(
1814 PeekThroughBitcasts(PN->getIncomingValueForBlock(BB)),
1815 FixedVectorType::get(Ty, 1));
1816 } else if (auto *Ins = cast<Instruction>(U);
1817 Sel && Ins->getParent() == BB) {
1818 // This happens when store or/and a speculative instruction between
1819 // load and store were hoisted to the BB. Make sure the masked load
1820 // inserted before its use.
1821 // We assume there's one of such use.
1822 Builder.SetInsertPoint(Ins);
1823 }
1824 }
1825 MaskedLoadStore = Builder.CreateMaskedLoad(
1826 FixedVectorType::get(Ty, 1), Op0, LI->getAlign(), Mask, PassThru);
1827 if (const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range)) {
1829 if (PassThru && !isa<PoisonValue>(PassThru)) {
1830 auto *C = dyn_cast<Constant>(PassThru);
1831 CR = C ? CR.unionWith(C->toConstantRange())
1832 : ConstantRange::getFull(CR.getBitWidth());
1833 }
1834 MaskedLoadStore->addRangeRetAttr(CR);
1835 }
1836 Value *NewLoadStore = Builder.CreateBitCast(MaskedLoadStore, Ty);
1837 if (PN)
1838 PN->setIncomingValue(PN->getBasicBlockIndex(BB), NewLoadStore);
1839 I->replaceAllUsesWith(NewLoadStore);
1840 } else {
1841 // Handle Store.
1842 auto *StoredVal = Builder.CreateBitCast(
1843 PeekThroughBitcasts(Op0), FixedVectorType::get(Op0->getType(), 1));
1844 MaskedLoadStore = Builder.CreateMaskedStore(
1845 StoredVal, I->getOperand(1), cast<StoreInst>(I)->getAlign(), Mask);
1846 }
1847 // For non-debug metadata, only !annotation, !range, !nonnull and !align are
1848 // kept when hoisting (see Instruction::dropUBImplyingAttrsAndMetadata).
1849 //
1850 // !nonnull, !align : Not support pointer type, no need to keep.
1851 // !range: Kept as a return attribute, widened over PassThru (see above).
1852 // !annotation: Not impact semantics. Keep it.
1853 I->dropUBImplyingAttrsAndUnknownMetadata({LLVMContext::MD_annotation});
1854 // FIXME: DIAssignID is not supported for masked store yet.
1855 // (Verifier::visitDIAssignIDMetadata)
1857 I->eraseMetadataIf([](unsigned MDKind, MDNode *Node) {
1858 return Node->getMetadataID() == Metadata::DIAssignIDKind;
1859 });
1860 MaskedLoadStore->copyMetadata(*I);
1861 I->eraseFromParent();
1862 }
1863}
1864
1866 const TargetTransformInfo &TTI) {
1867 // Not handle volatile or atomic.
1868 bool IsStore = false;
1869 if (auto *L = dyn_cast<LoadInst>(I)) {
1870 if (!L->isSimple() || !HoistLoadsWithCondFaulting)
1871 return false;
1872 } else if (auto *S = dyn_cast<StoreInst>(I)) {
1873 if (!S->isSimple() || !HoistStoresWithCondFaulting)
1874 return false;
1875 IsStore = true;
1876 } else
1877 return false;
1878
1879 // llvm.masked.load/store use i32 for alignment while load/store use i64.
1880 // That's why we have the alignment limitation.
1881 // FIXME: Update the prototype of the intrinsics?
1882 return TTI.hasConditionalLoadStoreForType(getLoadStoreType(I), IsStore) &&
1884}
1885
1886/// Hoist any common code in the successor blocks up into the block. This
1887/// function guarantees that BB dominates all successors. If AllInstsEqOnly is
1888/// given, only perform hoisting in case all successors blocks contain matching
1889/// instructions only. In that case, all instructions can be hoisted and the
1890/// original branch will be replaced and selects for PHIs are added.
1891bool SimplifyCFGOpt::hoistCommonCodeFromSuccessors(Instruction *TI,
1892 bool AllInstsEqOnly) {
1893 // This does very trivial matching, with limited scanning, to find identical
1894 // instructions in the two blocks. In particular, we don't want to get into
1895 // O(N1*N2*...) situations here where Ni are the sizes of these successors. As
1896 // such, we currently just scan for obviously identical instructions in an
1897 // identical order, possibly separated by the same number of non-identical
1898 // instructions.
1899 BasicBlock *BB = TI->getParent();
1900 unsigned int SuccSize = succ_size(BB);
1901 if (SuccSize < 2)
1902 return false;
1903
1904 // If either of the blocks has it's address taken, then we can't do this fold,
1905 // because the code we'd hoist would no longer run when we jump into the block
1906 // by it's address.
1907 SmallSetVector<BasicBlock *, 4> UniqueSuccessors(from_range, successors(BB));
1908 for (auto *Succ : UniqueSuccessors) {
1909 if (Succ->hasAddressTaken())
1910 return false;
1911 // Use getUniquePredecessor instead of getSinglePredecessor to support
1912 // multi-cases successors in switch.
1913 if (Succ->getUniquePredecessor())
1914 continue;
1915 // If Succ has >1 predecessors, continue to check if the Succ contains only
1916 // one `unreachable` inst. Since executing `unreachable` inst is an UB, we
1917 // can relax the condition based on the assumptiom that the program would
1918 // never enter Succ and trigger such an UB.
1919 if (isa<UnreachableInst>(*Succ->begin()))
1920 continue;
1921 return false;
1922 }
1923 // The second of pair is a SkipFlags bitmask.
1924 using SuccIterPair = std::pair<BasicBlock::iterator, unsigned>;
1925 SmallVector<SuccIterPair, 8> SuccIterPairs;
1926 for (auto *Succ : UniqueSuccessors) {
1927 BasicBlock::iterator SuccItr = Succ->begin();
1928 if (isa<PHINode>(*SuccItr))
1929 return false;
1930 SuccIterPairs.push_back(SuccIterPair(SuccItr, 0));
1931 }
1932
1933 if (AllInstsEqOnly) {
1934 // Check if all instructions in the successor blocks match. This allows
1935 // hoisting all instructions and removing the blocks we are hoisting from,
1936 // so does not add any new instructions.
1937
1938 // Check if sizes and terminators of all successors match.
1939 unsigned Size0 = UniqueSuccessors[0]->size();
1940 Instruction *Term0 = UniqueSuccessors[0]->getTerminator();
1941 bool AllSame =
1942 all_of(drop_begin(UniqueSuccessors), [Term0, Size0](BasicBlock *Succ) {
1943 return Succ->getTerminator()->isIdenticalTo(Term0) &&
1944 Succ->size() == Size0;
1945 });
1946 if (!AllSame)
1947 return false;
1948 LockstepReverseIterator<true> LRI(UniqueSuccessors.getArrayRef());
1949 while (LRI.isValid()) {
1950 Instruction *I0 = (*LRI)[0];
1951 if (any_of(*LRI, [I0](Instruction *I) {
1952 return !areIdenticalUpToCommutativity(I0, I);
1953 })) {
1954 return false;
1955 }
1956 --LRI;
1957 }
1958 // Now we know that all instructions in all successors can be hoisted. Let
1959 // the loop below handle the hoisting.
1960 }
1961
1962 // Count how many instructions were not hoisted so far. There's a limit on how
1963 // many instructions we skip, serving as a compilation time control as well as
1964 // preventing excessive increase of life ranges.
1965 unsigned NumSkipped = 0;
1966 // If we find an unreachable instruction at the beginning of a basic block, we
1967 // can still hoist instructions from the rest of the basic blocks.
1968 if (SuccIterPairs.size() > 2) {
1969 erase_if(SuccIterPairs,
1970 [](const auto &Pair) { return isa<UnreachableInst>(Pair.first); });
1971 if (SuccIterPairs.size() < 2)
1972 return false;
1973 }
1974
1975 bool Changed = false;
1976
1977 for (;;) {
1978 auto *SuccIterPairBegin = SuccIterPairs.begin();
1979 auto &BB1ItrPair = *SuccIterPairBegin++;
1980 auto OtherSuccIterPairRange =
1981 iterator_range(SuccIterPairBegin, SuccIterPairs.end());
1982 auto OtherSuccIterRange = make_first_range(OtherSuccIterPairRange);
1983
1984 Instruction *I1 = &*BB1ItrPair.first;
1985
1986 bool AllInstsAreIdentical = true;
1987 bool HasTerminator = I1->isTerminator();
1988 for (auto &SuccIter : OtherSuccIterRange) {
1989 Instruction *I2 = &*SuccIter;
1990 HasTerminator |= I2->isTerminator();
1991 if (AllInstsAreIdentical && (!areIdenticalUpToCommutativity(I1, I2) ||
1992 MMRAMetadata(*I1) != MMRAMetadata(*I2)))
1993 AllInstsAreIdentical = false;
1994 }
1995
1996 SmallVector<Instruction *, 8> OtherInsts;
1997 for (auto &SuccIter : OtherSuccIterRange)
1998 OtherInsts.push_back(&*SuccIter);
1999
2000 // If we are hoisting the terminator instruction, don't move one (making a
2001 // broken BB), instead clone it, and remove BI.
2002 if (HasTerminator) {
2003 // Even if BB, which contains only one unreachable instruction, is ignored
2004 // at the beginning of the loop, we can hoist the terminator instruction.
2005 // If any instructions remain in the block, we cannot hoist terminators.
2006 if (NumSkipped || !AllInstsAreIdentical) {
2007 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
2008 return Changed;
2009 }
2010
2011 return hoistSuccIdenticalTerminatorToSwitchOrIf(
2012 TI, I1, OtherInsts, UniqueSuccessors.getArrayRef()) ||
2013 Changed;
2014 }
2015
2016 if (AllInstsAreIdentical) {
2017 unsigned SkipFlagsBB1 = BB1ItrPair.second;
2018 AllInstsAreIdentical =
2019 isSafeToHoistInstr(I1, SkipFlagsBB1) &&
2020 all_of(OtherSuccIterPairRange, [=](const auto &Pair) {
2021 Instruction *I2 = &*Pair.first;
2022 unsigned SkipFlagsBB2 = Pair.second;
2023 // Even if the instructions are identical, it may not
2024 // be safe to hoist them if we have skipped over
2025 // instructions with side effects or their operands
2026 // weren't hoisted.
2027 return isSafeToHoistInstr(I2, SkipFlagsBB2) &&
2029 });
2030 }
2031
2032 // A musttail call must be immediately followed by a ret, so hoisting is
2033 // only legal if its ret is hoisted with it on the next iteration. That is,
2034 // no instruction has been skipped (the entire successor can be hoisted into
2035 // the predecessor) and the call is directly followed by a ret.
2036 if (auto *CI = dyn_cast<CallInst>(I1);
2037 AllInstsAreIdentical && CI && CI->isMustTailCall()) {
2038 AllInstsAreIdentical =
2039 NumSkipped == 0 && all_of(SuccIterPairs, [](const SuccIterPair &P) {
2040 return isa<ReturnInst>(*std::next(P.first));
2041 });
2042 }
2043
2044 if (AllInstsAreIdentical) {
2045 BB1ItrPair.first++;
2046 // For a normal instruction, we just move one to right before the
2047 // branch, then replace all uses of the other with the first. Finally,
2048 // we remove the now redundant second instruction.
2049 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
2050 // We've just hoisted DbgVariableRecords; move I1 after them (before TI)
2051 // and leave any that were not hoisted behind (by calling moveBefore
2052 // rather than moveBeforePreserving).
2053 I1->moveBefore(TI->getIterator());
2054 for (auto &SuccIter : OtherSuccIterRange) {
2055 Instruction *I2 = &*SuccIter++;
2056 assert(I2 != I1);
2057 if (!I2->use_empty())
2058 I2->replaceAllUsesWith(I1);
2059 I1->andIRFlags(I2);
2060 if (auto *CB = dyn_cast<CallBase>(I1)) {
2061 bool Success = CB->tryIntersectAttributes(cast<CallBase>(I2));
2062 assert(Success && "We should not be trying to hoist callbases "
2063 "with non-intersectable attributes");
2064 // For NDEBUG Compile.
2065 (void)Success;
2066 }
2067
2068 combineMetadataForCSE(I1, I2, true);
2069 // If we are combining select instructions, we mark the profile
2070 // explicitly unknown given we are hoisting the instruction across
2071 // control flow and we have no guarantee that the control flow condition
2072 // we are hoisting across is independent of the select condition.
2073 if (auto *SI = dyn_cast<SelectInst>(I1))
2075 // I1 and I2 are being combined into a single instruction. Its debug
2076 // location is the merged locations of the original instructions.
2077 I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
2078 I2->eraseFromParent();
2079 }
2080 // I1 now executes before the instructions we skipped.
2081 unsigned SkippedFlags = 0;
2082 for (const SuccIterPair &P : SuccIterPairs)
2083 SkippedFlags |= P.second;
2084 if (SkippedFlags & SkipImplicitControlFlow) {
2085 // One of them may throw or not return, so I1 is speculated.
2086 I1->dropUBImplyingAttrsAndMetadata();
2087 }
2088 if (!Changed)
2089 NumHoistCommonCode += SuccIterPairs.size();
2090 Changed = true;
2091 NumHoistCommonInstrs += SuccIterPairs.size();
2092 } else {
2093 if (NumSkipped >= HoistCommonSkipLimit) {
2094 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
2095 return Changed;
2096 }
2097 // We are about to skip over a pair of non-identical instructions. Record
2098 // if any have characteristics that would prevent reordering instructions
2099 // across them.
2100 for (auto &SuccIterPair : SuccIterPairs) {
2101 Instruction *I = &*SuccIterPair.first++;
2102 SuccIterPair.second |= skippedInstrFlags(I);
2103 }
2104 ++NumSkipped;
2105 }
2106 }
2107}
2108
2109bool SimplifyCFGOpt::hoistSuccIdenticalTerminatorToSwitchOrIf(
2110 Instruction *TI, Instruction *I1,
2111 SmallVectorImpl<Instruction *> &OtherSuccTIs,
2112 ArrayRef<BasicBlock *> UniqueSuccessors) {
2113
2114 auto *BI = dyn_cast<CondBrInst>(TI);
2115
2116 bool Changed = false;
2117 BasicBlock *TIParent = TI->getParent();
2118 BasicBlock *BB1 = I1->getParent();
2119
2120 // Use only for an if statement.
2121 auto *I2 = *OtherSuccTIs.begin();
2122 auto *BB2 = I2->getParent();
2123 if (BI) {
2124 assert(OtherSuccTIs.size() == 1);
2125 assert(BI->getSuccessor(0) == I1->getParent());
2126 assert(BI->getSuccessor(1) == I2->getParent());
2127 }
2128
2129 // In the case of an if statement, we try to hoist an invoke.
2130 // FIXME: Can we define a safety predicate for CallBr?
2131 // FIXME: Test case llvm/test/Transforms/SimplifyCFG/2009-06-15-InvokeCrash.ll
2132 // removed in 4c923b3b3fd0ac1edebf0603265ca3ba51724937 commit?
2133 if (isa<InvokeInst>(I1) && (!BI || !isSafeToHoistInvoke(BB1, BB2, I1, I2)))
2134 return false;
2135
2136 // TODO: callbr hoisting currently disabled pending further study.
2137 if (isa<CallBrInst>(I1))
2138 return false;
2139
2140 for (BasicBlock *Succ : successors(BB1)) {
2141 for (PHINode &PN : Succ->phis()) {
2142 Value *BB1V = PN.getIncomingValueForBlock(BB1);
2143 for (Instruction *OtherSuccTI : OtherSuccTIs) {
2144 Value *BB2V = PN.getIncomingValueForBlock(OtherSuccTI->getParent());
2145 if (BB1V == BB2V)
2146 continue;
2147
2148 // In the case of an if statement, check for
2149 // passingValueIsAlwaysUndefined here because we would rather eliminate
2150 // undefined control flow then converting it to a select.
2151 if (!BI || passingValueIsAlwaysUndefined(BB1V, &PN) ||
2153 return false;
2154 }
2155 }
2156 }
2157
2158 // Hoist DbgVariableRecords attached to the terminator to match dbg.*
2159 // intrinsic hoisting behaviour in hoistCommonCodeFromSuccessors.
2160 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherSuccTIs);
2161 // Clone the terminator and hoist it into the pred, without any debug info.
2162 Instruction *NT = I1->clone();
2163 NT->insertInto(TIParent, TI->getIterator());
2164 if (!NT->getType()->isVoidTy()) {
2165 I1->replaceAllUsesWith(NT);
2166 for (Instruction *OtherSuccTI : OtherSuccTIs)
2167 OtherSuccTI->replaceAllUsesWith(NT);
2168 NT->takeName(I1);
2169 }
2170 Changed = true;
2171 NumHoistCommonInstrs += OtherSuccTIs.size() + 1;
2172
2173 // Ensure terminator gets a debug location, even an unknown one, in case
2174 // it involves inlinable calls.
2176 Locs.push_back(I1->getDebugLoc());
2177 for (auto *OtherSuccTI : OtherSuccTIs)
2178 Locs.push_back(OtherSuccTI->getDebugLoc());
2179 NT->setDebugLoc(DebugLoc::getMergedLocations(Locs));
2180
2181 // PHIs created below will adopt NT's merged DebugLoc.
2182 IRBuilder<NoFolder> Builder(NT);
2183
2184 // In the case of an if statement, hoisting one of the terminators from our
2185 // successor is a great thing. Unfortunately, the successors of the if/else
2186 // blocks may have PHI nodes in them. If they do, all PHI entries for BB1/BB2
2187 // must agree for all PHI nodes, so we insert select instruction to compute
2188 // the final result.
2189 if (BI) {
2190 std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects;
2191 for (BasicBlock *Succ : successors(BB1)) {
2192 for (PHINode &PN : Succ->phis()) {
2193 Value *BB1V = PN.getIncomingValueForBlock(BB1);
2194 Value *BB2V = PN.getIncomingValueForBlock(BB2);
2195 if (BB1V == BB2V)
2196 continue;
2197
2198 // These values do not agree. Insert a select instruction before NT
2199 // that determines the right value.
2200 SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)];
2201 if (!SI) {
2202 // Propagate fast-math-flags from phi node to its replacement select.
2204 BI->getCondition(), BB1V, BB2V,
2205 isa<FPMathOperator>(PN) ? &PN : nullptr,
2206 BB1V->getName() + "." + BB2V->getName(), BI));
2207 }
2208
2209 // Make the PHI node use the select for all incoming values for BB1/BB2
2210 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
2211 if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2)
2212 PN.setIncomingValue(i, SI);
2213 }
2214 }
2215 }
2216
2218
2219 // Update any PHI nodes in our new successors.
2220 SmallPtrSet<BasicBlock *, 8> VisitedSuccs;
2221 for (BasicBlock *Succ : successors(BB1)) {
2222 addPredecessorToBlock(Succ, TIParent, BB1);
2223
2224 if (DTU && VisitedSuccs.insert(Succ).second)
2225 Updates.push_back({DominatorTree::Insert, TIParent, Succ});
2226 }
2227
2228 if (DTU) {
2229 // TI might be a switch with multi-cases destination, so we need to care for
2230 // the duplication of successors.
2231 for (BasicBlock *Succ : UniqueSuccessors)
2232 Updates.push_back({DominatorTree::Delete, TIParent, Succ});
2233 }
2234
2236 if (DTU)
2237 DTU->applyUpdates(Updates);
2238 return Changed;
2239}
2240
2241// TODO: Refine this. This should avoid cases like turning constant memcpy sizes
2242// into variables.
2244 int OpIdx) {
2245 // Divide/Remainder by constant is typically much cheaper than by variable.
2246 if (I->isIntDivRem())
2247 return OpIdx != 1;
2248 return !isa<IntrinsicInst>(I);
2249}
2250
2251// All instructions in Insts belong to different blocks that all unconditionally
2252// branch to a common successor. Analyze each instruction and return true if it
2253// would be possible to sink them into their successor, creating one common
2254// instruction instead. For every value that would be required to be provided by
2255// PHI node (because an operand varies in each input block), add to PHIOperands.
2258 DenseMap<const Use *, SmallVector<Value *, 4>> &PHIOperands) {
2259 // Prune out obviously bad instructions to move. Each instruction must have
2260 // the same number of uses, and we check later that the uses are consistent.
2261 std::optional<unsigned> NumUses;
2262 for (auto *I : Insts) {
2263 // These instructions may change or break semantics if moved.
2264 if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) ||
2265 I->getType()->isTokenTy())
2266 return false;
2267
2268 // Do not try to sink an instruction in an infinite loop - it can cause
2269 // this algorithm to infinite loop.
2270 if (I->getParent()->getSingleSuccessor() == I->getParent())
2271 return false;
2272
2273 // Conservatively return false if I is an inline-asm instruction. Sinking
2274 // and merging inline-asm instructions can potentially create arguments
2275 // that cannot satisfy the inline-asm constraints.
2276 // If the instruction has nomerge or convergent attribute, return false.
2277 if (const auto *C = dyn_cast<CallBase>(I))
2278 if (C->isInlineAsm() || C->cannotMerge() || C->isConvergent())
2279 return false;
2280
2281 if (!NumUses)
2282 NumUses = I->getNumUses();
2283 else if (NumUses != I->getNumUses())
2284 return false;
2285 }
2286
2287 const Instruction *I0 = Insts.front();
2288 const auto I0MMRA = MMRAMetadata(*I0);
2289 for (auto *I : Insts) {
2290 if (!I->isSameOperationAs(I0, Instruction::CompareUsingIntersectedAttrs))
2291 return false;
2292
2293 // Treat MMRAs conservatively. This pass can be quite aggressive and
2294 // could drop a lot of MMRAs otherwise.
2295 if (MMRAMetadata(*I) != I0MMRA)
2296 return false;
2297 }
2298
2299 // Uses must be consistent: If I0 is used in a phi node in the sink target,
2300 // then the other phi operands must match the instructions from Insts. This
2301 // also has to hold true for any phi nodes that would be created as a result
2302 // of sinking. Both of these cases are represented by PhiOperands.
2303 for (const Use &U : I0->uses()) {
2304 auto It = PHIOperands.find(&U);
2305 if (It == PHIOperands.end())
2306 // There may be uses in other blocks when sinking into a loop header.
2307 return false;
2308 if (!equal(Insts, It->second))
2309 return false;
2310 }
2311
2312 // For calls to be sinkable, they must all be indirect, or have same callee.
2313 // I.e. if we have two direct calls to different callees, we don't want to
2314 // turn that into an indirect call. Likewise, if we have an indirect call,
2315 // and a direct call, we don't actually want to have a single indirect call.
2316 if (isa<CallBase>(I0)) {
2317 auto IsIndirectCall = [](const Instruction *I) {
2318 return cast<CallBase>(I)->isIndirectCall();
2319 };
2320 bool HaveIndirectCalls = any_of(Insts, IsIndirectCall);
2321 bool AllCallsAreIndirect = all_of(Insts, IsIndirectCall);
2322 if (HaveIndirectCalls) {
2323 if (!AllCallsAreIndirect)
2324 return false;
2325 } else {
2326 // All callees must be identical.
2327 Value *Callee = nullptr;
2328 for (const Instruction *I : Insts) {
2329 Value *CurrCallee = cast<CallBase>(I)->getCalledOperand();
2330 if (!Callee)
2331 Callee = CurrCallee;
2332 else if (Callee != CurrCallee)
2333 return false;
2334 }
2335 }
2336 }
2337
2338 for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) {
2339 Value *Op = I0->getOperand(OI);
2340 auto SameAsI0 = [&I0, OI](const Instruction *I) {
2341 assert(I->getNumOperands() == I0->getNumOperands());
2342 return I->getOperand(OI) == I0->getOperand(OI);
2343 };
2344 if (!all_of(Insts, SameAsI0)) {
2345 auto CanReplaceOperand = [OI](const Instruction *I) {
2346 return canReplaceOperandWithVariable(I, OI);
2347 };
2349 !all_of(Insts, CanReplaceOperand))
2350 // We can't create a PHI from this operand.
2351 return false;
2352 auto &Ops = PHIOperands[&I0->getOperandUse(OI)];
2353 for (auto *I : Insts)
2354 Ops.push_back(I->getOperand(OI));
2355 }
2356 }
2357 return true;
2358}
2359
2360// Assuming canSinkInstructions(Blocks) has returned true, sink the last
2361// instruction of every block in Blocks to their common successor, commoning
2362// into one instruction.
2364 auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0);
2365
2366 // canSinkInstructions returning true guarantees that every block has at
2367 // least one non-terminator instruction.
2369 for (auto *BB : Blocks) {
2370 Instruction *I = BB->getTerminator();
2371 I = I->getPrevNode();
2372 Insts.push_back(I);
2373 }
2374
2375 // We don't need to do any more checking here; canSinkInstructions should
2376 // have done it all for us.
2377 SmallVector<Value*, 4> NewOperands;
2378 Instruction *I0 = Insts.front();
2379 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) {
2380 // This check is different to that in canSinkInstructions. There, we
2381 // cared about the global view once simplifycfg (and instcombine) have
2382 // completed - it takes into account PHIs that become trivially
2383 // simplifiable. However here we need a more local view; if an operand
2384 // differs we create a PHI and rely on instcombine to clean up the very
2385 // small mess we may make.
2386 bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) {
2387 return I->getOperand(O) != I0->getOperand(O);
2388 });
2389 if (!NeedPHI) {
2390 NewOperands.push_back(I0->getOperand(O));
2391 continue;
2392 }
2393
2394 // Create a new PHI in the successor block and populate it.
2395 auto *Op = I0->getOperand(O);
2396 assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!");
2397 auto *PN =
2398 PHINode::Create(Op->getType(), Insts.size(), Op->getName() + ".sink");
2399 PN->insertBefore(BBEnd->begin());
2400 for (auto *I : Insts)
2401 PN->addIncoming(I->getOperand(O), I->getParent());
2402 NewOperands.push_back(PN);
2403 }
2404
2405 // Arbitrarily use I0 as the new "common" instruction; remap its operands
2406 // and move it to the start of the successor block.
2407 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O)
2408 I0->getOperandUse(O).set(NewOperands[O]);
2409
2410 I0->moveBefore(*BBEnd, BBEnd->getFirstInsertionPt());
2411
2412 // Update metadata and IR flags, and merge debug locations.
2413 for (auto *I : Insts)
2414 if (I != I0) {
2415 // The debug location for the "common" instruction is the merged locations
2416 // of all the commoned instructions. We start with the original location
2417 // of the "common" instruction and iteratively merge each location in the
2418 // loop below.
2419 // This is an N-way merge, which will be inefficient if I0 is a CallInst.
2420 // However, as N-way merge for CallInst is rare, so we use simplified API
2421 // instead of using complex API for N-way merge.
2422 I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc());
2423 combineMetadataForCSE(I0, I, true);
2424 // We mark branch weights explicitly unknown for select instructions as we
2425 // might be sinking through control flow with conditions not independent
2426 // of the select condition, or pulling multiple conditions together
2427 // through a phi node.
2428 if (auto *SI = dyn_cast<SelectInst>(I0))
2430 I0->andIRFlags(I);
2431 if (auto *CB = dyn_cast<CallBase>(I0)) {
2432 bool Success = CB->tryIntersectAttributes(cast<CallBase>(I));
2433 assert(Success && "We should not be trying to sink callbases "
2434 "with non-intersectable attributes");
2435 // For NDEBUG Compile.
2436 (void)Success;
2437 }
2438 }
2439
2440 for (User *U : make_early_inc_range(I0->users())) {
2441 // canSinkLastInstruction checked that all instructions are only used by
2442 // phi nodes in a way that allows replacing the phi node with the common
2443 // instruction.
2444 auto *PN = cast<PHINode>(U);
2445 PN->replaceAllUsesWith(I0);
2446 PN->eraseFromParent();
2447 }
2448
2449 // Finally nuke all instructions apart from the common instruction.
2450 for (auto *I : Insts) {
2451 if (I == I0)
2452 continue;
2453 // The remaining uses are debug users, replace those with the common inst.
2454 // In most (all?) cases this just introduces a use-before-def.
2455 assert(I->user_empty() && "Inst unexpectedly still has non-dbg users");
2456 I->replaceAllUsesWith(I0);
2457 I->eraseFromParent();
2458 }
2459}
2460
2461/// Check whether BB's predecessors end with unconditional branches. If it is
2462/// true, sink any common code from the predecessors to BB.
2464 DomTreeUpdater *DTU) {
2465 // We support two situations:
2466 // (1) all incoming arcs are unconditional
2467 // (2) there are non-unconditional incoming arcs
2468 //
2469 // (2) is very common in switch defaults and
2470 // else-if patterns;
2471 //
2472 // if (a) f(1);
2473 // else if (b) f(2);
2474 //
2475 // produces:
2476 //
2477 // [if]
2478 // / \
2479 // [f(1)] [if]
2480 // | | \
2481 // | | |
2482 // | [f(2)]|
2483 // \ | /
2484 // [ end ]
2485 //
2486 // [end] has two unconditional predecessor arcs and one conditional. The
2487 // conditional refers to the implicit empty 'else' arc. This conditional
2488 // arc can also be caused by an empty default block in a switch.
2489 //
2490 // In this case, we attempt to sink code from all *unconditional* arcs.
2491 // If we can sink instructions from these arcs (determined during the scan
2492 // phase below) we insert a common successor for all unconditional arcs and
2493 // connect that to [end], to enable sinking:
2494 //
2495 // [if]
2496 // / \
2497 // [x(1)] [if]
2498 // | | \
2499 // | | \
2500 // | [x(2)] |
2501 // \ / |
2502 // [sink.split] |
2503 // \ /
2504 // [ end ]
2505 //
2506 SmallVector<BasicBlock*,4> UnconditionalPreds;
2507 bool HaveNonUnconditionalPredecessors = false;
2508 for (auto *PredBB : predecessors(BB)) {
2509 auto *PredBr = dyn_cast<UncondBrInst>(PredBB->getTerminator());
2510 if (PredBr)
2511 UnconditionalPreds.push_back(PredBB);
2512 else
2513 HaveNonUnconditionalPredecessors = true;
2514 }
2515 if (UnconditionalPreds.size() < 2)
2516 return false;
2517
2518 // We take a two-step approach to tail sinking. First we scan from the end of
2519 // each block upwards in lockstep. If the n'th instruction from the end of each
2520 // block can be sunk, those instructions are added to ValuesToSink and we
2521 // carry on. If we can sink an instruction but need to PHI-merge some operands
2522 // (because they're not identical in each instruction) we add these to
2523 // PHIOperands.
2524 // We prepopulate PHIOperands with the phis that already exist in BB.
2526 for (PHINode &PN : BB->phis()) {
2528 for (const Use &U : PN.incoming_values())
2529 IncomingVals.insert({PN.getIncomingBlock(U), &U});
2530 auto &Ops = PHIOperands[IncomingVals[UnconditionalPreds[0]]];
2531 for (BasicBlock *Pred : UnconditionalPreds)
2532 Ops.push_back(*IncomingVals[Pred]);
2533 }
2534
2535 int ScanIdx = 0;
2536 SmallPtrSet<Value*,4> InstructionsToSink;
2537 LockstepReverseIterator<true> LRI(UnconditionalPreds);
2538 while (LRI.isValid() &&
2539 canSinkInstructions(*LRI, PHIOperands)) {
2540 LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0]
2541 << "\n");
2542 InstructionsToSink.insert_range(*LRI);
2543 ++ScanIdx;
2544 --LRI;
2545 }
2546
2547 // If no instructions can be sunk, early-return.
2548 if (ScanIdx == 0)
2549 return false;
2550
2551 bool followedByDeoptOrUnreachable = IsBlockFollowedByDeoptOrUnreachable(BB);
2552
2553 if (!followedByDeoptOrUnreachable) {
2554 // Check whether this is the pointer operand of a load/store.
2555 auto IsMemOperand = [](Use &U) {
2556 auto *I = cast<Instruction>(U.getUser());
2557 if (isa<LoadInst>(I))
2558 return U.getOperandNo() == LoadInst::getPointerOperandIndex();
2559 if (isa<StoreInst>(I))
2560 return U.getOperandNo() == StoreInst::getPointerOperandIndex();
2561 return false;
2562 };
2563
2564 // Okay, we *could* sink last ScanIdx instructions. But how many can we
2565 // actually sink before encountering instruction that is unprofitable to
2566 // sink?
2567 auto ProfitableToSinkInstruction = [&](LockstepReverseIterator<true> &LRI) {
2568 unsigned NumPHIInsts = 0;
2569 for (Use &U : (*LRI)[0]->operands()) {
2570 auto It = PHIOperands.find(&U);
2571 if (It != PHIOperands.end() && !all_of(It->second, [&](Value *V) {
2572 return InstructionsToSink.contains(V);
2573 })) {
2574 ++NumPHIInsts;
2575 // Do not separate a load/store from the gep producing the address.
2576 // The gep can likely be folded into the load/store as an addressing
2577 // mode. Additionally, a load of a gep is easier to analyze than a
2578 // load of a phi.
2579 if (IsMemOperand(U) &&
2580 any_of(It->second, [](Value *V) { return isa<GEPOperator>(V); }))
2581 return false;
2582 // FIXME: this check is overly optimistic. We may end up not sinking
2583 // said instruction, due to the very same profitability check.
2584 // See @creating_too_many_phis in sink-common-code.ll.
2585 }
2586 }
2587 LLVM_DEBUG(dbgs() << "SINK: #phi insts: " << NumPHIInsts << "\n");
2588 return NumPHIInsts <= 1;
2589 };
2590
2591 // We've determined that we are going to sink last ScanIdx instructions,
2592 // and recorded them in InstructionsToSink. Now, some instructions may be
2593 // unprofitable to sink. But that determination depends on the instructions
2594 // that we are going to sink.
2595
2596 // First, forward scan: find the first instruction unprofitable to sink,
2597 // recording all the ones that are profitable to sink.
2598 // FIXME: would it be better, after we detect that not all are profitable.
2599 // to either record the profitable ones, or erase the unprofitable ones?
2600 // Maybe we need to choose (at runtime) the one that will touch least
2601 // instrs?
2602 LRI.reset();
2603 int Idx = 0;
2604 SmallPtrSet<Value *, 4> InstructionsProfitableToSink;
2605 while (Idx < ScanIdx) {
2606 if (!ProfitableToSinkInstruction(LRI)) {
2607 // Too many PHIs would be created.
2608 LLVM_DEBUG(
2609 dbgs() << "SINK: stopping here, too many PHIs would be created!\n");
2610 break;
2611 }
2612 InstructionsProfitableToSink.insert_range(*LRI);
2613 --LRI;
2614 ++Idx;
2615 }
2616
2617 // If no instructions can be sunk, early-return.
2618 if (Idx == 0)
2619 return false;
2620
2621 // Did we determine that (only) some instructions are unprofitable to sink?
2622 if (Idx < ScanIdx) {
2623 // Okay, some instructions are unprofitable.
2624 ScanIdx = Idx;
2625 InstructionsToSink = InstructionsProfitableToSink;
2626
2627 // But, that may make other instructions unprofitable, too.
2628 // So, do a backward scan, do any earlier instructions become
2629 // unprofitable?
2630 assert(
2631 !ProfitableToSinkInstruction(LRI) &&
2632 "We already know that the last instruction is unprofitable to sink");
2633 ++LRI;
2634 --Idx;
2635 while (Idx >= 0) {
2636 // If we detect that an instruction becomes unprofitable to sink,
2637 // all earlier instructions won't be sunk either,
2638 // so preemptively keep InstructionsProfitableToSink in sync.
2639 // FIXME: is this the most performant approach?
2640 for (auto *I : *LRI)
2641 InstructionsProfitableToSink.erase(I);
2642 if (!ProfitableToSinkInstruction(LRI)) {
2643 // Everything starting with this instruction won't be sunk.
2644 ScanIdx = Idx;
2645 InstructionsToSink = InstructionsProfitableToSink;
2646 }
2647 ++LRI;
2648 --Idx;
2649 }
2650 }
2651
2652 // If no instructions can be sunk, early-return.
2653 if (ScanIdx == 0)
2654 return false;
2655 }
2656
2657 bool Changed = false;
2658
2659 if (HaveNonUnconditionalPredecessors) {
2660 if (!followedByDeoptOrUnreachable) {
2661 // It is always legal to sink common instructions from unconditional
2662 // predecessors. However, if not all predecessors are unconditional,
2663 // this transformation might be pessimizing. So as a rule of thumb,
2664 // don't do it unless we'd sink at least one non-speculatable instruction.
2665 // See https://bugs.llvm.org/show_bug.cgi?id=30244
2666 LRI.reset();
2667 int Idx = 0;
2668 bool Profitable = false;
2669 while (Idx < ScanIdx) {
2670 if (!isSafeToSpeculativelyExecute((*LRI)[0])) {
2671 Profitable = true;
2672 break;
2673 }
2674 --LRI;
2675 ++Idx;
2676 }
2677 if (!Profitable)
2678 return false;
2679 }
2680
2681 LLVM_DEBUG(dbgs() << "SINK: Splitting edge\n");
2682 // We have a conditional edge and we're going to sink some instructions.
2683 // Insert a new block postdominating all blocks we're going to sink from.
2684 if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split", DTU))
2685 // Edges couldn't be split.
2686 return false;
2687 Changed = true;
2688 }
2689
2690 // Now that we've analyzed all potential sinking candidates, perform the
2691 // actual sink. We iteratively sink the last non-terminator of the source
2692 // blocks into their common successor unless doing so would require too
2693 // many PHI instructions to be generated (currently only one PHI is allowed
2694 // per sunk instruction).
2695 //
2696 // We can use InstructionsToSink to discount values needing PHI-merging that will
2697 // actually be sunk in a later iteration. This allows us to be more
2698 // aggressive in what we sink. This does allow a false positive where we
2699 // sink presuming a later value will also be sunk, but stop half way through
2700 // and never actually sink it which means we produce more PHIs than intended.
2701 // This is unlikely in practice though.
2702 int SinkIdx = 0;
2703 for (; SinkIdx != ScanIdx; ++SinkIdx) {
2704 LLVM_DEBUG(dbgs() << "SINK: Sink: "
2705 << *UnconditionalPreds[0]->getTerminator()->getPrevNode()
2706 << "\n");
2707
2708 // Because we've sunk every instruction in turn, the current instruction to
2709 // sink is always at index 0.
2710 LRI.reset();
2711
2712 sinkLastInstruction(UnconditionalPreds);
2713 NumSinkCommonInstrs++;
2714 Changed = true;
2715 }
2716 if (SinkIdx != 0)
2717 ++NumSinkCommonCode;
2718 return Changed;
2719}
2720
2721namespace {
2722
2723struct CompatibleSets {
2724 using SetTy = SmallVector<InvokeInst *, 2>;
2725
2727
2728 static bool shouldBelongToSameSet(ArrayRef<InvokeInst *> Invokes);
2729
2730 SetTy &getCompatibleSet(InvokeInst *II);
2731
2732 void insert(InvokeInst *II);
2733};
2734
2735CompatibleSets::SetTy &CompatibleSets::getCompatibleSet(InvokeInst *II) {
2736 // Perform a linear scan over all the existing sets, see if the new `invoke`
2737 // is compatible with any particular set. Since we know that all the `invokes`
2738 // within a set are compatible, only check the first `invoke` in each set.
2739 // WARNING: at worst, this has quadratic complexity.
2740 for (CompatibleSets::SetTy &Set : Sets) {
2741 if (CompatibleSets::shouldBelongToSameSet({Set.front(), II}))
2742 return Set;
2743 }
2744
2745 // Otherwise, we either had no sets yet, or this invoke forms a new set.
2746 return Sets.emplace_back();
2747}
2748
2749void CompatibleSets::insert(InvokeInst *II) {
2750 getCompatibleSet(II).emplace_back(II);
2751}
2752
2753bool CompatibleSets::shouldBelongToSameSet(ArrayRef<InvokeInst *> Invokes) {
2754 assert(Invokes.size() == 2 && "Always called with exactly two candidates.");
2755
2756 // Can we theoretically merge these `invoke`s?
2757 auto IsIllegalToMerge = [](InvokeInst *II) {
2758 return II->cannotMerge() || II->isInlineAsm();
2759 };
2760 if (any_of(Invokes, IsIllegalToMerge))
2761 return false;
2762
2763 // Either both `invoke`s must be direct,
2764 // or both `invoke`s must be indirect.
2765 auto IsIndirectCall = [](InvokeInst *II) { return II->isIndirectCall(); };
2766 bool HaveIndirectCalls = any_of(Invokes, IsIndirectCall);
2767 bool AllCallsAreIndirect = all_of(Invokes, IsIndirectCall);
2768 if (HaveIndirectCalls) {
2769 if (!AllCallsAreIndirect)
2770 return false;
2771 } else {
2772 // All callees must be identical.
2773 Value *Callee = nullptr;
2774 for (InvokeInst *II : Invokes) {
2775 Value *CurrCallee = II->getCalledOperand();
2776 assert(CurrCallee && "There is always a called operand.");
2777 if (!Callee)
2778 Callee = CurrCallee;
2779 else if (Callee != CurrCallee)
2780 return false;
2781 }
2782 }
2783
2784 // Either both `invoke`s must not have a normal destination,
2785 // or both `invoke`s must have a normal destination,
2786 auto HasNormalDest = [](InvokeInst *II) {
2787 return !isa<UnreachableInst>(II->getNormalDest()->getFirstNonPHIIt());
2788 };
2789 if (any_of(Invokes, HasNormalDest)) {
2790 // Do not merge `invoke` that does not have a normal destination with one
2791 // that does have a normal destination, even though doing so would be legal.
2792 if (!all_of(Invokes, HasNormalDest))
2793 return false;
2794
2795 // All normal destinations must be identical.
2796 BasicBlock *NormalBB = nullptr;
2797 for (InvokeInst *II : Invokes) {
2798 BasicBlock *CurrNormalBB = II->getNormalDest();
2799 assert(CurrNormalBB && "There is always a 'continue to' basic block.");
2800 if (!NormalBB)
2801 NormalBB = CurrNormalBB;
2802 else if (NormalBB != CurrNormalBB)
2803 return false;
2804 }
2805
2806 // In the normal destination, the incoming values for these two `invoke`s
2807 // must be compatible.
2808 SmallPtrSet<Value *, 16> EquivalenceSet(llvm::from_range, Invokes);
2810 NormalBB, {Invokes[0]->getParent(), Invokes[1]->getParent()},
2811 &EquivalenceSet))
2812 return false;
2813
2814 // Merging replaces all uses of each `invoke`'s result with the result of
2815 // the merged `invoke`. This is only correct if each result is used solely
2816 // by PHI nodes in the normal destination, on the edge from that same
2817 // `invoke`. For example, if the second `invoke` is inside a loop that is
2818 // entered through the first `invoke`'s normal destination, a use of the
2819 // first result inside that loop would see the second result after merging.
2820 auto HasOnlyIncomingPHIUses = [NormalBB](InvokeInst *II) {
2821 return all_of(II->uses(), [NormalBB, II](const Use &U) {
2822 auto *PN = dyn_cast<PHINode>(U.getUser());
2823 return PN && PN->getParent() == NormalBB &&
2824 PN->getIncomingBlock(U) == II->getParent();
2825 });
2826 };
2827 if (!all_of(Invokes, HasOnlyIncomingPHIUses))
2828 return false;
2829 }
2830
2831#ifndef NDEBUG
2832 // All unwind destinations must be identical.
2833 // We know that because we have started from said unwind destination.
2834 BasicBlock *UnwindBB = nullptr;
2835 for (InvokeInst *II : Invokes) {
2836 BasicBlock *CurrUnwindBB = II->getUnwindDest();
2837 assert(CurrUnwindBB && "There is always an 'unwind to' basic block.");
2838 if (!UnwindBB)
2839 UnwindBB = CurrUnwindBB;
2840 else
2841 assert(UnwindBB == CurrUnwindBB && "Unexpected unwind destination.");
2842 }
2843#endif
2844
2845 // In the unwind destination, the incoming values for these two `invoke`s
2846 // must be compatible.
2848 Invokes.front()->getUnwindDest(),
2849 {Invokes[0]->getParent(), Invokes[1]->getParent()}))
2850 return false;
2851
2852 // Ignoring arguments, these `invoke`s must be identical,
2853 // including operand bundles.
2854 const InvokeInst *II0 = Invokes.front();
2855 for (auto *II : Invokes.drop_front())
2856 if (!II->isSameOperationAs(II0, Instruction::CompareUsingIntersectedAttrs))
2857 return false;
2858
2859 // Can we theoretically form the data operands for the merged `invoke`?
2860 auto IsIllegalToMergeArguments = [](auto Ops) {
2861 Use &U0 = std::get<0>(Ops);
2862 Use &U1 = std::get<1>(Ops);
2863 if (U0 == U1)
2864 return false;
2866 U0.getOperandNo());
2867 };
2868 assert(Invokes.size() == 2 && "Always called with exactly two candidates.");
2869 if (any_of(zip(Invokes[0]->data_ops(), Invokes[1]->data_ops()),
2870 IsIllegalToMergeArguments))
2871 return false;
2872
2873 return true;
2874}
2875
2876} // namespace
2877
2878// Merge all invokes in the provided set, all of which are compatible
2879// as per the `CompatibleSets::shouldBelongToSameSet()`.
2881 DomTreeUpdater *DTU) {
2882 assert(Invokes.size() >= 2 && "Must have at least two invokes to merge.");
2883
2885 if (DTU)
2886 Updates.reserve(2 + 3 * Invokes.size());
2887
2888 bool HasNormalDest =
2889 !isa<UnreachableInst>(Invokes[0]->getNormalDest()->getFirstNonPHIIt());
2890
2891 // Clone one of the invokes into a new basic block.
2892 // Since they are all compatible, it doesn't matter which invoke is cloned.
2893 InvokeInst *MergedInvoke = [&Invokes, HasNormalDest]() {
2894 InvokeInst *II0 = Invokes.front();
2895 BasicBlock *II0BB = II0->getParent();
2896 BasicBlock *InsertBeforeBlock =
2897 II0->getParent()->getIterator()->getNextNode();
2898 Function *Func = II0BB->getParent();
2899 LLVMContext &Ctx = II0->getContext();
2900
2901 BasicBlock *MergedInvokeBB = BasicBlock::Create(
2902 Ctx, II0BB->getName() + ".invoke", Func, InsertBeforeBlock);
2903
2904 auto *MergedInvoke = cast<InvokeInst>(II0->clone());
2905 // NOTE: all invokes have the same attributes, so no handling needed.
2906 MergedInvoke->insertInto(MergedInvokeBB, MergedInvokeBB->end());
2907
2908 if (!HasNormalDest) {
2909 // This set does not have a normal destination,
2910 // so just form a new block with unreachable terminator.
2911 BasicBlock *MergedNormalDest = BasicBlock::Create(
2912 Ctx, II0BB->getName() + ".cont", Func, InsertBeforeBlock);
2913 auto *UI = new UnreachableInst(Ctx, MergedNormalDest);
2914 UI->setDebugLoc(DebugLoc::getTemporary());
2915 MergedInvoke->setNormalDest(MergedNormalDest);
2916 }
2917
2918 // The unwind destination, however, remainds identical for all invokes here.
2919
2920 return MergedInvoke;
2921 }();
2922
2923 if (DTU) {
2924 // Predecessor blocks that contained these invokes will now branch to
2925 // the new block that contains the merged invoke, ...
2926 for (InvokeInst *II : Invokes)
2927 Updates.push_back(
2928 {DominatorTree::Insert, II->getParent(), MergedInvoke->getParent()});
2929
2930 // ... which has the new `unreachable` block as normal destination,
2931 // or unwinds to the (same for all `invoke`s in this set) `landingpad`,
2932 for (BasicBlock *SuccBBOfMergedInvoke : successors(MergedInvoke))
2933 Updates.push_back({DominatorTree::Insert, MergedInvoke->getParent(),
2934 SuccBBOfMergedInvoke});
2935
2936 // Since predecessor blocks now unconditionally branch to a new block,
2937 // they no longer branch to their original successors.
2938 for (InvokeInst *II : Invokes)
2939 for (BasicBlock *SuccOfPredBB : successors(II->getParent()))
2940 Updates.push_back(
2941 {DominatorTree::Delete, II->getParent(), SuccOfPredBB});
2942 }
2943
2944 bool IsIndirectCall = Invokes[0]->isIndirectCall();
2945
2946 // Form the merged operands for the merged invoke.
2947 for (Use &U : MergedInvoke->operands()) {
2948 // Only PHI together the indirect callees and data operands.
2949 if (MergedInvoke->isCallee(&U)) {
2950 if (!IsIndirectCall)
2951 continue;
2952 } else if (!MergedInvoke->isDataOperand(&U))
2953 continue;
2954
2955 // Don't create trivial PHI's with all-identical incoming values.
2956 bool NeedPHI = any_of(Invokes, [&U](InvokeInst *II) {
2957 return II->getOperand(U.getOperandNo()) != U.get();
2958 });
2959 if (!NeedPHI)
2960 continue;
2961
2962 // Form a PHI out of all the data ops under this index.
2964 U->getType(), /*NumReservedValues=*/Invokes.size(), "", MergedInvoke->getIterator());
2965 for (InvokeInst *II : Invokes)
2966 PN->addIncoming(II->getOperand(U.getOperandNo()), II->getParent());
2967
2968 U.set(PN);
2969 }
2970
2971 // We've ensured that each PHI node has compatible (identical) incoming values
2972 // when coming from each of the `invoke`s in the current merge set,
2973 // so update the PHI nodes accordingly.
2974 for (BasicBlock *Succ : successors(MergedInvoke))
2975 addPredecessorToBlock(Succ, /*NewPred=*/MergedInvoke->getParent(),
2976 /*ExistPred=*/Invokes.front()->getParent());
2977
2978 // And finally, replace the original `invoke`s with an unconditional branch
2979 // to the block with the merged `invoke`. Also, give that merged `invoke`
2980 // the merged debugloc of all the original `invoke`s.
2981 DILocation *MergedDebugLoc = nullptr;
2982 for (InvokeInst *II : Invokes) {
2983 // Compute the debug location common to all the original `invoke`s.
2984 if (!MergedDebugLoc)
2985 MergedDebugLoc = II->getDebugLoc();
2986 else
2987 MergedDebugLoc =
2988 DebugLoc::getMergedLocation(MergedDebugLoc, II->getDebugLoc());
2989
2990 // And replace the old `invoke` with an unconditionally branch
2991 // to the block with the merged `invoke`.
2992 for (BasicBlock *OrigSuccBB : successors(II->getParent()))
2993 OrigSuccBB->removePredecessor(II->getParent());
2994 auto *BI = UncondBrInst::Create(MergedInvoke->getParent(), II->getParent());
2995 // The unconditional branch is part of the replacement for the original
2996 // invoke, so should use its DebugLoc.
2997 BI->setDebugLoc(II->getDebugLoc());
2998 bool Success = MergedInvoke->tryIntersectAttributes(II);
2999 assert(Success && "Merged invokes with incompatible attributes");
3000 // For NDEBUG Compile
3001 (void)Success;
3002 II->replaceAllUsesWith(MergedInvoke);
3003 II->eraseFromParent();
3004 ++NumInvokesMerged;
3005 }
3006 MergedInvoke->setDebugLoc(MergedDebugLoc);
3007 ++NumInvokeSetsFormed;
3008
3009 if (DTU)
3010 DTU->applyUpdates(Updates);
3011}
3012
3013/// If this block is a `landingpad` exception handling block, categorize all
3014/// the predecessor `invoke`s into sets, with all `invoke`s in each set
3015/// being "mergeable" together, and then merge invokes in each set together.
3016///
3017/// This is a weird mix of hoisting and sinking. Visually, it goes from:
3018/// [...] [...]
3019/// | |
3020/// [invoke0] [invoke1]
3021/// / \ / \
3022/// [cont0] [landingpad] [cont1]
3023/// to:
3024/// [...] [...]
3025/// \ /
3026/// [invoke]
3027/// / \
3028/// [cont] [landingpad]
3029///
3030/// But of course we can only do that if the invokes share the `landingpad`,
3031/// edges invoke0->cont0 and invoke1->cont1 are "compatible",
3032/// and the invoked functions are "compatible".
3035 return false;
3036
3037 bool Changed = false;
3038
3039 // FIXME: generalize to all exception handling blocks?
3040 if (!BB->isLandingPad())
3041 return Changed;
3042
3043 CompatibleSets Grouper;
3044
3045 // Record all the predecessors of this `landingpad`. As per verifier,
3046 // the only allowed predecessor is the unwind edge of an `invoke`.
3047 // We want to group "compatible" `invokes` into the same set to be merged.
3048 for (BasicBlock *PredBB : predecessors(BB))
3049 Grouper.insert(cast<InvokeInst>(PredBB->getTerminator()));
3050
3051 // And now, merge `invoke`s that were grouped togeter.
3052 for (ArrayRef<InvokeInst *> Invokes : Grouper.Sets) {
3053 if (Invokes.size() < 2)
3054 continue;
3055 Changed = true;
3056 mergeCompatibleInvokesImpl(Invokes, DTU);
3057 }
3058
3059 return Changed;
3060}
3061
3062namespace {
3063/// Track ephemeral values, which should be ignored for cost-modelling
3064/// purposes. Requires walking instructions in reverse order.
3065class EphemeralValueTracker {
3066 SmallPtrSet<const Instruction *, 32> EphValues;
3067
3068 bool isEphemeral(const Instruction *I) {
3069 if (isa<AssumeInst>(I))
3070 return true;
3071 return !I->mayHaveSideEffects() && !I->isTerminator() &&
3072 all_of(I->users(), [&](const User *U) {
3073 return EphValues.count(cast<Instruction>(U));
3074 });
3075 }
3076
3077public:
3078 bool track(const Instruction *I) {
3079 if (isEphemeral(I)) {
3080 EphValues.insert(I);
3081 return true;
3082 }
3083 return false;
3084 }
3085
3086 bool contains(const Instruction *I) const { return EphValues.contains(I); }
3087};
3088} // namespace
3089
3090/// Determine if we can hoist sink a sole store instruction out of a
3091/// conditional block.
3092///
3093/// We are looking for code like the following:
3094/// BrBB:
3095/// store i32 %add, i32* %arrayidx2
3096/// ... // No other stores or function calls (we could be calling a memory
3097/// ... // function).
3098/// %cmp = icmp ult %x, %y
3099/// br i1 %cmp, label %EndBB, label %ThenBB
3100/// ThenBB:
3101/// store i32 %add5, i32* %arrayidx2
3102/// br label EndBB
3103/// EndBB:
3104/// ...
3105/// We are going to transform this into:
3106/// BrBB:
3107/// store i32 %add, i32* %arrayidx2
3108/// ... //
3109/// %cmp = icmp ult %x, %y
3110/// %add.add5 = select i1 %cmp, i32 %add, %add5
3111/// store i32 %add.add5, i32* %arrayidx2
3112/// ...
3113///
3114/// \return The value from the previous access if the store can be hoisted into
3115/// the predecessor block. PreviousAccess is set to that access. Return
3116/// null otherwise.
3118 BasicBlock *StoreBB, BasicBlock *EndBB,
3119 Instruction *&PreviousAccess) {
3120 StoreInst *StoreToHoist = dyn_cast<StoreInst>(I);
3121 if (!StoreToHoist)
3122 return nullptr;
3123
3124 // Volatile or atomic.
3125 if (!StoreToHoist->isSimple())
3126 return nullptr;
3127
3128 Value *StorePtr = StoreToHoist->getPointerOperand();
3129 Type *StoreTy = StoreToHoist->getValueOperand()->getType();
3130
3131 // Look for a store to the same pointer in BrBB.
3132 unsigned MaxNumInstToLookAt = 9;
3133 // Skip pseudo probe intrinsic calls which are not really killing any memory
3134 // accesses.
3135 for (Instruction &CurI : reverse(*BrBB)) {
3136 if (!MaxNumInstToLookAt)
3137 break;
3138 --MaxNumInstToLookAt;
3139
3140 if (isa<PseudoProbeInst>(CurI))
3141 continue;
3142
3143 // Could be calling an instruction that affects memory like free().
3144 if (CurI.mayWriteToMemory() && !isa<StoreInst>(CurI))
3145 return nullptr;
3146
3147 if (auto *SI = dyn_cast<StoreInst>(&CurI)) {
3148 // Found the previous store to same location and type. Make sure it is
3149 // simple, to avoid introducing a spurious non-atomic write after an
3150 // atomic write.
3151 if (SI->getPointerOperand() == StorePtr &&
3152 SI->getValueOperand()->getType() == StoreTy && SI->isSimple() &&
3153 SI->getAlign() >= StoreToHoist->getAlign()) {
3154 // Found the previous store, return its value operand.
3155 PreviousAccess = SI;
3156 return SI->getValueOperand();
3157 }
3158 return nullptr; // Unknown store.
3159 }
3160
3161 if (auto *LI = dyn_cast<LoadInst>(&CurI)) {
3162 if (LI->getPointerOperand() == StorePtr && LI->getType() == StoreTy &&
3163 LI->isSimple() && LI->getAlign() >= StoreToHoist->getAlign()) {
3164 Value *Obj = getUnderlyingObject(StorePtr);
3165 bool ExplicitlyDereferenceableOnly;
3166 // The dereferenceability query here is only required to satisfy the
3167 // writable contract, actual dereferenceability is proven by the
3168 // presence of an access. As such, we can ignore frees.
3169 if (isWritableObject(Obj, ExplicitlyDereferenceableOnly) &&
3172 .WithoutRet) &&
3173 (!ExplicitlyDereferenceableOnly ||
3174 isDereferenceablePointer(StorePtr, StoreTy, LI->getDataLayout(),
3175 /*IgnoreFree=*/true))) {
3176 // Found a previous load, return it.
3177 PreviousAccess = LI;
3178 return LI;
3179 }
3180 }
3181 // The load didn't work out, but we may still find a store.
3182 }
3183 }
3184
3185 return nullptr;
3186}
3187
3188/// Estimate the cost of the insertion(s) and check that the PHI nodes can be
3189/// converted to selects.
3191 BasicBlock *EndBB,
3192 unsigned &SpeculatedInstructions,
3193 InstructionCost &Cost,
3194 const TargetTransformInfo &TTI) {
3196 BB->getParent()->hasMinSize()
3199
3200 bool HaveRewritablePHIs = false;
3201 for (PHINode &PN : EndBB->phis()) {
3202 Value *OrigV = PN.getIncomingValueForBlock(BB);
3203 Value *ThenV = PN.getIncomingValueForBlock(ThenBB);
3204
3205 // FIXME: Try to remove some of the duplication with
3206 // hoistCommonCodeFromSuccessors. Skip PHIs which are trivial.
3207 if (ThenV == OrigV)
3208 continue;
3209
3210 Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(),
3211 Type::getInt1Ty(PN.getContext()),
3212 CmpInst::BAD_ICMP_PREDICATE, CostKind);
3213
3214 // Don't convert to selects if we could remove undefined behavior instead.
3215 if (passingValueIsAlwaysUndefined(OrigV, &PN) ||
3217 return false;
3218
3219 HaveRewritablePHIs = true;
3220 ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV);
3221 ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV);
3222 if (!OrigCE && !ThenCE)
3223 continue; // Known cheap (FIXME: Maybe not true for aggregates).
3224
3225 InstructionCost OrigCost = OrigCE ? computeSpeculationCost(OrigCE, TTI) : 0;
3226 InstructionCost ThenCost = ThenCE ? computeSpeculationCost(ThenCE, TTI) : 0;
3227 InstructionCost MaxCost =
3229 if (OrigCost + ThenCost > MaxCost)
3230 return false;
3231
3232 // Account for the cost of an unfolded ConstantExpr which could end up
3233 // getting expanded into Instructions.
3234 // FIXME: This doesn't account for how many operations are combined in the
3235 // constant expression.
3236 ++SpeculatedInstructions;
3237 if (SpeculatedInstructions > 1)
3238 return false;
3239 }
3240
3241 return HaveRewritablePHIs;
3242}
3243
3245 std::optional<bool> Invert,
3246 const TargetTransformInfo &TTI) {
3247 // If the branch is non-unpredictable, and is predicted to *not* branch to
3248 // the `then` block, then avoid speculating it.
3249 if (BI->getMetadata(LLVMContext::MD_unpredictable))
3250 return true;
3251
3252 uint64_t TWeight, FWeight;
3253 if (!extractBranchWeights(*BI, TWeight, FWeight) || (TWeight + FWeight) == 0)
3254 return true;
3255
3256 if (!Invert.has_value())
3257 return false;
3258
3259 uint64_t EndWeight = *Invert ? TWeight : FWeight;
3260 BranchProbability BIEndProb =
3261 BranchProbability::getBranchProbability(EndWeight, TWeight + FWeight);
3262 BranchProbability Likely = TTI.getPredictableBranchThreshold();
3263 return BIEndProb < Likely;
3264}
3265
3266/// Speculate a conditional basic block flattening the CFG.
3267///
3268/// Note that this is a very risky transform currently. Speculating
3269/// instructions like this is most often not desirable. Instead, there is an MI
3270/// pass which can do it with full awareness of the resource constraints.
3271/// However, some cases are "obvious" and we should do directly. An example of
3272/// this is speculating a single, reasonably cheap instruction.
3273///
3274/// There is only one distinct advantage to flattening the CFG at the IR level:
3275/// it makes very common but simplistic optimizations such as are common in
3276/// instcombine and the DAG combiner more powerful by removing CFG edges and
3277/// modeling their effects with easier to reason about SSA value graphs.
3278///
3279///
3280/// An illustration of this transform is turning this IR:
3281/// \code
3282/// BB:
3283/// %cmp = icmp ult %x, %y
3284/// br i1 %cmp, label %EndBB, label %ThenBB
3285/// ThenBB:
3286/// %sub = sub %x, %y
3287/// br label BB2
3288/// EndBB:
3289/// %phi = phi [ %sub, %ThenBB ], [ 0, %BB ]
3290/// ...
3291/// \endcode
3292///
3293/// Into this IR:
3294/// \code
3295/// BB:
3296/// %cmp = icmp ult %x, %y
3297/// %sub = sub %x, %y
3298/// %cond = select i1 %cmp, 0, %sub
3299/// ...
3300/// \endcode
3301///
3302/// \returns true if the conditional block is removed.
3303bool SimplifyCFGOpt::speculativelyExecuteBB(CondBrInst *BI,
3304 BasicBlock *ThenBB) {
3305 if (!Options.SpeculateBlocks)
3306 return false;
3307
3308 BasicBlock *BB = BI->getParent();
3309 BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0);
3310 InstructionCost Budget =
3312
3313 // If ThenBB is actually on the false edge of the conditional branch, remember
3314 // to swap the select operands later.
3315 bool Invert = false;
3316 if (ThenBB != BI->getSuccessor(0)) {
3317 assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?");
3318 Invert = true;
3319 }
3320 assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block");
3321
3322 if (!isProfitableToSpeculate(BI, Invert, TTI))
3323 return false;
3324
3325 // Keep a count of how many times instructions are used within ThenBB when
3326 // they are candidates for sinking into ThenBB. Specifically:
3327 // - They are defined in BB, and
3328 // - They have no side effects, and
3329 // - All of their uses are in ThenBB.
3330 SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts;
3331
3332 SmallVector<Instruction *, 4> SpeculatedPseudoProbes;
3333
3334 unsigned SpeculatedInstructions = 0;
3335 bool HoistLoadsStores = Options.HoistLoadsStoresWithCondFaulting;
3336 SmallVector<Instruction *, 2> SpeculatedConditionalLoadsStores;
3337 Value *SpeculatedStoreValue = nullptr;
3338 StoreInst *SpeculatedStore = nullptr;
3339 Instruction *PreviousStoreAccess = nullptr;
3340 EphemeralValueTracker EphTracker;
3341 for (Instruction &I : reverse(drop_end(*ThenBB))) {
3342 // Skip pseudo probes. The consequence is we lose track of the branch
3343 // probability for ThenBB, which is fine since the optimization here takes
3344 // place regardless of the branch probability.
3345 if (isa<PseudoProbeInst>(I)) {
3346 // The probe should be deleted so that it will not be over-counted when
3347 // the samples collected on the non-conditional path are counted towards
3348 // the conditional path. We leave it for the counts inference algorithm to
3349 // figure out a proper count for an unknown probe.
3350 SpeculatedPseudoProbes.push_back(&I);
3351 continue;
3352 }
3353
3354 // Ignore ephemeral values, they will be dropped by the transform.
3355 if (EphTracker.track(&I))
3356 continue;
3357
3358 // Only speculatively execute a single instruction (not counting the
3359 // terminator) for now.
3360 bool IsSafeCheapLoadStore = HoistLoadsStores &&
3362 SpeculatedConditionalLoadsStores.size() <
3364 // Not count load/store into cost if target supports conditional faulting
3365 // b/c it's cheap to speculate it.
3366 if (IsSafeCheapLoadStore)
3367 SpeculatedConditionalLoadsStores.push_back(&I);
3368 else
3369 ++SpeculatedInstructions;
3370
3371 if (SpeculatedInstructions > 1)
3372 return false;
3373
3374 // Don't hoist the instruction if it's unsafe or expensive.
3375 if (!IsSafeCheapLoadStore &&
3377 !(HoistCondStores && !SpeculatedStoreValue &&
3378 (SpeculatedStoreValue = isSafeToSpeculateStore(&I, BB, ThenBB, EndBB,
3379 PreviousStoreAccess))))
3380 return false;
3381 if (!IsSafeCheapLoadStore && !SpeculatedStoreValue &&
3384 return false;
3385
3386 // Store the store speculation candidate.
3387 if (!SpeculatedStore && SpeculatedStoreValue)
3388 SpeculatedStore = cast<StoreInst>(&I);
3389
3390 // Do not hoist the instruction if any of its operands are defined but not
3391 // used in BB. The transformation will prevent the operand from
3392 // being sunk into the use block.
3393 for (Use &Op : I.operands()) {
3395 if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects())
3396 continue; // Not a candidate for sinking.
3397
3398 ++SinkCandidateUseCounts[OpI];
3399 }
3400 }
3401
3402 // Consider any sink candidates which are only used in ThenBB as costs for
3403 // speculation. Note, while we iterate over a DenseMap here, we are summing
3404 // and so iteration order isn't significant.
3405 for (const auto &[Inst, Count] : SinkCandidateUseCounts)
3406 if (Inst->hasNUses(Count)) {
3407 ++SpeculatedInstructions;
3408 if (SpeculatedInstructions > 1)
3409 return false;
3410 }
3411
3412 // Check that we can insert the selects and that it's not too expensive to do
3413 // so.
3414 bool Convert =
3415 SpeculatedStore != nullptr || !SpeculatedConditionalLoadsStores.empty();
3417 Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB,
3418 SpeculatedInstructions, Cost, TTI);
3419 if (!Convert || Cost > Budget)
3420 return false;
3421
3422 // If we get here, we can hoist the instruction and if-convert.
3423 LLVM_DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";);
3424
3425 Instruction *Sel = nullptr;
3426 Value *BrCond = BI->getCondition();
3427 // Insert a select of the value of the speculated store.
3428 if (SpeculatedStoreValue) {
3429 IRBuilder<NoFolder> Builder(BI);
3430 Value *OrigV = SpeculatedStore->getValueOperand();
3431 Value *TrueV = SpeculatedStore->getValueOperand();
3432 Value *FalseV = SpeculatedStoreValue;
3433 if (Invert)
3434 std::swap(TrueV, FalseV);
3435 Value *S = Builder.CreateSelect(
3436 BrCond, TrueV, FalseV, "spec.store.select", BI);
3437 Sel = cast<Instruction>(S);
3438 SpeculatedStore->setOperand(0, S);
3439 SpeculatedStore->applyMergedLocation(BI->getDebugLoc(),
3440 SpeculatedStore->getDebugLoc());
3441 // The value stored is still conditional, but the store itself is now
3442 // unconditionally executed, so we must be sure that any linked dbg.assign
3443 // intrinsics are tracking the new stored value (the result of the
3444 // select). If we don't, and the store were to be removed by another pass
3445 // (e.g. DSE), then we'd eventually end up emitting a location describing
3446 // the conditional value, unconditionally.
3447 //
3448 // === Before this transformation ===
3449 // pred:
3450 // store %one, %x.dest, !DIAssignID !1
3451 // dbg.assign %one, "x", ..., !1, ...
3452 // br %cond if.then
3453 //
3454 // if.then:
3455 // store %two, %x.dest, !DIAssignID !2
3456 // dbg.assign %two, "x", ..., !2, ...
3457 //
3458 // === After this transformation ===
3459 // pred:
3460 // store %one, %x.dest, !DIAssignID !1
3461 // dbg.assign %one, "x", ..., !1
3462 /// ...
3463 // %merge = select %cond, %two, %one
3464 // store %merge, %x.dest, !DIAssignID !2
3465 // dbg.assign %merge, "x", ..., !2
3466 for (DbgVariableRecord *DbgAssign :
3467 at::getDVRAssignmentMarkers(SpeculatedStore))
3468 if (llvm::is_contained(DbgAssign->location_ops(), OrigV))
3469 DbgAssign->replaceVariableLocationOp(OrigV, S);
3470 }
3471
3472 // Metadata can be dependent on the condition we are hoisting above.
3473 // Strip all UB-implying metadata on the instruction. Drop the debug loc
3474 // to avoid making it appear as if the condition is a constant, which would
3475 // be misleading while debugging.
3476 // Similarly strip attributes that maybe dependent on condition we are
3477 // hoisting above.
3478 for (auto &I : make_early_inc_range(*ThenBB)) {
3479 if (!SpeculatedStoreValue || &I != SpeculatedStore) {
3480 I.dropLocation();
3481 I.dropUBImplyingAttrsAndMetadata();
3482 } else {
3483 assert(PreviousStoreAccess && "Missing previous store access");
3484 AAMDNodes MergedAA = SpeculatedStore->getAAMetadata().merge(
3485 PreviousStoreAccess->getAAMetadata());
3486 I.dropUBImplyingAttrsAndMetadata();
3487 I.setAAMetadata(MergedAA);
3488 }
3489
3490 // Drop ephemeral values.
3491 if (EphTracker.contains(&I)) {
3492 I.replaceAllUsesWith(PoisonValue::get(I.getType()));
3493 I.eraseFromParent();
3494 }
3495 }
3496
3497 // Hoist the instructions.
3498 // Drop DbgVariableRecords attached to these instructions.
3499 for (auto &It : *ThenBB)
3500 for (DbgRecord &DR : make_early_inc_range(It.getDbgRecordRange()))
3501 // Drop all records except assign-kind DbgVariableRecords (dbg.assign
3502 // equivalent).
3503 if (DbgVariableRecord *DVR = dyn_cast<DbgVariableRecord>(&DR);
3504 !DVR || !DVR->isDbgAssign())
3505 It.dropOneDbgRecord(&DR);
3506 BB->splice(BI->getIterator(), ThenBB, ThenBB->begin(),
3507 std::prev(ThenBB->end()));
3508
3509 if (!SpeculatedConditionalLoadsStores.empty())
3510 hoistConditionalLoadsStores(BI, SpeculatedConditionalLoadsStores, Invert,
3511 Sel);
3512
3513 // Insert selects and rewrite the PHI operands.
3514 IRBuilder<NoFolder> Builder(BI);
3515 for (PHINode &PN : EndBB->phis()) {
3516 unsigned OrigI = PN.getBasicBlockIndex(BB);
3517 unsigned ThenI = PN.getBasicBlockIndex(ThenBB);
3518 Value *OrigV = PN.getIncomingValue(OrigI);
3519 Value *ThenV = PN.getIncomingValue(ThenI);
3520
3521 // Skip PHIs which are trivial.
3522 if (OrigV == ThenV)
3523 continue;
3524
3525 // Create a select whose true value is the speculatively executed value and
3526 // false value is the pre-existing value. Swap them if the branch
3527 // destinations were inverted.
3528 Value *TrueV = ThenV, *FalseV = OrigV;
3529 if (Invert)
3530 std::swap(TrueV, FalseV);
3531 // Propagate fast-math flags from the phi node to the replacement select.
3532 Value *V = Builder.CreateSelectFMF(
3533 BrCond, TrueV, FalseV, PN.getFastMathFlagsOrNone(), "spec.select", BI);
3534 PN.setIncomingValue(OrigI, V);
3535 PN.setIncomingValue(ThenI, V);
3536 }
3537
3538 // Remove speculated pseudo probes.
3539 for (Instruction *I : SpeculatedPseudoProbes)
3540 I->eraseFromParent();
3541
3542 ++NumSpeculations;
3543 return true;
3544}
3545
3547
3548// Return false if number of blocks searched is too much.
3549static bool findReaching(BasicBlock *BB, BasicBlock *DefBB,
3550 BlocksSet &ReachesNonLocalUses) {
3551 if (BB == DefBB)
3552 return true;
3553 if (!ReachesNonLocalUses.insert(BB).second)
3554 return true;
3555
3556 if (ReachesNonLocalUses.size() > MaxJumpThreadingLiveBlocks)
3557 return false;
3558 for (BasicBlock *Pred : predecessors(BB))
3559 if (!findReaching(Pred, DefBB, ReachesNonLocalUses))
3560 return false;
3561 return true;
3562}
3563
3564/// Return true if threading \p PredBBs through \p Header to \p RealDest
3565/// eliminates the loop's single backedge. \p Header must have two predecessors
3566/// (one entry and one backedge), the threaded predecessor must be dominated by
3567/// \p Header, and \p RealDest must not reach it without passing through
3568/// \p Header.
3570 ArrayRef<BasicBlock *> PredBBs,
3571 BasicBlock *RealDest) {
3572 if (!Header->hasNPredecessors(2))
3573 return false;
3574 BlocksSet Visited;
3575 SmallVector<BasicBlock *, 8> Worklist(PredBBs);
3576 while (!Worklist.empty()) {
3577 BasicBlock *BB = Worklist.pop_back_val();
3578 if (BB == Header || !Visited.insert(BB).second)
3579 continue;
3580 if (BB == RealDest)
3581 return false;
3583 return false;
3584 if (pred_empty(BB))
3585 return false;
3586 append_range(Worklist, predecessors(BB));
3587 }
3588 return true;
3589}
3590
3591/// Return true if we can thread a branch across this block.
3593 BlocksSet &NonLocalUseBlocks) {
3594 int Size = 0;
3595 EphemeralValueTracker EphTracker;
3596
3597 // Walk the loop in reverse so that we can identify ephemeral values properly
3598 // (values only feeding assumes).
3599 for (Instruction &I : reverse(*BB)) {
3600 // Can't fold blocks that contain noduplicate or convergent calls.
3601 if (CallInst *CI = dyn_cast<CallInst>(&I))
3602 if (CI->cannotDuplicate() || CI->isConvergent())
3603 return false;
3604
3605 // Ignore ephemeral values which are deleted during codegen.
3606 // We will delete Phis while threading, so Phis should not be accounted in
3607 // block's size.
3608 if (!EphTracker.track(&I) && !isa<PHINode>(I)) {
3609 if (Size++ > MaxSmallBlockSize)
3610 return false; // Don't clone large BB's.
3611 }
3612
3613 // Record blocks with non-local uses of values defined in the current basic
3614 // block.
3615 for (User *U : I.users()) {
3617 BasicBlock *UsedInBB = UI->getParent();
3618 if (UsedInBB == BB) {
3619 if (isa<PHINode>(UI))
3620 return false;
3621 } else
3622 NonLocalUseBlocks.insert(UsedInBB);
3623 }
3624
3625 // Looks ok, continue checking.
3626 }
3627
3628 return true;
3629}
3630
3632 BasicBlock *To) {
3633 // Don't look past the block defining the value, we might get the value from
3634 // a previous loop iteration.
3635 auto *I = dyn_cast<Instruction>(V);
3636 if (I && I->getParent() == To)
3637 return nullptr;
3638
3639 // We know the value if the From block branches on it.
3640 auto *BI = dyn_cast<CondBrInst>(From->getTerminator());
3641 if (BI && BI->getCondition() == V &&
3642 BI->getSuccessor(0) != BI->getSuccessor(1))
3643 return BI->getSuccessor(0) == To ? ConstantInt::getTrue(BI->getContext())
3645
3646 return nullptr;
3647}
3648
3650 return CB->isConvergent() && !isa<ConvergenceControlInst>(CB) &&
3652}
3653
3655 BasicBlock *StopBB) {
3656 static constexpr unsigned MaxInstructionsToScan = 512;
3657
3658 // Walk predecessors of StopBB to find blocks that can reach it. Only
3659 // convergent calls on a cycle with StopBB matter - a convergent call on a
3660 // path to function exit cannot have its dynamic instance changed by
3661 // threading.
3662 SmallPtrSet<BasicBlock *, 8> CanReachStop;
3663 SmallPtrSet<BasicBlock *, 8> BlocksWithUncontrolledConvergentCalls;
3665 for (BasicBlock *Pred : predecessors(StopBB))
3666 Worklist.push_back(Pred);
3667
3668 // Cache blocks with relevant calls while building CanReachStop. This keeps
3669 // the instruction scan bounded without a separate block limit.
3670 unsigned NumScannedInstructions = 0;
3671 while (!Worklist.empty()) {
3672 BasicBlock *BB = Worklist.pop_back_val();
3673 if (BB == StopBB)
3674 continue;
3675 if (!CanReachStop.insert(BB).second)
3676 continue;
3677
3678 for (Instruction &I : *BB) {
3679 if (++NumScannedInstructions > MaxInstructionsToScan)
3680 return true;
3681 auto *CB = dyn_cast<CallBase>(&I);
3682 if (CB && isUncontrolledConvergentCall(CB)) {
3683 BlocksWithUncontrolledConvergentCalls.insert(BB);
3684 break;
3685 }
3686 }
3687
3688 append_range(Worklist, predecessors(BB));
3689 }
3690
3691 if (!CanReachStop.contains(From))
3692 return false;
3693
3695 Worklist.push_back(From);
3696
3697 while (!Worklist.empty()) {
3698 BasicBlock *BB = Worklist.pop_back_val();
3699 if (BB == StopBB || !CanReachStop.contains(BB))
3700 continue;
3701
3702 if (!Visited.insert(BB).second)
3703 continue;
3704
3705 if (BlocksWithUncontrolledConvergentCalls.contains(BB))
3706 return true;
3707
3708 append_range(Worklist, successors(BB));
3709 }
3710
3711 return false;
3712}
3713
3714/// If we have a conditional branch on something for which we know the constant
3715/// value in predecessors (e.g. a phi node in the current block), thread edges
3716/// from the predecessor to their ultimate destination.
3719 AssumptionCache *AC, const DataLayout &DL, bool KeepCanonicalLoop) {
3721 BasicBlock *BB = BI->getParent();
3722 Value *Cond = BI->getCondition();
3724 if (PN && PN->getParent() == BB) {
3725 // Degenerate case of a single entry PHI.
3726 if (PN->getNumIncomingValues() == 1) {
3728 return true;
3729 }
3730
3731 for (Use &U : PN->incoming_values())
3732 if (auto *CB = dyn_cast<ConstantInt>(U))
3733 KnownValues[CB].insert(PN->getIncomingBlock(U));
3734 } else {
3735 for (BasicBlock *Pred : predecessors(BB)) {
3736 if (ConstantInt *CB = getKnownValueOnEdge(Cond, Pred, BB))
3737 KnownValues[CB].insert(Pred);
3738 }
3739 }
3740
3741 if (KnownValues.empty())
3742 return false;
3743
3744 // Now we know that this block has multiple preds and two succs.
3745 // Check that the block is small enough and record which non-local blocks use
3746 // values defined in the block.
3747
3748 BlocksSet NonLocalUseBlocks;
3749 BlocksSet ReachesNonLocalUseBlocks;
3750 if (!blockIsSimpleEnoughToThreadThrough(BB, NonLocalUseBlocks))
3751 return false;
3752
3753 // Jump-threading can only be done to destinations where no values defined
3754 // in BB are live.
3755
3756 // Quickly check if both destinations have uses. If so, jump-threading cannot
3757 // be done.
3758 if (NonLocalUseBlocks.contains(BI->getSuccessor(0)) &&
3759 NonLocalUseBlocks.contains(BI->getSuccessor(1)))
3760 return false;
3761
3762 // Search backward from NonLocalUseBlocks to find which blocks
3763 // reach non-local uses.
3764 for (BasicBlock *UseBB : NonLocalUseBlocks)
3765 // Give up if too many blocks are searched.
3766 if (!findReaching(UseBB, BB, ReachesNonLocalUseBlocks))
3767 return false;
3768
3769 for (const auto &Pair : KnownValues) {
3770 ConstantInt *CB = Pair.first;
3771 ArrayRef<BasicBlock *> PredBBs = Pair.second.getArrayRef();
3772 BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue());
3773
3774 // Okay, we now know that all edges from PredBB should be revectored to
3775 // branch to RealDest.
3776 if (RealDest == BB)
3777 continue; // Skip self loops.
3778
3779 // Skip if the predecessor's terminator is an indirect branch.
3780 if (any_of(PredBBs, [](BasicBlock *PredBB) {
3781 return isa<IndirectBrInst>(PredBB->getTerminator());
3782 }))
3783 continue;
3784
3785 // Only revector to RealDest if no values defined in BB are live.
3786 if (ReachesNonLocalUseBlocks.contains(RealDest))
3787 continue;
3788
3789 if (KeepCanonicalLoop &&
3790 !isSafeToThreadThroughLoopHeader(BB, PredBBs, RealDest))
3791 continue;
3792
3793 // Threading through a branch can bypass a reconvergence point. If the
3794 // destination can execute an uncontrolled convergent operation before
3795 // returning to this block, this may change the dynamic instance of that
3796 // operation.
3797 if (TTI.hasBranchDivergence(BB->getParent()) &&
3799 continue;
3800
3801 LLVM_DEBUG({
3802 dbgs() << "Condition " << *Cond << " in " << BB->getName()
3803 << " has value " << *Pair.first << " in predecessors:\n";
3804 for (const BasicBlock *PredBB : Pair.second)
3805 dbgs() << " " << PredBB->getName() << "\n";
3806 dbgs() << "Threading to destination " << RealDest->getName() << ".\n";
3807 });
3808
3809 // Split the predecessors we are threading into a new edge block. We'll
3810 // clone the instructions into this block, and then redirect it to RealDest.
3811 BasicBlock *EdgeBB = SplitBlockPredecessors(BB, PredBBs, ".critedge", DTU);
3812 if (!EdgeBB)
3813 continue;
3814
3815 // TODO: These just exist to reduce test diff, we can drop them if we like.
3816 EdgeBB->setName(RealDest->getName() + ".critedge");
3817 EdgeBB->moveBefore(RealDest);
3818
3819 // Update PHI nodes.
3820 addPredecessorToBlock(RealDest, EdgeBB, BB);
3821
3822 // BB may have instructions that are being threaded over. Clone these
3823 // instructions into EdgeBB. We know that there will be no uses of the
3824 // cloned instructions outside of EdgeBB.
3825 BasicBlock::iterator InsertPt = EdgeBB->getFirstInsertionPt();
3826 ValueToValueMapTy TranslateMap; // Track translated values.
3827 TranslateMap[Cond] = CB;
3828
3829 // RemoveDIs: track instructions that we optimise away while folding, so
3830 // that we can copy DbgVariableRecords from them later.
3831 BasicBlock::iterator SrcDbgCursor = BB->begin();
3832 for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
3833 if (PHINode *PN = dyn_cast<PHINode>(BBI)) {
3834 TranslateMap[PN] = PN->getIncomingValueForBlock(EdgeBB);
3835 continue;
3836 }
3837 // Clone the instruction.
3838 Instruction *N = BBI->clone();
3839 // Insert the new instruction into its new home.
3840 N->insertInto(EdgeBB, InsertPt);
3841
3842 if (BBI->hasName())
3843 N->setName(BBI->getName() + ".c");
3844
3845 // Update operands due to translation.
3846 // Key Instructions: Remap all the atom groups.
3847 if (const DebugLoc &DL = BBI->getDebugLoc())
3848 mapAtomInstance(DL, TranslateMap);
3849 RemapInstruction(N, TranslateMap,
3851
3852 // Check for trivial simplification.
3853 if (Value *V = simplifyInstruction(N, {DL, nullptr, nullptr, AC})) {
3854 if (!BBI->use_empty())
3855 TranslateMap[&*BBI] = V;
3856 if (!N->mayHaveSideEffects()) {
3857 N->eraseFromParent(); // Instruction folded away, don't need actual
3858 // inst
3859 N = nullptr;
3860 }
3861 } else {
3862 if (!BBI->use_empty())
3863 TranslateMap[&*BBI] = N;
3864 }
3865 if (N) {
3866 // Copy all debug-info attached to instructions from the last we
3867 // successfully clone, up to this instruction (they might have been
3868 // folded away).
3869 for (; SrcDbgCursor != BBI; ++SrcDbgCursor)
3870 N->cloneDebugInfoFrom(&*SrcDbgCursor);
3871 SrcDbgCursor = std::next(BBI);
3872 // Clone debug-info on this instruction too.
3873 N->cloneDebugInfoFrom(&*BBI);
3874
3875 // Register the new instruction with the assumption cache if necessary.
3876 if (auto *Assume = dyn_cast<AssumeInst>(N))
3877 if (AC)
3878 AC->registerAssumption(Assume);
3879 }
3880 }
3881
3882 for (; &*SrcDbgCursor != BI; ++SrcDbgCursor)
3883 InsertPt->cloneDebugInfoFrom(&*SrcDbgCursor);
3884 InsertPt->cloneDebugInfoFrom(BI);
3885
3886 BB->removePredecessor(EdgeBB);
3887 UncondBrInst *EdgeBI = cast<UncondBrInst>(EdgeBB->getTerminator());
3888 EdgeBI->setSuccessor(0, RealDest);
3889 EdgeBI->setDebugLoc(BI->getDebugLoc());
3890
3891 if (DTU) {
3893 Updates.push_back({DominatorTree::Delete, EdgeBB, BB});
3894 Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest});
3895 DTU->applyUpdates(Updates);
3896 }
3897
3898 // For simplicity, we created a separate basic block for the edge. Merge
3899 // it back into the predecessor if possible. This not only avoids
3900 // unnecessary SimplifyCFG iterations, but also makes sure that we don't
3901 // bypass the check for trivial cycles above.
3902 MergeBlockIntoPredecessor(EdgeBB, DTU);
3903
3904 // Signal repeat, simplifying any other constants.
3905 return std::nullopt;
3906 }
3907
3908 return false;
3909}
3910
3911bool SimplifyCFGOpt::foldCondBranchOnValueKnownInPredecessor(CondBrInst *BI) {
3912 // Threading through a loop header can introduce a cycle that bypasses the
3913 // header.
3914 bool KeepCanonicalLoop =
3915 Options.NeedCanonicalLoop && is_contained(LoopHeaders, BI->getParent());
3916
3917 std::optional<bool> Result;
3918 bool EverChanged = false;
3919 do {
3920 // Note that None means "we changed things, but recurse further."
3922 BI, TTI, DTU, Options.AC, DL, KeepCanonicalLoop);
3923 EverChanged |= Result == std::nullopt || *Result;
3924 } while (Result == std::nullopt);
3925 return EverChanged;
3926}
3927
3928/// Given a BB that starts with the specified two-entry PHI node,
3929/// see if we can eliminate it.
3932 const DataLayout &DL,
3933 bool SpeculateUnpredictables) {
3934 // Ok, this is a two entry PHI node. Check to see if this is a simple "if
3935 // statement", which has a very simple dominance structure. Basically, we
3936 // are trying to find the condition that is being branched on, which
3937 // subsequently causes this merge to happen. We really want control
3938 // dependence information for this check, but simplifycfg can't keep it up
3939 // to date, and this catches most of the cases we care about anyway.
3940 BasicBlock *BB = PN->getParent();
3941
3942 BasicBlock *IfTrue, *IfFalse;
3943 CondBrInst *DomBI = GetIfCondition(BB, IfTrue, IfFalse);
3944 if (!DomBI)
3945 return false;
3946 Value *IfCond = DomBI->getCondition();
3947 // Don't bother if the branch will be constant folded trivially.
3948 if (isa<ConstantInt>(IfCond))
3949 return false;
3950
3951 BasicBlock *DomBlock = DomBI->getParent();
3953 llvm::copy_if(PN->blocks(), std::back_inserter(IfBlocks),
3954 [](BasicBlock *IfBlock) {
3955 return isa<UncondBrInst>(IfBlock->getTerminator());
3956 });
3957 assert((IfBlocks.size() == 1 || IfBlocks.size() == 2) &&
3958 "Will have either one or two blocks to speculate.");
3959
3960 // If the branch is non-unpredictable, see if we either predictably jump to
3961 // the merge bb (if we have only a single 'then' block), or if we predictably
3962 // jump to one specific 'then' block (if we have two of them).
3963 // It isn't beneficial to speculatively execute the code
3964 // from the block that we know is predictably not entered.
3965 bool IsUnpredictable = DomBI->getMetadata(LLVMContext::MD_unpredictable);
3966 if (!IsUnpredictable) {
3967 uint64_t TWeight, FWeight;
3968 if (extractBranchWeights(*DomBI, TWeight, FWeight) &&
3969 (TWeight + FWeight) != 0) {
3970 BranchProbability BITrueProb =
3971 BranchProbability::getBranchProbability(TWeight, TWeight + FWeight);
3972 BranchProbability Likely = TTI.getPredictableBranchThreshold();
3973 BranchProbability BIFalseProb = BITrueProb.getCompl();
3974 if (IfBlocks.size() == 1) {
3975 BranchProbability BIBBProb =
3976 DomBI->getSuccessor(0) == BB ? BITrueProb : BIFalseProb;
3977 if (BIBBProb >= Likely)
3978 return false;
3979 } else {
3980 if (BITrueProb >= Likely || BIFalseProb >= Likely)
3981 return false;
3982 }
3983 }
3984 }
3985
3986 // Don't try to fold an unreachable block. For example, the phi node itself
3987 // can't be the candidate if-condition for a select that we want to form.
3988 if (auto *IfCondPhiInst = dyn_cast<PHINode>(IfCond))
3989 if (IfCondPhiInst->getParent() == BB)
3990 return false;
3991
3992 // Okay, we found that we can merge this two-entry phi node into a select.
3993 // Doing so would require us to fold *all* two entry phi nodes in this block.
3994 // At some point this becomes non-profitable (particularly if the target
3995 // doesn't support cmov's). Only do this transformation if there are two or
3996 // fewer PHI nodes in this block.
3997 unsigned NumPhis = 0;
3998 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I)
3999 if (NumPhis > 2)
4000 return false;
4001
4002 // Loop over the PHI's seeing if we can promote them all to select
4003 // instructions. While we are at it, keep track of the instructions
4004 // that need to be moved to the dominating block.
4005 SmallPtrSet<Instruction *, 4> AggressiveInsts;
4006 SmallPtrSet<Instruction *, 2> ZeroCostInstructions;
4007 InstructionCost Cost = 0;
4008 InstructionCost Budget =
4010 if (SpeculateUnpredictables && IsUnpredictable)
4011 Budget += TTI.getBranchMispredictPenalty();
4012
4013 bool Changed = false;
4014 for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
4015 PHINode *PN = cast<PHINode>(II++);
4016 if (Value *V = simplifyInstruction(PN, {DL, PN})) {
4017 PN->replaceAllUsesWith(V);
4018 PN->eraseFromParent();
4019 Changed = true;
4020 continue;
4021 }
4022
4023 if (!dominatesMergePoint(PN->getIncomingValue(0), BB, DomBI,
4024 AggressiveInsts, Cost, Budget, TTI, AC,
4025 ZeroCostInstructions) ||
4026 !dominatesMergePoint(PN->getIncomingValue(1), BB, DomBI,
4027 AggressiveInsts, Cost, Budget, TTI, AC,
4028 ZeroCostInstructions))
4029 return Changed;
4030 }
4031
4032 // If we folded the first phi, PN dangles at this point. Refresh it. If
4033 // we ran out of PHIs then we simplified them all.
4034 PN = dyn_cast<PHINode>(BB->begin());
4035 if (!PN)
4036 return true;
4037
4038 // Don't fold i1 branches on PHIs which contain binary operators or
4039 // (possibly inverted) select form of or/ands if their parameters are
4040 // an equality test.
4041 auto IsBinOpOrAndEq = [](Value *V) {
4043 if (match(V, m_CombineOr(
4048 m_Cmp(Pred, m_Value(), m_Value()))))) {
4049 return CmpInst::isEquality(Pred);
4050 }
4051 return false;
4052 };
4053 if (PN->getType()->isIntegerTy(1) &&
4054 (IsBinOpOrAndEq(PN->getIncomingValue(0)) ||
4055 IsBinOpOrAndEq(PN->getIncomingValue(1)) || IsBinOpOrAndEq(IfCond)))
4056 return Changed;
4057
4058 // If all PHI nodes are promotable, check to make sure that all instructions
4059 // in the predecessor blocks can be promoted as well. If not, we won't be able
4060 // to get rid of the control flow, so it's not worth promoting to select
4061 // instructions.
4062 for (BasicBlock *IfBlock : IfBlocks)
4063 for (BasicBlock::iterator I = IfBlock->begin(); !I->isTerminator(); ++I)
4064 if (!AggressiveInsts.count(&*I) && !isa<PseudoProbeInst>(I)) {
4065 // This is not an aggressive instruction that we can promote.
4066 // Because of this, we won't be able to get rid of the control flow, so
4067 // the xform is not worth it.
4068 return Changed;
4069 }
4070
4071 // If either of the blocks has it's address taken, we can't do this fold.
4072 if (any_of(IfBlocks,
4073 [](BasicBlock *IfBlock) { return IfBlock->hasAddressTaken(); }))
4074 return Changed;
4075
4076 LLVM_DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfCond;
4077 if (IsUnpredictable) dbgs() << " (unpredictable)";
4078 dbgs() << " T: " << IfTrue->getName()
4079 << " F: " << IfFalse->getName() << "\n");
4080
4081 // If we can still promote the PHI nodes after this gauntlet of tests,
4082 // do all of the PHI's now.
4083
4084 // Move all 'aggressive' instructions, which are defined in the
4085 // conditional parts of the if's up to the dominating block.
4086 for (BasicBlock *IfBlock : IfBlocks)
4087 hoistAllInstructionsInto(DomBlock, DomBI, IfBlock);
4088
4089 IRBuilder<NoFolder> Builder(DomBI);
4090 // Propagate fast-math-flags from phi nodes to replacement selects.
4091 while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
4092 // Change the PHI node into a select instruction.
4093 Value *TrueVal = PN->getIncomingValueForBlock(IfTrue);
4094 Value *FalseVal = PN->getIncomingValueForBlock(IfFalse);
4095
4096 Value *Sel = Builder.CreateSelectFMF(IfCond, TrueVal, FalseVal,
4097 isa<FPMathOperator>(PN) ? PN : nullptr,
4098 "", DomBI);
4099 PN->replaceAllUsesWith(Sel);
4100 Sel->takeName(PN);
4101 PN->eraseFromParent();
4102 }
4103
4104 // At this point, all IfBlocks are empty, so our if statement
4105 // has been flattened. Change DomBlock to jump directly to our new block to
4106 // avoid other simplifycfg's kicking in on the diamond.
4107 Builder.CreateBr(BB);
4108
4110 if (DTU) {
4111 Updates.push_back({DominatorTree::Insert, DomBlock, BB});
4112 for (auto *Successor : successors(DomBlock))
4113 Updates.push_back({DominatorTree::Delete, DomBlock, Successor});
4114 }
4115
4116 DomBI->eraseFromParent();
4117 if (DTU)
4118 DTU->applyUpdates(Updates);
4119
4120 return true;
4121}
4122
4125 Value *RHS, const Twine &Name = "") {
4126 // Try to relax logical op to binary op.
4127 if (impliesPoison(RHS, LHS))
4128 return Builder.CreateBinOp(Opc, LHS, RHS, Name);
4129 if (Opc == Instruction::And)
4130 return Builder.CreateLogicalAnd(LHS, RHS, Name);
4131 if (Opc == Instruction::Or)
4132 return Builder.CreateLogicalOr(LHS, RHS, Name);
4133 llvm_unreachable("Invalid logical opcode");
4134}
4135
4136/// Return true if either PBI or BI has branch weight available, and store
4137/// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does
4138/// not have branch weight, use 1:1 as its weight.
4140 uint64_t &PredTrueWeight,
4141 uint64_t &PredFalseWeight,
4142 uint64_t &SuccTrueWeight,
4143 uint64_t &SuccFalseWeight) {
4144 bool PredHasWeights =
4145 extractBranchWeights(*PBI, PredTrueWeight, PredFalseWeight);
4146 bool SuccHasWeights =
4147 extractBranchWeights(*BI, SuccTrueWeight, SuccFalseWeight);
4148 if (PredHasWeights || SuccHasWeights) {
4149 if (!PredHasWeights)
4150 PredTrueWeight = PredFalseWeight = 1;
4151 if (!SuccHasWeights)
4152 SuccTrueWeight = SuccFalseWeight = 1;
4153 return true;
4154 } else {
4155 return false;
4156 }
4157}
4158
4159/// Determine if the two branches share a common destination and deduce a glue
4160/// that joins the branches' conditions to arrive at the common destination if
4161/// that would be profitable.
4162static std::optional<std::tuple<BasicBlock *, Instruction::BinaryOps, bool>>
4164 const TargetTransformInfo *TTI) {
4165 assert(BI && PBI && "Both blocks must end with a conditional branches.");
4167 "PredBB must be a predecessor of BB.");
4168
4169 // We have the potential to fold the conditions together, but if the
4170 // predecessor branch is predictable, we may not want to merge them.
4171 uint64_t PTWeight, PFWeight;
4172 BranchProbability PBITrueProb, Likely;
4173 if (TTI && !PBI->getMetadata(LLVMContext::MD_unpredictable) &&
4174 extractBranchWeights(*PBI, PTWeight, PFWeight) &&
4175 (PTWeight + PFWeight) != 0) {
4176 PBITrueProb =
4177 BranchProbability::getBranchProbability(PTWeight, PTWeight + PFWeight);
4178 Likely = TTI->getPredictableBranchThreshold();
4179 }
4180
4181 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
4182 // Speculate the 2nd condition unless the 1st is probably true.
4183 if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
4184 return {{BI->getSuccessor(0), Instruction::Or, false}};
4185 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
4186 // Speculate the 2nd condition unless the 1st is probably false.
4187 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
4188 return {{BI->getSuccessor(1), Instruction::And, false}};
4189 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
4190 // Speculate the 2nd condition unless the 1st is probably true.
4191 if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
4192 return {{BI->getSuccessor(1), Instruction::And, true}};
4193 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
4194 // Speculate the 2nd condition unless the 1st is probably false.
4195 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
4196 return {{BI->getSuccessor(0), Instruction::Or, true}};
4197 }
4198 return std::nullopt;
4199}
4200
4202 DomTreeUpdater *DTU,
4203 MemorySSAUpdater *MSSAU,
4204 const TargetTransformInfo *TTI) {
4205 BasicBlock *BB = BI->getParent();
4206 BasicBlock *PredBlock = PBI->getParent();
4207
4208 // Determine if the two branches share a common destination.
4209 BasicBlock *CommonSucc;
4211 bool InvertPredCond;
4212 std::tie(CommonSucc, Opc, InvertPredCond) =
4214
4215 LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB);
4216
4218 *BB->getModule(), ConstantFolder{},
4220 // The builder is used to create instructions to eliminate the branch in
4221 // BB. If BB's terminator has !annotation metadata, add it to the new
4222 // instructions.
4223 I->copyMetadata(*BB->getTerminator(), LLVMContext::MD_annotation);
4224 }));
4225 Builder.SetInsertPoint(PBI);
4226
4227 // If we need to invert the condition in the pred block to match, do so now.
4228 if (InvertPredCond) {
4229 InvertBranch(PBI, Builder);
4230 }
4231
4232 BasicBlock *UniqueSucc =
4233 PBI->getSuccessor(0) == BB ? BI->getSuccessor(0) : BI->getSuccessor(1);
4234
4235 // Before cloning instructions, notify the successor basic block that it
4236 // is about to have a new predecessor. This will update PHI nodes,
4237 // which will allow us to update live-out uses of bonus instructions.
4238 addPredecessorToBlock(UniqueSucc, PredBlock, BB, MSSAU);
4239
4240 // Try to update branch weights.
4241 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
4242 SmallVector<uint64_t, 2> MDWeights;
4243 if (extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
4244 SuccTrueWeight, SuccFalseWeight)) {
4245
4246 if (PBI->getSuccessor(0) == BB) {
4247 // PBI: br i1 %x, BB, FalseDest
4248 // BI: br i1 %y, UniqueSucc, FalseDest
4249 // TrueWeight is TrueWeight for PBI * TrueWeight for BI.
4250 MDWeights.push_back(PredTrueWeight * SuccTrueWeight);
4251 // FalseWeight is FalseWeight for PBI * TotalWeight for BI +
4252 // TrueWeight for PBI * FalseWeight for BI.
4253 // We assume that total weights of a CondBrInst can fit into 32 bits.
4254 // Therefore, we will not have overflow using 64-bit arithmetic.
4255 MDWeights.push_back(PredFalseWeight * (SuccFalseWeight + SuccTrueWeight) +
4256 PredTrueWeight * SuccFalseWeight);
4257 } else {
4258 // PBI: br i1 %x, TrueDest, BB
4259 // BI: br i1 %y, TrueDest, UniqueSucc
4260 // TrueWeight is TrueWeight for PBI * TotalWeight for BI +
4261 // FalseWeight for PBI * TrueWeight for BI.
4262 MDWeights.push_back(PredTrueWeight * (SuccFalseWeight + SuccTrueWeight) +
4263 PredFalseWeight * SuccTrueWeight);
4264 // FalseWeight is FalseWeight for PBI * FalseWeight for BI.
4265 MDWeights.push_back(PredFalseWeight * SuccFalseWeight);
4266 }
4267
4268 setFittedBranchWeights(*PBI, MDWeights, /*IsExpected=*/false,
4269 /*ElideAllZero=*/true);
4270
4271 // TODO: If BB is reachable from all paths through PredBlock, then we
4272 // could replace PBI's branch probabilities with BI's.
4273 } else {
4274 PBI->setMetadata(LLVMContext::MD_prof, nullptr);
4276 }
4277
4278 // Now, update the CFG.
4279 PBI->setSuccessor(PBI->getSuccessor(0) != BB, UniqueSucc);
4280
4281 if (DTU)
4282 DTU->applyUpdates({{DominatorTree::Insert, PredBlock, UniqueSucc},
4283 {DominatorTree::Delete, PredBlock, BB}});
4284
4285 // If BI was a loop latch, it may have had associated loop metadata.
4286 // We need to copy it to the new latch, that is, PBI.
4287 if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop))
4288 PBI->setMetadata(LLVMContext::MD_loop, LoopMD);
4289
4290 ValueToValueMapTy VMap; // maps original values to cloned values
4292
4293 Module *M = BB->getModule();
4294
4295 PredBlock->getTerminator()->cloneDebugInfoFrom(BB->getTerminator());
4296 for (DbgVariableRecord &DVR :
4298 RemapDbgRecord(M, &DVR, VMap,
4300 }
4301
4302 // Now that the Cond was cloned into the predecessor basic block,
4303 // or/and the two conditions together.
4304 Value *BICond = VMap[BI->getCondition()];
4305 PBI->setCondition(
4306 createLogicalOp(Builder, Opc, PBI->getCondition(), BICond, "or.cond"));
4307 if (auto *SI = dyn_cast<SelectInst>(PBI->getCondition())) {
4308 if (!MDWeights.empty()) {
4309 assert(isSelectInRoleOfConjunctionOrDisjunction(SI));
4310 setFittedBranchWeights(*SI, {MDWeights[0], MDWeights[1]},
4311 /*IsExpected=*/false, /*ElideAllZero=*/true);
4312 } else {
4314 }
4315 }
4316
4317 ++NumFoldBranchToCommonDest;
4318 return true;
4319}
4320
4321/// Return if an instruction's type or any of its operands' types are a vector
4322/// type.
4323static bool isVectorOp(Instruction &I) {
4324 return I.getType()->isVectorTy() || any_of(I.operands(), [](Use &U) {
4325 return U->getType()->isVectorTy();
4326 });
4327}
4328
4329/// If this basic block is simple enough, and if a predecessor branches to us
4330/// and one of our successors, fold the block into the predecessor and use
4331/// logical operations to pick the right destination.
4333 MemorySSAUpdater *MSSAU,
4334 const TargetTransformInfo *TTI,
4335 AssumptionCache *AC,
4336 unsigned BonusInstThreshold) {
4337 BasicBlock *BB = BI->getParent();
4341
4343
4345 Cond->getParent() != BB || !Cond->hasOneUse())
4346 return false;
4347
4348 // Finally, don't infinitely unroll conditional loops.
4349 if (is_contained(successors(BB), BB))
4350 return false;
4351
4352 // With which predecessors will we want to deal with?
4354 for (BasicBlock *PredBlock : predecessors(BB)) {
4355 CondBrInst *PBI = dyn_cast<CondBrInst>(PredBlock->getTerminator());
4356
4357 // Check that we have two conditional branches. If there is a PHI node in
4358 // the common successor, verify that the same value flows in from both
4359 // blocks.
4360 if (!PBI || !safeToMergeTerminators(BI, PBI))
4361 continue;
4362
4363 // Determine if the two branches share a common destination.
4364 BasicBlock *CommonSucc;
4366 bool InvertPredCond;
4367 if (auto Recipe = shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI))
4368 std::tie(CommonSucc, Opc, InvertPredCond) = *Recipe;
4369 else
4370 continue;
4371
4372 // Check the cost of inserting the necessary logic before performing the
4373 // transformation.
4374 if (TTI) {
4375 Type *Ty = BI->getCondition()->getType();
4376 InstructionCost Cost = TTI->getArithmeticInstrCost(Opc, Ty, CostKind);
4377 if (InvertPredCond && (!PBI->getCondition()->hasOneUse() ||
4378 !isa<CmpInst>(PBI->getCondition())))
4379 Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind);
4380
4382 continue;
4383 }
4384
4385 // Ok, we do want to deal with this predecessor. Record it.
4386 Preds.emplace_back(PredBlock);
4387 }
4388
4389 // If there aren't any predecessors into which we can fold,
4390 // don't bother checking the cost.
4391 if (Preds.empty())
4392 return false;
4393
4394 // Only allow this transformation if computing the condition doesn't involve
4395 // too many instructions and these involved instructions can be executed
4396 // unconditionally. We denote all involved instructions except the condition
4397 // as "bonus instructions", and only allow this transformation when the
4398 // number of the bonus instructions we'll need to create when cloning into
4399 // each predecessor does not exceed a certain threshold.
4400 unsigned NumBonusInsts = 0;
4401 bool SawVectorOp = false;
4402 const unsigned PredCount = Preds.size();
4403 // Speculated instructions will be inserted before the terminator of the
4404 // predecessor. Only handle the simple case of one predecessor.
4405 const Instruction *CtxI =
4406 PredCount == 1 ? Preds[0]->getTerminator() : nullptr;
4407 for (Instruction &I : *BB) {
4408 // Don't check the branch condition comparison itself.
4409 if (&I == Cond)
4410 continue;
4411 // Ignore the terminator.
4413 continue;
4414 // Pseudo probes aren't speculatable but can be dropped on fold.
4416 continue;
4417 // I must be safe to execute unconditionally.
4418 if (!isSafeToSpeculativelyExecute(&I, CtxI, AC))
4419 return false;
4420 SawVectorOp |= isVectorOp(I);
4421
4422 // Account for the cost of duplicating this instruction into each
4423 // predecessor. Ignore free instructions.
4424 if (!TTI || TTI->getInstructionCost(&I, CostKind) !=
4426 NumBonusInsts += PredCount;
4427
4428 // Early exits once we reach the limit.
4429 if (NumBonusInsts >
4430 BonusInstThreshold * BranchFoldToCommonDestVectorMultiplier)
4431 return false;
4432 }
4433
4434 auto IsBCSSAUse = [BB, &I](Use &U) {
4435 auto *UI = cast<Instruction>(U.getUser());
4436 if (auto *PN = dyn_cast<PHINode>(UI))
4437 return PN->getIncomingBlock(U) == BB;
4438 return UI->getParent() == BB && I.comesBefore(UI);
4439 };
4440
4441 // Does this instruction require rewriting of uses?
4442 if (!all_of(I.uses(), IsBCSSAUse))
4443 return false;
4444 }
4445 if (NumBonusInsts >
4446 BonusInstThreshold *
4447 (SawVectorOp ? BranchFoldToCommonDestVectorMultiplier : 1))
4448 return false;
4449
4450 // Ok, we have the budget. Perform the transformation.
4451 for (BasicBlock *PredBlock : Preds) {
4452 auto *PBI = cast<CondBrInst>(PredBlock->getTerminator());
4453 return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI);
4454 }
4455 return false;
4456}
4457
4458// If there is only one store in BB1 and BB2, return it, otherwise return
4459// nullptr.
4461 StoreInst *S = nullptr;
4462 for (auto *BB : {BB1, BB2}) {
4463 if (!BB)
4464 continue;
4465 for (auto &I : *BB)
4466 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4467 if (S)
4468 // Multiple stores seen.
4469 return nullptr;
4470 else
4471 S = SI;
4472 }
4473 }
4474 return S;
4475}
4476
4478 Value *AlternativeV = nullptr) {
4479 // PHI is going to be a PHI node that allows the value V that is defined in
4480 // BB to be referenced in BB's only successor.
4481 //
4482 // If AlternativeV is nullptr, the only value we care about in PHI is V. It
4483 // doesn't matter to us what the other operand is (it'll never get used). We
4484 // could just create a new PHI with an undef incoming value, but that could
4485 // increase register pressure if EarlyCSE/InstCombine can't fold it with some
4486 // other PHI. So here we directly look for some PHI in BB's successor with V
4487 // as an incoming operand. If we find one, we use it, else we create a new
4488 // one.
4489 //
4490 // If AlternativeV is not nullptr, we care about both incoming values in PHI.
4491 // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV]
4492 // where OtherBB is the single other predecessor of BB's only successor.
4493 PHINode *PHI = nullptr;
4494 BasicBlock *Succ = BB->getSingleSuccessor();
4495
4496 for (auto I = Succ->begin(); isa<PHINode>(I); ++I)
4497 if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) {
4498 PHI = cast<PHINode>(I);
4499 if (!AlternativeV)
4500 break;
4501
4502 assert(Succ->hasNPredecessors(2));
4503 auto PredI = pred_begin(Succ);
4504 BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI;
4505 if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV)
4506 break;
4507 PHI = nullptr;
4508 }
4509 if (PHI)
4510 return PHI;
4511
4512 // If V is not an instruction defined in BB, just return it.
4513 if (!AlternativeV &&
4515 return V;
4516
4517 PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge");
4518 PHI->insertBefore(Succ->begin());
4519 PHI->addIncoming(V, BB);
4520 for (BasicBlock *PredBB : predecessors(Succ))
4521 if (PredBB != BB)
4522 PHI->addIncoming(
4523 AlternativeV ? AlternativeV : PoisonValue::get(V->getType()), PredBB);
4524 return PHI;
4525}
4526
4528 BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB,
4529 BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond,
4530 DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI) {
4531 // For every pointer, there must be exactly two stores, one coming from
4532 // PTB or PFB, and the other from QTB or QFB. We don't support more than one
4533 // store (to any address) in PTB,PFB or QTB,QFB.
4534 // FIXME: We could relax this restriction with a bit more work and performance
4535 // testing.
4536 StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB);
4537 StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB);
4538 if (!PStore || !QStore)
4539 return false;
4540
4541 // Now check the stores are compatible.
4542 if (!QStore->isUnordered() || !PStore->isUnordered() ||
4543 PStore->getOrdering() != QStore->getOrdering() ||
4544 PStore->getSyncScopeID() != QStore->getSyncScopeID() ||
4545 PStore->getValueOperand()->getType() !=
4546 QStore->getValueOperand()->getType())
4547 return false;
4548
4549 // Check that sinking the store won't cause program behavior changes. Sinking
4550 // the store out of the Q blocks won't change any behavior as we're sinking
4551 // from a block to its unconditional successor. But we're moving a store from
4552 // the P blocks down through the middle block (QBI) and past both QFB and QTB.
4553 // So we need to check that there are no aliasing loads or stores in
4554 // QBI, QTB and QFB. We also need to check there are no conflicting memory
4555 // operations between PStore and the end of its parent block.
4556 //
4557 // The ideal way to do this is to query AliasAnalysis, but we don't
4558 // preserve AA currently so that is dangerous. Be super safe and just
4559 // check there are no other memory operations at all.
4560 for (auto &I : *QFB->getSinglePredecessor())
4561 if (I.mayReadOrWriteMemory())
4562 return false;
4563 for (auto &I : *QFB)
4564 if (&I != QStore && I.mayReadOrWriteMemory())
4565 return false;
4566 if (QTB)
4567 for (auto &I : *QTB)
4568 if (&I != QStore && I.mayReadOrWriteMemory())
4569 return false;
4570 for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end();
4571 I != E; ++I)
4572 if (&*I != PStore && I->mayReadOrWriteMemory())
4573 return false;
4574
4575 // If we're not in aggressive mode, we only optimize if we have some
4576 // confidence that by optimizing we'll allow P and/or Q to be if-converted.
4577 auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) {
4578 if (!BB)
4579 return true;
4580 // Heuristic: if the block can be if-converted/phi-folded and the
4581 // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
4582 // thread this store.
4583 InstructionCost Cost = 0;
4584 InstructionCost Budget =
4586 for (auto &I : *BB) {
4587 // Consider terminator instruction to be free.
4588 if (I.isTerminator())
4589 continue;
4590 // If this is one the stores that we want to speculate out of this BB,
4591 // then don't count it's cost, consider it to be free.
4592 if (auto *S = dyn_cast<StoreInst>(&I))
4593 if (llvm::find(FreeStores, S))
4594 continue;
4595 // Else, we have a white-list of instructions that we are ak speculating.
4597 return false; // Not in white-list - not worthwhile folding.
4598 // And finally, if this is a non-free instruction that we are okay
4599 // speculating, ensure that we consider the speculation budget.
4600 Cost +=
4601 TTI.getInstructionCost(&I, TargetTransformInfo::TCK_SizeAndLatency);
4602 if (Cost > Budget)
4603 return false; // Eagerly refuse to fold as soon as we're out of budget.
4604 }
4605 assert(Cost <= Budget &&
4606 "When we run out of budget we will eagerly return from within the "
4607 "per-instruction loop.");
4608 return true;
4609 };
4610
4611 const std::array<StoreInst *, 2> FreeStores = {PStore, QStore};
4613 (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) ||
4614 !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores)))
4615 return false;
4616
4617 // If PostBB has more than two predecessors, we need to split it so we can
4618 // sink the store.
4619 if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) {
4620 // We know that QFB's only successor is PostBB. And QFB has a single
4621 // predecessor. If QTB exists, then its only successor is also PostBB.
4622 // If QTB does not exist, then QFB's only predecessor has a conditional
4623 // branch to QFB and PostBB.
4624 BasicBlock *TruePred = QTB ? QTB : QFB->getSinglePredecessor();
4625 BasicBlock *NewBB =
4626 SplitBlockPredecessors(PostBB, {QFB, TruePred}, "condstore.split", DTU);
4627 if (!NewBB)
4628 return false;
4629 PostBB = NewBB;
4630 }
4631
4632 // OK, we're going to sink the stores to PostBB. The store has to be
4633 // conditional though, so first create the predicate.
4634 CondBrInst *PBranch =
4636 CondBrInst *QBranch =
4638 Value *PCond = PBranch->getCondition();
4639 Value *QCond = QBranch->getCondition();
4640
4642 PStore->getParent());
4644 QStore->getParent(), PPHI);
4645
4646 IRBuilder<> QB(PostBB->getFirstInsertionPt());
4647
4648 InvertPCond ^= (PStore->getParent() != PTB);
4649 InvertQCond ^= (QStore->getParent() != QTB);
4650 Value *PPred = InvertPCond ? QB.CreateNot(PCond) : PCond;
4651 Value *QPred = InvertQCond ? QB.CreateNot(QCond) : QCond;
4652
4653 Value *CombinedPred = QB.CreateOr(PPred, QPred);
4654
4655 BasicBlock::iterator InsertPt = QB.GetInsertPoint();
4656 auto *T = SplitBlockAndInsertIfThen(CombinedPred, InsertPt,
4657 /*Unreachable=*/false,
4658 /*BranchWeights=*/nullptr, DTU);
4659 if (hasBranchWeightMD(*PBranch) && hasBranchWeightMD(*QBranch)) {
4660 SmallVector<uint32_t, 2> PWeights, QWeights;
4661 extractBranchWeights(*PBranch, PWeights);
4662 extractBranchWeights(*QBranch, QWeights);
4663 if (InvertPCond)
4664 std::swap(PWeights[0], PWeights[1]);
4665 if (InvertQCond)
4666 std::swap(QWeights[0], QWeights[1]);
4667 auto CombinedWeights = getDisjunctionWeights(PWeights, QWeights);
4669 {CombinedWeights[0], CombinedWeights[1]},
4670 /*IsExpected=*/false, /*ElideAllZero=*/true);
4671 }
4672
4673 QB.SetInsertPoint(T);
4674 StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address));
4675 combineMetadataForCSE(QStore, PStore, true);
4676 SI->copyMetadata(*QStore);
4677 // Update any dbg.assign intrinsics to track the merged value (QPHI) instead
4678 // of the original constant values, likely making these identical.
4679 for (auto *DbgAssign : at::getDVRAssignmentMarkers(SI)) {
4680 if (llvm::is_contained(DbgAssign->location_ops(),
4681 PStore->getValueOperand()))
4682 DbgAssign->replaceVariableLocationOp(PStore->getValueOperand(), QPHI);
4683 if (llvm::is_contained(DbgAssign->location_ops(),
4684 QStore->getValueOperand()))
4685 DbgAssign->replaceVariableLocationOp(QStore->getValueOperand(), QPHI);
4686 }
4687
4688 // Choose the minimum alignment. If we could prove both stores execute, we
4689 // could use biggest one. In this case, though, we only know that one of the
4690 // stores executes. And we don't know it's safe to take the alignment from a
4691 // store that doesn't execute.
4692 SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign()));
4693
4694 if (QStore->isAtomic())
4695 SI->setAtomic(QStore->getOrdering(), QStore->getSyncScopeID());
4696
4697 QStore->eraseFromParent();
4698 PStore->eraseFromParent();
4699
4700 return true;
4701}
4702
4704 DomTreeUpdater *DTU, const DataLayout &DL,
4705 const TargetTransformInfo &TTI) {
4706 // The intention here is to find diamonds or triangles (see below) where each
4707 // conditional block contains a store to the same address. Both of these
4708 // stores are conditional, so they can't be unconditionally sunk. But it may
4709 // be profitable to speculatively sink the stores into one merged store at the
4710 // end, and predicate the merged store on the union of the two conditions of
4711 // PBI and QBI.
4712 //
4713 // This can reduce the number of stores executed if both of the conditions are
4714 // true, and can allow the blocks to become small enough to be if-converted.
4715 // This optimization will also chain, so that ladders of test-and-set
4716 // sequences can be if-converted away.
4717 //
4718 // We only deal with simple diamonds or triangles:
4719 //
4720 // PBI or PBI or a combination of the two
4721 // / \ | \
4722 // PTB PFB | PFB
4723 // \ / | /
4724 // QBI QBI
4725 // / \ | \
4726 // QTB QFB | QFB
4727 // \ / | /
4728 // PostBB PostBB
4729 //
4730 // We model triangles as a type of diamond with a nullptr "true" block.
4731 // Triangles are canonicalized so that the fallthrough edge is represented by
4732 // a true condition, as in the diagram above.
4733 BasicBlock *PTB = PBI->getSuccessor(0);
4734 BasicBlock *PFB = PBI->getSuccessor(1);
4735 BasicBlock *QTB = QBI->getSuccessor(0);
4736 BasicBlock *QFB = QBI->getSuccessor(1);
4737 BasicBlock *PostBB = QFB->getSingleSuccessor();
4738
4739 // Make sure we have a good guess for PostBB. If QTB's only successor is
4740 // QFB, then QFB is a better PostBB.
4741 if (QTB->getSingleSuccessor() == QFB)
4742 PostBB = QFB;
4743
4744 // If we couldn't find a good PostBB, stop.
4745 if (!PostBB)
4746 return false;
4747
4748 bool InvertPCond = false, InvertQCond = false;
4749 // Canonicalize fallthroughs to the true branches.
4750 if (PFB == QBI->getParent()) {
4751 std::swap(PFB, PTB);
4752 InvertPCond = true;
4753 }
4754 if (QFB == PostBB) {
4755 std::swap(QFB, QTB);
4756 InvertQCond = true;
4757 }
4758
4759 // From this point on we can assume PTB or QTB may be fallthroughs but PFB
4760 // and QFB may not. Model fallthroughs as a nullptr block.
4761 if (PTB == QBI->getParent())
4762 PTB = nullptr;
4763 if (QTB == PostBB)
4764 QTB = nullptr;
4765
4766 // Legality bailouts. We must have at least the non-fallthrough blocks and
4767 // the post-dominating block, and the non-fallthroughs must only have one
4768 // predecessor.
4769 auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) {
4770 return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S;
4771 };
4772 if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) ||
4773 !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB))
4774 return false;
4775 if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) ||
4776 (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB)))
4777 return false;
4778 if (!QBI->getParent()->hasNUses(2))
4779 return false;
4780
4781 // OK, this is a sequence of two diamonds or triangles.
4782 // Check if there are stores in PTB or PFB that are repeated in QTB or QFB.
4783 SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses;
4784 for (auto *BB : {PTB, PFB}) {
4785 if (!BB)
4786 continue;
4787 for (auto &I : *BB)
4789 PStoreAddresses.insert(SI->getPointerOperand());
4790 }
4791 for (auto *BB : {QTB, QFB}) {
4792 if (!BB)
4793 continue;
4794 for (auto &I : *BB)
4796 QStoreAddresses.insert(SI->getPointerOperand());
4797 }
4798
4799 set_intersect(PStoreAddresses, QStoreAddresses);
4800 // set_intersect mutates PStoreAddresses in place. Rename it here to make it
4801 // clear what it contains.
4802 auto &CommonAddresses = PStoreAddresses;
4803
4804 bool Changed = false;
4805 for (auto *Address : CommonAddresses)
4806 Changed |=
4807 mergeConditionalStoreToAddress(PTB, PFB, QTB, QFB, PostBB, Address,
4808 InvertPCond, InvertQCond, DTU, DL, TTI);
4809 return Changed;
4810}
4811
4812/// If the previous block ended with a widenable branch, determine if reusing
4813/// the target block is profitable and legal. This will have the effect of
4814/// "widening" PBI, but doesn't require us to reason about hosting safety.
4816 DomTreeUpdater *DTU) {
4817 // TODO: This can be generalized in two important ways:
4818 // 1) We can allow phi nodes in IfFalseBB and simply reuse all the input
4819 // values from the PBI edge.
4820 // 2) We can sink side effecting instructions into BI's fallthrough
4821 // successor provided they doesn't contribute to computation of
4822 // BI's condition.
4823 BasicBlock *IfTrueBB = PBI->getSuccessor(0);
4824 BasicBlock *IfFalseBB = PBI->getSuccessor(1);
4825 if (!isWidenableBranch(PBI) || IfTrueBB != BI->getParent() ||
4826 !BI->getParent()->getSinglePredecessor())
4827 return false;
4828 if (!IfFalseBB->phis().empty())
4829 return false; // TODO
4830 // This helps avoid infinite loop with SimplifyCondBranchToCondBranch which
4831 // may undo the transform done here.
4832 // TODO: There might be a more fine-grained solution to this.
4833 if (!llvm::succ_empty(IfFalseBB))
4834 return false;
4835 // Use lambda to lazily compute expensive condition after cheap ones.
4836 auto NoSideEffects = [](BasicBlock &BB) {
4837 return llvm::none_of(BB, [](const Instruction &I) {
4838 return I.mayWriteToMemory() || I.mayHaveSideEffects();
4839 });
4840 };
4841 if (BI->getSuccessor(1) != IfFalseBB && // no inf looping
4842 BI->getSuccessor(1)->getTerminatingDeoptimizeCall() && // profitability
4843 NoSideEffects(*BI->getParent())) {
4844 auto *OldSuccessor = BI->getSuccessor(1);
4845 OldSuccessor->removePredecessor(BI->getParent());
4846 BI->setSuccessor(1, IfFalseBB);
4847 if (DTU)
4848 DTU->applyUpdates(
4849 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
4850 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
4851 return true;
4852 }
4853 if (BI->getSuccessor(0) != IfFalseBB && // no inf looping
4854 BI->getSuccessor(0)->getTerminatingDeoptimizeCall() && // profitability
4855 NoSideEffects(*BI->getParent())) {
4856 auto *OldSuccessor = BI->getSuccessor(0);
4857 OldSuccessor->removePredecessor(BI->getParent());
4858 BI->setSuccessor(0, IfFalseBB);
4859 if (DTU)
4860 DTU->applyUpdates(
4861 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
4862 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
4863 return true;
4864 }
4865 return false;
4866}
4867
4868/// If we have a conditional branch as a predecessor of another block,
4869/// this function tries to simplify it. We know
4870/// that PBI and BI are both conditional branches, and BI is in one of the
4871/// successor blocks of PBI - PBI branches to BI.
4873 DomTreeUpdater *DTU,
4874 const DataLayout &DL,
4875 const TargetTransformInfo &TTI) {
4876 BasicBlock *BB = BI->getParent();
4877
4878 // If this block ends with a branch instruction, and if there is a
4879 // predecessor that ends on a branch of the same condition, make
4880 // this conditional branch redundant.
4881 if (PBI->getCondition() == BI->getCondition() &&
4882 PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
4883 // Okay, the outcome of this conditional branch is statically
4884 // knowable. If this block had a single pred, handle specially, otherwise
4885 // foldCondBranchOnValueKnownInPredecessor() will handle it.
4886 if (BB->getSinglePredecessor()) {
4887 // Turn this into a branch on constant.
4888 bool CondIsTrue = PBI->getSuccessor(0) == BB;
4889 BI->setCondition(
4890 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue));
4891 return true; // Nuke the branch on constant.
4892 }
4893 }
4894
4895 // If the previous block ended with a widenable branch, determine if reusing
4896 // the target block is profitable and legal. This will have the effect of
4897 // "widening" PBI, but doesn't require us to reason about hosting safety.
4898 if (tryWidenCondBranchToCondBranch(PBI, BI, DTU))
4899 return true;
4900
4901 // If both branches are conditional and both contain stores to the same
4902 // address, remove the stores from the conditionals and create a conditional
4903 // merged store at the end.
4904 if (MergeCondStores && mergeConditionalStores(PBI, BI, DTU, DL, TTI))
4905 return true;
4906
4907 // If this is a conditional branch in an empty block, and if any
4908 // predecessors are a conditional branch to one of our destinations,
4909 // fold the conditions into logical ops and one cond br.
4910
4911 // Ignore dbg intrinsics.
4912 if (&*BB->begin() != BI)
4913 return false;
4914
4915 int PBIOp, BIOp;
4916 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
4917 PBIOp = 0;
4918 BIOp = 0;
4919 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
4920 PBIOp = 0;
4921 BIOp = 1;
4922 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
4923 PBIOp = 1;
4924 BIOp = 0;
4925 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
4926 PBIOp = 1;
4927 BIOp = 1;
4928 } else {
4929 return false;
4930 }
4931
4932 // Check to make sure that the other destination of this branch
4933 // isn't BB itself. If so, this is an infinite loop that will
4934 // keep getting unwound.
4935 if (PBI->getSuccessor(PBIOp) == BB)
4936 return false;
4937
4938 // If predecessor's branch probability to BB is too low don't merge branches.
4939 SmallVector<uint32_t, 2> PredWeights;
4940 if (!PBI->getMetadata(LLVMContext::MD_unpredictable) &&
4941 extractBranchWeights(*PBI, PredWeights) &&
4942 (static_cast<uint64_t>(PredWeights[0]) + PredWeights[1]) != 0) {
4943
4945 PredWeights[PBIOp],
4946 static_cast<uint64_t>(PredWeights[0]) + PredWeights[1]);
4947
4948 BranchProbability Likely = TTI.getPredictableBranchThreshold();
4949 if (CommonDestProb >= Likely)
4950 return false;
4951 }
4952
4953 // Do not perform this transformation if it would require
4954 // insertion of a large number of select instructions. For targets
4955 // without predication/cmovs, this is a big pessimization.
4956
4957 BasicBlock *CommonDest = PBI->getSuccessor(PBIOp);
4958 BasicBlock *RemovedDest = PBI->getSuccessor(PBIOp ^ 1);
4959 unsigned NumPhis = 0;
4960 for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II);
4961 ++II, ++NumPhis) {
4962 if (NumPhis > 2) // Disable this xform.
4963 return false;
4964 }
4965
4966 // Finally, if everything is ok, fold the branches to logical ops.
4967 BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1);
4968
4969 LLVM_DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()
4970 << "AND: " << *BI->getParent());
4971
4973
4974 // If OtherDest *is* BB, then BB is a basic block with a single conditional
4975 // branch in it, where one edge (OtherDest) goes back to itself but the other
4976 // exits. We don't *know* that the program avoids the infinite loop
4977 // (even though that seems likely). If we do this xform naively, we'll end up
4978 // recursively unpeeling the loop. Since we know that (after the xform is
4979 // done) that the block *is* infinite if reached, we just make it an obviously
4980 // infinite loop with no cond branch.
4981 if (OtherDest == BB) {
4982 // Insert it at the end of the function, because it's either code,
4983 // or it won't matter if it's hot. :)
4984 BasicBlock *InfLoopBlock =
4985 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
4986 UncondBrInst::Create(InfLoopBlock, InfLoopBlock);
4987 if (DTU)
4988 Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
4989 OtherDest = InfLoopBlock;
4990 }
4991
4992 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent());
4993
4994 // BI may have other predecessors. Because of this, we leave
4995 // it alone, but modify PBI.
4996
4997 // Make sure we get to CommonDest on True&True directions.
4998 Value *PBICond = PBI->getCondition();
4999 IRBuilder<NoFolder> Builder(PBI);
5000 if (PBIOp)
5001 PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not");
5002
5003 Value *BICond = BI->getCondition();
5004 if (BIOp)
5005 BICond = Builder.CreateNot(BICond, BICond->getName() + ".not");
5006
5007 // Merge the conditions.
5008 Value *Cond =
5009 createLogicalOp(Builder, Instruction::Or, PBICond, BICond, "brmerge");
5010
5011 // Modify PBI to branch on the new condition to the new dests.
5012 PBI->setCondition(Cond);
5013 PBI->setSuccessor(0, CommonDest);
5014 PBI->setSuccessor(1, OtherDest);
5015
5016 if (DTU) {
5017 Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest});
5018 Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest});
5019
5020 DTU->applyUpdates(Updates);
5021 }
5022
5023 // Update branch weight for PBI.
5024 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
5025 uint64_t PredCommon, PredOther, SuccCommon, SuccOther;
5026 bool HasWeights =
5027 extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
5028 SuccTrueWeight, SuccFalseWeight);
5029 if (HasWeights) {
5030 PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
5031 PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
5032 SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
5033 SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
5034 // The weight to CommonDest should be PredCommon * SuccTotal +
5035 // PredOther * SuccCommon.
5036 // The weight to OtherDest should be PredOther * SuccOther.
5037 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) +
5038 PredOther * SuccCommon,
5039 PredOther * SuccOther};
5040
5041 setFittedBranchWeights(*PBI, NewWeights, /*IsExpected=*/false,
5042 /*ElideAllZero=*/true);
5043 // Cond may be a select instruction with the first operand set to "true", or
5044 // the second to "false" (see how createLogicalOp works for `and` and `or`)
5045 if (auto *SI = dyn_cast<SelectInst>(Cond)) {
5046 assert(isSelectInRoleOfConjunctionOrDisjunction(SI));
5047 // The select is predicated on PBICond
5048 assert(SI->getCondition() == PBICond);
5049 // The corresponding probabilities are what was referred to above as
5050 // PredCommon and PredOther.
5051 setFittedBranchWeights(*SI, {PredCommon, PredOther},
5052 /*IsExpected=*/false, /*ElideAllZero=*/true);
5053 }
5054 }
5055
5056 // OtherDest may have phi nodes. If so, add an entry from PBI's
5057 // block that are identical to the entries for BI's block.
5058 addPredecessorToBlock(OtherDest, PBI->getParent(), BB);
5059
5060 // We know that the CommonDest already had an edge from PBI to
5061 // it. If it has PHIs though, the PHIs may have different
5062 // entries for BB and PBI's BB. If so, insert a select to make
5063 // them agree.
5064 for (PHINode &PN : CommonDest->phis()) {
5065 Value *BIV = PN.getIncomingValueForBlock(BB);
5066 unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent());
5067 Value *PBIV = PN.getIncomingValue(PBBIdx);
5068 if (BIV != PBIV) {
5069 // Insert a select in PBI to pick the right value.
5071 Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux"));
5072 PN.setIncomingValue(PBBIdx, NV);
5073 // The select has the same condition as PBI, in the same BB. The
5074 // probabilities don't change.
5075 if (HasWeights) {
5076 uint64_t TrueWeight = PBIOp ? PredFalseWeight : PredTrueWeight;
5077 uint64_t FalseWeight = PBIOp ? PredTrueWeight : PredFalseWeight;
5078 setFittedBranchWeights(*NV, {TrueWeight, FalseWeight},
5079 /*IsExpected=*/false, /*ElideAllZero=*/true);
5080 }
5081 }
5082 }
5083
5084 LLVM_DEBUG(dbgs() << "INTO: " << *PBI->getParent());
5085 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent());
5086
5087 // This basic block is probably dead. We know it has at least
5088 // one fewer predecessor.
5089 return true;
5090}
5091
5092// Simplifies a terminator by replacing it with a branch to TrueBB if Cond is
5093// true or to FalseBB if Cond is false.
5094// Takes care of updating the successors and removing the old terminator.
5095// Also makes sure not to introduce new successors by assuming that edges to
5096// non-successor TrueBBs and FalseBBs aren't reachable.
5097bool SimplifyCFGOpt::simplifyTerminatorOnSelect(Instruction *OldTerm,
5098 Value *Cond, BasicBlock *TrueBB,
5099 BasicBlock *FalseBB,
5100 uint32_t TrueWeight,
5101 uint32_t FalseWeight) {
5102 auto *BB = OldTerm->getParent();
5103 // Remove any superfluous successor edges from the CFG.
5104 // First, figure out which successors to preserve.
5105 // If TrueBB and FalseBB are equal, only try to preserve one copy of that
5106 // successor.
5107 BasicBlock *KeepEdge1 = TrueBB;
5108 BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr;
5109
5110 SmallSetVector<BasicBlock *, 2> RemovedSuccessors;
5111
5112 // Then remove the rest.
5113 for (BasicBlock *Succ : successors(OldTerm)) {
5114 // Make sure only to keep exactly one copy of each edge.
5115 if (Succ == KeepEdge1)
5116 KeepEdge1 = nullptr;
5117 else if (Succ == KeepEdge2)
5118 KeepEdge2 = nullptr;
5119 else {
5120 Succ->removePredecessor(BB,
5121 /*KeepOneInputPHIs=*/true);
5122
5123 if (Succ != TrueBB && Succ != FalseBB)
5124 RemovedSuccessors.insert(Succ);
5125 }
5126 }
5127
5128 IRBuilder<> Builder(OldTerm);
5129 Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc());
5130
5131 // Insert an appropriate new terminator.
5132 if (!KeepEdge1 && !KeepEdge2) {
5133 if (TrueBB == FalseBB) {
5134 // We were only looking for one successor, and it was present.
5135 // Create an unconditional branch to it.
5136 Builder.CreateBr(TrueBB);
5137 } else {
5138 // We found both of the successors we were looking for.
5139 // Create a conditional branch sharing the condition of the select.
5140 CondBrInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
5141 setBranchWeights(*NewBI, {TrueWeight, FalseWeight},
5142 /*IsExpected=*/false, /*ElideAllZero=*/true);
5143 }
5144 } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
5145 // Neither of the selected blocks were successors, so this
5146 // terminator must be unreachable.
5147 new UnreachableInst(OldTerm->getContext(), OldTerm->getIterator());
5148 } else {
5149 // One of the selected values was a successor, but the other wasn't.
5150 // Insert an unconditional branch to the one that was found;
5151 // the edge to the one that wasn't must be unreachable.
5152 if (!KeepEdge1) {
5153 // Only TrueBB was found.
5154 Builder.CreateBr(TrueBB);
5155 } else {
5156 // Only FalseBB was found.
5157 Builder.CreateBr(FalseBB);
5158 }
5159 }
5160
5162
5163 if (DTU) {
5164 SmallVector<DominatorTree::UpdateType, 2> Updates;
5165 Updates.reserve(RemovedSuccessors.size());
5166 for (auto *RemovedSuccessor : RemovedSuccessors)
5167 Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor});
5168 DTU->applyUpdates(Updates);
5169 }
5170
5171 return true;
5172}
5173
5174// Folds switch(select(icmp eq X, C, K, X)) into switch(X), retargeting
5175// (or adding) the case for C to wherever K currently dispatches to:
5176// %cmp = icmp eq T %x, C
5177// %key = select i1 %cmp, T K, T %x
5178// switch T %key, label %default [ T K, label %case_k ... ]
5179// becomes
5180// switch T %x, label %default [ T C, label %case_k
5181// T K, label %case_k ... ]
5182bool SimplifyCFGOpt::simplifySwitchOnSelectRemap(SwitchInst *SI,
5183 SelectInst *Select, Value *X,
5184 ConstantInt *C, bool Negate) {
5185 Value *TrueVal = Select->getTrueValue();
5186 Value *FalseVal = Select->getFalseValue();
5187 if (Negate)
5188 std::swap(TrueVal, FalseVal);
5189 if (FalseVal != X)
5190 return false;
5191 auto *K = dyn_cast<ConstantInt>(TrueVal);
5192 if (!K)
5193 return false;
5194
5195 BasicBlock *DestFork = SI->findCaseValue(K)->getCaseSuccessor();
5196 auto CaseC = SI->findCaseValue(C);
5197 bool IsDefault = CaseC == SI->case_default();
5198 // Save before setSuccessor()/addCase() change it.
5199 BasicBlock *OldDest = CaseC->getCaseSuccessor();
5200 BasicBlock *BB = SI->getParent();
5201
5202 // Switch on X first: removePredecessor() below may fold X away if it is a
5203 // PHI in OldDest, and the RAUW must update the switch condition too.
5204 SI->setCondition(X);
5205
5206 if (OldDest != DestFork) {
5207 if (!IsDefault)
5208 OldDest->removePredecessor(BB);
5209 if (IsDefault)
5210 SI->addCase(C, DestFork);
5211 else
5212 CaseC->setSuccessor(DestFork);
5213 // Not a new edge (BB->DestFork exists via K), just adding the PHI
5214 // entry.
5215 addPredecessorToBlock(DestFork, BB, BB);
5216
5217 if (!IsDefault) {
5218 // Edge to OldDest is gone only if nothing else still uses it.
5219 bool OldDestStillTargeted = any_of(
5220 successors(SI), [&](BasicBlock *Succ) { return Succ == OldDest; });
5221 if (DTU && !OldDestStillTargeted)
5222 DTU->applyUpdates({{DominatorTree::Delete, BB, OldDest}});
5223 }
5224
5225 // Update the profile information on the switch if we had a profile
5226 // for both it and the select instruction.
5227 SmallVector<uint32_t> SwitchWeights;
5228 bool SwitchHasBranchWeights = extractBranchWeights(*SI, SwitchWeights);
5229 // If we add a case, ensure the length of the branch weights list matches
5230 // to make iterating over them easier later.
5231 if (IsDefault)
5232 SwitchWeights.push_back(0);
5235 bool SelectHasBranchWeights =
5237 uint64_t SelectTotalWeight = SelectTrueWeight + SelectFalseWeight;
5238 if (Negate)
5240 if (SwitchHasBranchWeights && SelectHasBranchWeights &&
5242 // We update the branch weights by subtracting P(x=C) from the probability
5243 // of case K in the switch (what C redirect to before the transformation),
5244 // plugging the probability for case C into the switch (which we derive
5245 // from the select), and ensuring everything is scaled to have a common
5246 // denominator.
5247 uint64_t SwitchTotalWeight = sum_of(SwitchWeights, uint64_t{0});
5248 SmallVector<uint64_t> NewSwitchWeights;
5249 NewSwitchWeights.reserve(SwitchWeights.size());
5250 NewSwitchWeights.push_back(SwitchWeights[0] * SelectTotalWeight);
5251 for (const auto &[SwitchCase, SwitchWeight] :
5252 zip(SI->cases(), drop_begin(SwitchWeights))) {
5253 if (SwitchCase.getCaseValue() == C) {
5254 NewSwitchWeights.push_back(SwitchTotalWeight * SelectTrueWeight);
5255 } else if (SwitchCase.getCaseValue() == K) {
5256 // In reality, P(key=K) > P(x=C) should always hold, but explicitly
5257 // guard against bad profiles here to prevent underflow by saturating
5258 // to zero.
5259 uint64_t ProbabilityKeyEqualsK = SwitchWeight * SelectTotalWeight;
5260 uint64_t ProbabilityXEqualsC = SelectTrueWeight * SwitchTotalWeight;
5261 uint64_t ProbabilityXEqualsK =
5262 ProbabilityKeyEqualsK > ProbabilityXEqualsC
5263 ? ProbabilityKeyEqualsK - ProbabilityXEqualsC
5264 : 0;
5265 NewSwitchWeights.push_back(ProbabilityXEqualsK);
5266 } else {
5267 NewSwitchWeights.push_back(SwitchWeight * SelectTotalWeight);
5268 }
5269 }
5270 setFittedBranchWeights(*SI, NewSwitchWeights, /*IsExpected=*/false);
5271 } else if (SwitchHasBranchWeights) {
5272 // If we only have branch weights on the switch, we cannot reconstruct
5273 // branch weights correctly, so mark them as unknown if the function has
5274 // a profile count. Reset the branch weights first to ensure we remove
5275 // the now invalid branch weights if the function is not otherwise
5276 // profiled.
5277 SI->setMetadata(LLVMContext::MD_prof, nullptr);
5279 }
5280 }
5281
5282 // The compare/select are now dead.
5284 return true;
5285}
5286
5287// Replaces
5288// (switch (select cond, X, Y)) on constant X, Y
5289// with a branch - conditional if X and Y lead to distinct BBs,
5290// unconditional otherwise.
5291bool SimplifyCFGOpt::simplifySwitchOnSelect(SwitchInst *SI,
5292 SelectInst *Select) {
5293 CmpPredicate Pred;
5294 Value *X;
5295 ConstantInt *C;
5296 if (Select->hasOneUse() &&
5297 match(Select->getCondition(),
5300 simplifySwitchOnSelectRemap(SI, Select, X, C, Pred == ICmpInst::ICMP_NE))
5301 return true;
5302
5303 // Check for constant integer values in the select.
5304 ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue());
5305 ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue());
5306 if (!TrueVal || !FalseVal)
5307 return false;
5308
5309 // Find the relevant condition and destinations.
5310 Value *Condition = Select->getCondition();
5311 BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor();
5312 BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor();
5313
5314 // Get weight for TrueBB and FalseBB.
5315 uint32_t TrueWeight = 0, FalseWeight = 0;
5316 SmallVector<uint64_t, 8> Weights;
5317 bool HasWeights = hasBranchWeightMD(*SI);
5318 if (HasWeights) {
5319 getBranchWeights(SI, Weights);
5320 if (Weights.size() == 1 + SI->getNumCases()) {
5321 TrueWeight =
5322 (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()];
5323 FalseWeight =
5324 (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()];
5325 }
5326 }
5327
5328 // Perform the actual simplification.
5329 return simplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight,
5330 FalseWeight);
5331}
5332
5333// Replaces
5334// (indirectbr (select cond, blockaddress(@fn, BlockA),
5335// blockaddress(@fn, BlockB)))
5336// with
5337// (br cond, BlockA, BlockB).
5338bool SimplifyCFGOpt::simplifyIndirectBrOnSelect(IndirectBrInst *IBI,
5339 SelectInst *SI) {
5340 // Check that both operands of the select are block addresses.
5341 BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue());
5342 BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue());
5343 if (!TBA || !FBA)
5344 return false;
5345
5346 // Extract the actual blocks.
5347 BasicBlock *TrueBB = TBA->getBasicBlock();
5348 BasicBlock *FalseBB = FBA->getBasicBlock();
5349
5350 // The select's profile becomes the profile of the conditional branch that
5351 // replaces the indirect branch.
5352 SmallVector<uint32_t> SelectBranchWeights(2);
5353 extractBranchWeights(*SI, SelectBranchWeights);
5354 // Perform the actual simplification.
5355 return simplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB,
5356 SelectBranchWeights[0],
5357 SelectBranchWeights[1]);
5358}
5359
5360/// This is called when we find an icmp instruction
5361/// (a seteq/setne with a constant) as the only instruction in a
5362/// block that ends with an uncond branch. We are looking for a very specific
5363/// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In
5364/// this case, we merge the first two "or's of icmp" into a switch, but then the
5365/// default value goes to an uncond block with a seteq in it, we get something
5366/// like:
5367///
5368/// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ]
5369/// DEFAULT:
5370/// %tmp = icmp eq i8 %A, 92
5371/// br label %end
5372/// end:
5373/// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ]
5374///
5375/// We prefer to split the edge to 'end' so that there is a true/false entry to
5376/// the PHI, merging the third icmp into the switch.
5377bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt(
5378 ICmpInst *ICI, IRBuilder<> &Builder) {
5379 // Select == nullptr means we assume that there is a hidden no-op select
5380 // instruction of `_ = select %icmp, true, false` after `%icmp = icmp ...`
5381 return tryToSimplifyUncondBranchWithICmpSelectInIt(ICI, nullptr, Builder);
5382}
5383
5384/// Similar to tryToSimplifyUncondBranchWithICmpInIt, but handle a more generic
5385/// case. This is called when we find an icmp instruction (a seteq/setne with a
5386/// constant) and its following select instruction as the only TWO instructions
5387/// in a block that ends with an uncond branch. We are looking for a very
5388/// specific pattern that occurs when "
5389/// if (A == 1) return C1;
5390/// if (A == 2) return C2;
5391/// if (A < 3) return C3;
5392/// return C4;
5393/// " gets simplified. In this case, we merge the first two "branches of icmp"
5394/// into a switch, but then the default value goes to an uncond block with a lt
5395/// icmp and select in it, as InstCombine can not simplify "A < 3" as "A == 2".
5396/// After SimplifyCFG and other subsequent optimizations (e.g., SCCP), we might
5397/// get something like:
5398///
5399/// case1:
5400/// switch i8 %A, label %DEFAULT [ i8 0, label %end i8 1, label %case2 ]
5401/// case2:
5402/// br label %end
5403/// DEFAULT:
5404/// %tmp = icmp eq i8 %A, 2
5405/// %val = select i1 %tmp, i8 C3, i8 C4
5406/// br label %end
5407/// end:
5408/// _ = phi i8 [ C1, %case1 ], [ C2, %case2 ], [ %val, %DEFAULT ]
5409///
5410/// We prefer to split the edge to 'end' so that there are TWO entries of V3/V4
5411/// to the PHI, merging the icmp & select into the switch, as follows:
5412///
5413/// case1:
5414/// switch i8 %A, label %DEFAULT [
5415/// i8 0, label %end
5416/// i8 1, label %case2
5417/// i8 2, label %case3
5418/// ]
5419/// case2:
5420/// br label %end
5421/// case3:
5422/// br label %end
5423/// DEFAULT:
5424/// br label %end
5425/// end:
5426/// _ = phi i8 [ C1, %case1 ], [ C2, %case2 ], [ C3, %case2 ], [ C4, %DEFAULT]
5427bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpSelectInIt(
5428 ICmpInst *ICI, SelectInst *Select, IRBuilder<> &Builder) {
5429 BasicBlock *BB = ICI->getParent();
5430
5431 // If the block has any PHIs in it or the icmp/select has multiple uses, it is
5432 // too complex.
5433 /// TODO: support multi-phis in succ BB of select's BB.
5434 if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse() ||
5435 (Select && !Select->hasOneUse()))
5436 return false;
5437
5438 // The pattern we're looking for is where our only predecessor is a switch on
5439 // 'V' and this block is the default case for the switch. In this case we can
5440 // fold the compared value into the switch to simplify things.
5442 if (!Pred || !isa<SwitchInst>(Pred->getTerminator()))
5443 return false;
5444
5445 Value *IcmpCond;
5446 ConstantInt *NewCaseVal;
5447 CmpPredicate Predicate;
5448
5449 // Match icmp X, C
5450 if (!match(ICI,
5451 m_ICmp(Predicate, m_Value(IcmpCond), m_ConstantInt(NewCaseVal))))
5452 return false;
5453
5454 Value *SelectCond, *SelectTrueVal, *SelectFalseVal;
5456 if (!Select) {
5457 // If Select == nullptr, we can assume that there is a hidden no-op select
5458 // just after icmp
5459 SelectCond = ICI;
5460 SelectTrueVal = Builder.getTrue();
5461 SelectFalseVal = Builder.getFalse();
5462 User = ICI->user_back();
5463 } else {
5464 SelectCond = Select->getCondition();
5465 // Check if the select condition is the same as the icmp condition.
5466 if (SelectCond != ICI)
5467 return false;
5468 SelectTrueVal = Select->getTrueValue();
5469 SelectFalseVal = Select->getFalseValue();
5470 User = Select->user_back();
5471 }
5472
5473 SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator());
5474 if (SI->getCondition() != IcmpCond)
5475 return false;
5476
5477 // If BB is reachable on a non-default case, then we simply know the value of
5478 // V in this block. Substitute it and constant fold the icmp instruction
5479 // away.
5480 if (SI->getDefaultDest() != BB) {
5481 ConstantInt *VVal = SI->findCaseDest(BB);
5482 assert(VVal && "Should have a unique destination value");
5483 ICI->setOperand(0, VVal);
5484
5485 if (Value *V = simplifyInstruction(ICI, {DL, ICI})) {
5486 ICI->replaceAllUsesWith(V);
5487 ICI->eraseFromParent();
5488 }
5489 // BB is now empty, so it is likely to simplify away.
5490 return requestResimplify();
5491 }
5492
5493 // Ok, the block is reachable from the default dest. If the constant we're
5494 // comparing exists in one of the other edges, then we can constant fold ICI
5495 // and zap it.
5496 if (SI->findCaseValue(NewCaseVal) != SI->case_default()) {
5497 Value *V;
5498 if (Predicate == ICmpInst::ICMP_EQ)
5500 else
5502
5503 ICI->replaceAllUsesWith(V);
5504 ICI->eraseFromParent();
5505 // BB is now empty, so it is likely to simplify away.
5506 return requestResimplify();
5507 }
5508
5509 // The use of the select has to be in the 'end' block, by the only PHI node in
5510 // the block.
5511 BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0);
5512 PHINode *PHIUse = dyn_cast<PHINode>(User);
5513 if (PHIUse == nullptr || PHIUse != &SuccBlock->front() ||
5515 return false;
5516
5517 // If the icmp is a SETEQ, then the default dest gets SelectFalseVal, the new
5518 // edge gets SelectTrueVal in the PHI.
5519 Value *DefaultCst = SelectFalseVal;
5520 Value *NewCst = SelectTrueVal;
5521
5522 if (ICI->getPredicate() == ICmpInst::ICMP_NE)
5523 std::swap(DefaultCst, NewCst);
5524
5525 // Replace Select (which is used by the PHI for the default value) with
5526 // SelectFalseVal or SelectTrueVal depending on if ICI is EQ or NE.
5527 if (Select) {
5528 Select->replaceAllUsesWith(DefaultCst);
5529 Select->eraseFromParent();
5530 } else {
5531 ICI->replaceAllUsesWith(DefaultCst);
5532 }
5533 ICI->eraseFromParent();
5534
5535 SmallVector<DominatorTree::UpdateType, 2> Updates;
5536
5537 // Okay, the switch goes to this block on a default value. Add an edge from
5538 // the switch to the merge point on the compared value.
5539 BasicBlock *NewBB =
5540 BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB);
5541 {
5542 SwitchInstProfUpdateWrapper SIW(*SI);
5543 auto W0 = SIW.getSuccessorWeight(0);
5545 if (W0) {
5546 NewW = ((uint64_t(*W0) + 1) >> 1);
5547 SIW.setSuccessorWeight(0, *NewW);
5548 }
5549 SIW.addCase(NewCaseVal, NewBB, NewW);
5550 if (DTU)
5551 Updates.push_back({DominatorTree::Insert, Pred, NewBB});
5552 }
5553
5554 // NewBB branches to the phi block, add the uncond branch and the phi entry.
5555 Builder.SetInsertPoint(NewBB);
5556 Builder.SetCurrentDebugLocation(SI->getDebugLoc());
5557 Builder.CreateBr(SuccBlock);
5558 PHIUse->addIncoming(NewCst, NewBB);
5559 if (DTU) {
5560 Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock});
5561 DTU->applyUpdates(Updates);
5562 }
5563 return true;
5564}
5565
5566/// Check to see if it is branching on an or/and chain of icmp instructions, and
5567/// fold it into a switch instruction if so.
5568bool SimplifyCFGOpt::simplifyBranchOnICmpChain(CondBrInst *BI,
5569 IRBuilder<> &Builder,
5570 const DataLayout &DL) {
5572 if (!Cond)
5573 return false;
5574
5575 // Change br (X == 0 | X == 1), T, F into a switch instruction.
5576 // If this is a bunch of seteq's or'd together, or if it's a bunch of
5577 // 'setne's and'ed together, collect them.
5578
5579 // Try to gather values from a chain of and/or to be turned into a switch
5580 ConstantComparesGatherer ConstantCompare(Cond, DL);
5581 // Unpack the result
5582 SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals;
5583 Value *CompVal = ConstantCompare.CompValue;
5584 unsigned UsedICmps = ConstantCompare.UsedICmps;
5585 Value *ExtraCase = ConstantCompare.Extra;
5586 bool TrueWhenEqual = ConstantCompare.IsEq;
5587
5588 // If we didn't have a multiply compared value, fail.
5589 if (!CompVal)
5590 return false;
5591
5592 // Avoid turning single icmps into a switch.
5593 if (UsedICmps <= 1)
5594 return false;
5595
5596 // There might be duplicate constants in the list, which the switch
5597 // instruction can't handle, remove them now.
5599 Values.erase(llvm::unique(Values), Values.end());
5600
5601 // If Extra was used, we require at least two switch values to do the
5602 // transformation. A switch with one value is just a conditional branch.
5603 if (ExtraCase && Values.size() < 2)
5604 return false;
5605
5606 SmallVector<uint32_t> BranchWeights;
5607 const bool HasProfile = extractBranchWeights(*BI, BranchWeights);
5608
5609 // Figure out which block is which destination.
5610 BasicBlock *DefaultBB = BI->getSuccessor(1);
5611 BasicBlock *EdgeBB = BI->getSuccessor(0);
5612 if (!TrueWhenEqual) {
5613 std::swap(DefaultBB, EdgeBB);
5614 if (HasProfile)
5615 std::swap(BranchWeights[0], BranchWeights[1]);
5616 }
5617
5618 BasicBlock *BB = BI->getParent();
5619
5620 LLVM_DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()
5621 << " cases into SWITCH. BB is:\n"
5622 << *BB);
5623
5624 SmallVector<DominatorTree::UpdateType, 2> Updates;
5625
5626 // If there are any extra values that couldn't be folded into the switch
5627 // then we evaluate them with an explicit branch first. Split the block
5628 // right before the condbr to handle it.
5629 if (ExtraCase) {
5630 BasicBlock *NewBB = SplitBlock(BB, BI, DTU, /*LI=*/nullptr,
5631 /*MSSAU=*/nullptr, "switch.early.test");
5632
5633 // Remove the uncond branch added to the old block.
5634 Instruction *OldTI = BB->getTerminator();
5635 Builder.SetInsertPoint(OldTI);
5636
5637 // There can be an unintended UB if extra values are Poison. Before the
5638 // transformation, extra values may not be evaluated according to the
5639 // condition, and it will not raise UB. But after transformation, we are
5640 // evaluating extra values before checking the condition, and it will raise
5641 // UB. It can be solved by adding freeze instruction to extra values.
5642 AssumptionCache *AC = Options.AC;
5643
5644 if (!isGuaranteedNotToBeUndefOrPoison(ExtraCase, AC, BI, nullptr))
5645 ExtraCase = Builder.CreateFreeze(ExtraCase);
5646
5647 // We don't have any info about this condition.
5648 auto *Br = TrueWhenEqual ? Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB)
5649 : Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB);
5651
5652 OldTI->eraseFromParent();
5653
5654 if (DTU)
5655 Updates.push_back({DominatorTree::Insert, BB, EdgeBB});
5656
5657 // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
5658 // for the edge we just added.
5659 addPredecessorToBlock(EdgeBB, BB, NewBB);
5660
5661 LLVM_DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCase
5662 << "\nEXTRABB = " << *BB);
5663 BB = NewBB;
5664 }
5665
5666 Builder.SetInsertPoint(BI);
5667 // Convert pointer to int before we switch.
5668 if (CompVal->getType()->isPointerTy()) {
5669 assert(!DL.hasUnstableRepresentation(CompVal->getType()) &&
5670 "Should not end up here with unstable pointers");
5671 CompVal = Builder.CreatePtrToInt(
5672 CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr");
5673 }
5674
5675 // Check if we can represent the values as a contiguous range. If so, we use a
5676 // range check + conditional branch instead of a switch.
5677 if (Values.front()->getValue() - Values.back()->getValue() ==
5678 Values.size() - 1) {
5679 ConstantRange RangeToCheck = ConstantRange::getNonEmpty(
5680 Values.back()->getValue(), Values.front()->getValue() + 1);
5681 APInt Offset, RHS;
5682 ICmpInst::Predicate Pred;
5683 RangeToCheck.getEquivalentICmp(Pred, RHS, Offset);
5684 Value *X = CompVal;
5685 if (!Offset.isZero())
5686 X = Builder.CreateAdd(X, ConstantInt::get(CompVal->getType(), Offset));
5687 Value *Cond =
5688 Builder.CreateICmp(Pred, X, ConstantInt::get(CompVal->getType(), RHS));
5689 CondBrInst *NewBI = Builder.CreateCondBr(Cond, EdgeBB, DefaultBB);
5690 if (HasProfile)
5691 setBranchWeights(*NewBI, BranchWeights, /*IsExpected=*/false);
5692 else
5694 if (MDNode *Unpredictable = BI->getMetadata(LLVMContext::MD_unpredictable))
5695 NewBI->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
5696 // We don't need to update PHI nodes since we don't add any new edges.
5697 } else {
5698 // Create the new switch instruction now.
5699 SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size());
5700 if (MDNode *Unpredictable = BI->getMetadata(LLVMContext::MD_unpredictable))
5701 New->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
5702 if (HasProfile) {
5703 // We know the weight of the default case. We don't know the weight of the
5704 // other cases, but rather than completely lose profiling info, we split
5705 // the remaining probability equally over them.
5706 SmallVector<uint32_t> NewWeights(Values.size() + 1);
5707 NewWeights[0] = BranchWeights[1]; // this is the default, and we swapped
5708 // if TrueWhenEqual.
5709 for (auto &V : drop_begin(NewWeights))
5710 V = BranchWeights[0] / Values.size();
5711 setBranchWeights(*New, NewWeights, /*IsExpected=*/false);
5712 } else {
5714 }
5715
5716 // Add all of the 'cases' to the switch instruction.
5717 for (ConstantInt *Val : Values)
5718 New->addCase(Val, EdgeBB);
5719
5720 // We added edges from PI to the EdgeBB. As such, if there were any
5721 // PHI nodes in EdgeBB, they need entries to be added corresponding to
5722 // the number of edges added.
5723 for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) {
5724 PHINode *PN = cast<PHINode>(BBI);
5725 Value *InVal = PN->getIncomingValueForBlock(BB);
5726 for (unsigned i = 0, e = Values.size() - 1; i != e; ++i)
5727 PN->addIncoming(InVal, BB);
5728 }
5729 }
5730
5731 // Erase the old branch instruction.
5733 if (DTU)
5734 DTU->applyUpdates(Updates);
5735
5736 LLVM_DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n');
5737 return true;
5738}
5739
5740bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
5741 if (isa<PHINode>(RI->getValue()))
5742 return simplifyCommonResume(RI);
5743 else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHIIt()) &&
5744 RI->getValue() == &*RI->getParent()->getFirstNonPHIIt())
5745 // The resume must unwind the exception that caused control to branch here.
5746 return simplifySingleResume(RI);
5747
5748 return false;
5749}
5750
5751// Check if cleanup block is empty
5753 for (Instruction &I : R) {
5754 auto *II = dyn_cast<IntrinsicInst>(&I);
5755 if (!II || II->getIntrinsicID() != Intrinsic::lifetime_end)
5756 return false;
5757 }
5758 return true;
5759}
5760
5761// Simplify resume that is shared by several landing pads (phi of landing pad).
5762bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
5763 BasicBlock *BB = RI->getParent();
5764
5765 // Check that there are no other instructions except for debug and lifetime
5766 // intrinsics between the phi's and resume instruction.
5767 if (!isCleanupBlockEmpty(make_range(RI->getParent()->getFirstNonPHIIt(),
5768 BB->getTerminator()->getIterator())))
5769 return false;
5770
5771 SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks;
5772 auto *PhiLPInst = cast<PHINode>(RI->getValue());
5773
5774 // Check incoming blocks to see if any of them are trivial.
5775 for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End;
5776 Idx++) {
5777 auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx);
5778 auto *IncomingValue = PhiLPInst->getIncomingValue(Idx);
5779
5780 // If the block has other successors, we can not delete it because
5781 // it has other dependents.
5782 if (IncomingBB->getUniqueSuccessor() != BB)
5783 continue;
5784
5785 auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHIIt());
5786 // Not the landing pad that caused the control to branch here.
5787 if (IncomingValue != LandingPad)
5788 continue;
5789
5791 make_range(LandingPad->getNextNode(), IncomingBB->getTerminator())))
5792 TrivialUnwindBlocks.insert(IncomingBB);
5793 }
5794
5795 // If no trivial unwind blocks, don't do any simplifications.
5796 if (TrivialUnwindBlocks.empty())
5797 return false;
5798
5799 // Turn all invokes that unwind here into calls.
5800 for (auto *TrivialBB : TrivialUnwindBlocks) {
5801 // Blocks that will be simplified should be removed from the phi node.
5802 // Note there could be multiple edges to the resume block, and we need
5803 // to remove them all.
5804 while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1)
5805 BB->removePredecessor(TrivialBB, true);
5806
5807 for (BasicBlock *Pred :
5809 removeUnwindEdge(Pred, DTU);
5810 ++NumInvokes;
5811 }
5812
5813 // In each SimplifyCFG run, only the current processed block can be erased.
5814 // Otherwise, it will break the iteration of SimplifyCFG pass. So instead
5815 // of erasing TrivialBB, we only remove the branch to the common resume
5816 // block so that we can later erase the resume block since it has no
5817 // predecessors.
5818 TrivialBB->getTerminator()->eraseFromParent();
5819 new UnreachableInst(RI->getContext(), TrivialBB);
5820 if (DTU)
5821 DTU->applyUpdates({{DominatorTree::Delete, TrivialBB, BB}});
5822 }
5823
5824 // Delete the resume block if all its predecessors have been removed.
5825 if (pred_empty(BB))
5826 DeleteDeadBlock(BB, DTU);
5827
5828 return !TrivialUnwindBlocks.empty();
5829}
5830
5831// Simplify resume that is only used by a single (non-phi) landing pad.
5832bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) {
5833 BasicBlock *BB = RI->getParent();
5834 auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHIIt());
5835 assert(RI->getValue() == LPInst &&
5836 "Resume must unwind the exception that caused control to here");
5837
5838 // Check that there are no other instructions except for debug intrinsics.
5840 make_range<Instruction *>(LPInst->getNextNode(), RI)))
5841 return false;
5842
5843 // Turn all invokes that unwind here into calls and delete the basic block.
5844 for (BasicBlock *Pred : llvm::make_early_inc_range(predecessors(BB))) {
5845 removeUnwindEdge(Pred, DTU);
5846 ++NumInvokes;
5847 }
5848
5849 // The landingpad is now unreachable. Zap it.
5850 DeleteDeadBlock(BB, DTU);
5851 return true;
5852}
5853
5855 // If this is a trivial cleanup pad that executes no instructions, it can be
5856 // eliminated. If the cleanup pad continues to the caller, any predecessor
5857 // that is an EH pad will be updated to continue to the caller and any
5858 // predecessor that terminates with an invoke instruction will have its invoke
5859 // instruction converted to a call instruction. If the cleanup pad being
5860 // simplified does not continue to the caller, each predecessor will be
5861 // updated to continue to the unwind destination of the cleanup pad being
5862 // simplified.
5863 BasicBlock *BB = RI->getParent();
5864 CleanupPadInst *CPInst = RI->getCleanupPad();
5865 if (CPInst->getParent() != BB)
5866 // This isn't an empty cleanup.
5867 return false;
5868
5869 // We cannot kill the pad if it has multiple uses. This typically arises
5870 // from unreachable basic blocks.
5871 if (!CPInst->hasOneUse())
5872 return false;
5873
5874 // Check that there are no other instructions except for benign intrinsics.
5876 make_range<Instruction *>(CPInst->getNextNode(), RI)))
5877 return false;
5878
5879 // If the cleanup return we are simplifying unwinds to the caller, this will
5880 // set UnwindDest to nullptr.
5881 BasicBlock *UnwindDest = RI->getUnwindDest();
5882
5883 // We're about to remove BB from the control flow. Before we do, sink any
5884 // PHINodes into the unwind destination. Doing this before changing the
5885 // control flow avoids some potentially slow checks, since we can currently
5886 // be certain that UnwindDest and BB have no common predecessors (since they
5887 // are both EH pads).
5888 if (UnwindDest) {
5889 // First, go through the PHI nodes in UnwindDest and update any nodes that
5890 // reference the block we are removing
5891 for (PHINode &DestPN : UnwindDest->phis()) {
5892 int Idx = DestPN.getBasicBlockIndex(BB);
5893 // Since BB unwinds to UnwindDest, it has to be in the PHI node.
5894 assert(Idx != -1);
5895 // This PHI node has an incoming value that corresponds to a control
5896 // path through the cleanup pad we are removing. If the incoming
5897 // value is in the cleanup pad, it must be a PHINode (because we
5898 // verified above that the block is otherwise empty). Otherwise, the
5899 // value is either a constant or a value that dominates the cleanup
5900 // pad being removed.
5901 //
5902 // Because BB and UnwindDest are both EH pads, all of their
5903 // predecessors must unwind to these blocks, and since no instruction
5904 // can have multiple unwind destinations, there will be no overlap in
5905 // incoming blocks between SrcPN and DestPN.
5906 Value *SrcVal = DestPN.getIncomingValue(Idx);
5907 PHINode *SrcPN = dyn_cast<PHINode>(SrcVal);
5908
5909 bool NeedPHITranslation = SrcPN && SrcPN->getParent() == BB;
5910 for (auto *Pred : predecessors(BB)) {
5911 Value *Incoming =
5912 NeedPHITranslation ? SrcPN->getIncomingValueForBlock(Pred) : SrcVal;
5913 DestPN.addIncoming(Incoming, Pred);
5914 }
5915 }
5916
5917 // Sink any remaining PHI nodes directly into UnwindDest.
5918 BasicBlock::iterator InsertPt = UnwindDest->getFirstNonPHIIt();
5919 for (PHINode &PN : make_early_inc_range(BB->phis())) {
5920 if (PN.use_empty() || !PN.isUsedOutsideOfBlock(BB))
5921 // If the PHI node has no uses or all of its uses are in this basic
5922 // block (meaning they are debug or lifetime intrinsics), just leave
5923 // it. It will be erased when we erase BB below.
5924 continue;
5925
5926 // Otherwise, sink this PHI node into UnwindDest.
5927 // Any predecessors to UnwindDest which are not already represented
5928 // must be back edges which inherit the value from the path through
5929 // BB. In this case, the PHI value must reference itself.
5930 for (auto *pred : predecessors(UnwindDest))
5931 if (pred != BB)
5932 PN.addIncoming(&PN, pred);
5933 PN.moveBefore(InsertPt);
5934 // Also, add a dummy incoming value for the original BB itself,
5935 // so that the PHI is well-formed until we drop said predecessor.
5936 PN.addIncoming(PoisonValue::get(PN.getType()), BB);
5937 }
5938 }
5939
5940 std::vector<DominatorTree::UpdateType> Updates;
5941
5942 // We use make_early_inc_range here because we will remove all predecessors.
5944 if (UnwindDest == nullptr) {
5945 if (DTU) {
5946 DTU->applyUpdates(Updates);
5947 Updates.clear();
5948 }
5949 removeUnwindEdge(PredBB, DTU);
5950 ++NumInvokes;
5951 } else {
5952 BB->removePredecessor(PredBB);
5953 Instruction *TI = PredBB->getTerminator();
5954 TI->replaceUsesOfWith(BB, UnwindDest);
5955 if (DTU) {
5956 Updates.push_back({DominatorTree::Insert, PredBB, UnwindDest});
5957 Updates.push_back({DominatorTree::Delete, PredBB, BB});
5958 }
5959 }
5960 }
5961
5962 if (DTU)
5963 DTU->applyUpdates(Updates);
5964
5965 DeleteDeadBlock(BB, DTU);
5966
5967 return true;
5968}
5969
5970// Try to merge two cleanuppads together.
5972 // Skip any cleanuprets which unwind to caller, there is nothing to merge
5973 // with.
5974 BasicBlock *UnwindDest = RI->getUnwindDest();
5975 if (!UnwindDest)
5976 return false;
5977
5978 // This cleanupret isn't the only predecessor of this cleanuppad, it wouldn't
5979 // be safe to merge without code duplication.
5980 if (UnwindDest->getSinglePredecessor() != RI->getParent())
5981 return false;
5982
5983 // Verify that our cleanuppad's unwind destination is another cleanuppad.
5984 auto *SuccessorCleanupPad = dyn_cast<CleanupPadInst>(&UnwindDest->front());
5985 if (!SuccessorCleanupPad)
5986 return false;
5987
5988 CleanupPadInst *PredecessorCleanupPad = RI->getCleanupPad();
5989 // Replace any uses of the successor cleanupad with the predecessor pad
5990 // The only cleanuppad uses should be this cleanupret, it's cleanupret and
5991 // funclet bundle operands.
5992 SuccessorCleanupPad->replaceAllUsesWith(PredecessorCleanupPad);
5993 // Remove the old cleanuppad.
5994 SuccessorCleanupPad->eraseFromParent();
5995 // Now, we simply replace the cleanupret with a branch to the unwind
5996 // destination.
5997 UncondBrInst::Create(UnwindDest, RI->getParent());
5998 RI->eraseFromParent();
5999
6000 return true;
6001}
6002
6003bool SimplifyCFGOpt::simplifyCleanupReturn(CleanupReturnInst *RI) {
6004 // It is possible to transiantly have an undef cleanuppad operand because we
6005 // have deleted some, but not all, dead blocks.
6006 // Eventually, this block will be deleted.
6007 if (isa<UndefValue>(RI->getOperand(0)))
6008 return false;
6009
6010 if (mergeCleanupPad(RI))
6011 return true;
6012
6013 if (removeEmptyCleanup(RI, DTU))
6014 return true;
6015
6016 return false;
6017}
6018
6019// WARNING: keep in sync with InstCombinerImpl::visitUnreachableInst()!
6020bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
6021 BasicBlock *BB = UI->getParent();
6022
6023 bool Changed = false;
6024
6025 // Ensure that any debug-info records that used to occur after the Unreachable
6026 // are moved to in front of it -- otherwise they'll "dangle" at the end of
6027 // the block.
6029
6030 // Debug-info records on the unreachable inst itself should be deleted, as
6031 // below we delete everything past the final executable instruction.
6032 UI->dropDbgRecords();
6033
6034 // If there are any instructions immediately before the unreachable that can
6035 // be removed, do so.
6036 while (UI->getIterator() != BB->begin()) {
6038 --BBI;
6039
6041 break; // Can not drop any more instructions. We're done here.
6042 // Otherwise, this instruction can be freely erased,
6043 // even if it is not side-effect free.
6044
6045 // Note that deleting EH's here is in fact okay, although it involves a bit
6046 // of subtle reasoning. If this inst is an EH, all the predecessors of this
6047 // block will be the unwind edges of Invoke/CatchSwitch/CleanupReturn,
6048 // and we can therefore guarantee this block will be erased.
6049
6050 // If we're deleting this, we're deleting any subsequent debug info, so
6051 // delete DbgRecords.
6052 BBI->dropDbgRecords();
6053
6054 // Delete this instruction (any uses are guaranteed to be dead)
6055 BBI->replaceAllUsesWith(PoisonValue::get(BBI->getType()));
6056 BBI->eraseFromParent();
6057 Changed = true;
6058 }
6059
6060 // If the unreachable instruction is the first in the block, take a gander
6061 // at all of the predecessors of this instruction, and simplify them.
6062 if (&BB->front() != UI)
6063 return Changed;
6064
6065 std::vector<DominatorTree::UpdateType> Updates;
6066
6067 SmallSetVector<BasicBlock *, 8> Preds(pred_begin(BB), pred_end(BB));
6068 for (BasicBlock *Predecessor : Preds) {
6069 Instruction *TI = Predecessor->getTerminator();
6070 IRBuilder<> Builder(TI);
6071 if (isa<UncondBrInst>(TI)) {
6072 new UnreachableInst(TI->getContext(), TI->getIterator());
6073 TI->eraseFromParent();
6074 Changed = true;
6075 if (DTU)
6076 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
6077 } else if (auto *BI = dyn_cast<CondBrInst>(TI)) {
6078 // We could either have a proper unconditional branch,
6079 // or a degenerate conditional branch with matching destinations.
6080 if (BI->getSuccessor(0) == BI->getSuccessor(1)) {
6081 new UnreachableInst(TI->getContext(), TI->getIterator());
6082 TI->eraseFromParent();
6083 Changed = true;
6084 } else {
6085 Value* Cond = BI->getCondition();
6086 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
6087 "The destinations are guaranteed to be different here.");
6088 CallInst *Assumption;
6089 if (BI->getSuccessor(0) == BB) {
6090 Assumption = Builder.CreateAssumption(Builder.CreateNot(Cond));
6091 Builder.CreateBr(BI->getSuccessor(1));
6092 } else {
6093 assert(BI->getSuccessor(1) == BB && "Incorrect CFG");
6094 Assumption = Builder.CreateAssumption(Cond);
6095 Builder.CreateBr(BI->getSuccessor(0));
6096 }
6097 if (Options.AC)
6098 Options.AC->registerAssumption(cast<AssumeInst>(Assumption));
6099
6101 Changed = true;
6102 }
6103 if (DTU)
6104 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
6105 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
6106 SwitchInstProfUpdateWrapper SU(*SI);
6107 for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) {
6108 if (i->getCaseSuccessor() != BB) {
6109 ++i;
6110 continue;
6111 }
6112 BB->removePredecessor(SU->getParent());
6113 i = SU.removeCase(i);
6114 e = SU->case_end();
6115 Changed = true;
6116 }
6117 // Note that the default destination can't be removed!
6118 if (DTU && SI->getDefaultDest() != BB)
6119 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
6120 } else if (auto *II = dyn_cast<InvokeInst>(TI)) {
6121 if (II->getUnwindDest() == BB) {
6122 if (DTU) {
6123 DTU->applyUpdates(Updates);
6124 Updates.clear();
6125 }
6126 auto *CI = cast<CallInst>(removeUnwindEdge(TI->getParent(), DTU));
6127 if (!CI->doesNotThrow())
6128 CI->setDoesNotThrow();
6129 Changed = true;
6130 }
6131 } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) {
6132 if (CSI->getUnwindDest() == BB) {
6133 if (DTU) {
6134 DTU->applyUpdates(Updates);
6135 Updates.clear();
6136 }
6137 removeUnwindEdge(TI->getParent(), DTU);
6138 Changed = true;
6139 continue;
6140 }
6141
6142 for (CatchSwitchInst::handler_iterator I = CSI->handler_begin(),
6143 E = CSI->handler_end();
6144 I != E; ++I) {
6145 if (*I == BB) {
6146 CSI->removeHandler(I);
6147 --I;
6148 --E;
6149 Changed = true;
6150 }
6151 }
6152 if (DTU)
6153 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
6154 if (CSI->getNumHandlers() == 0) {
6155 if (CSI->hasUnwindDest()) {
6156 // Redirect all predecessors of the block containing CatchSwitchInst
6157 // to instead branch to the CatchSwitchInst's unwind destination.
6158 if (DTU) {
6159 for (auto *PredecessorOfPredecessor : predecessors(Predecessor)) {
6160 Updates.push_back({DominatorTree::Insert,
6161 PredecessorOfPredecessor,
6162 CSI->getUnwindDest()});
6163 Updates.push_back({DominatorTree::Delete,
6164 PredecessorOfPredecessor, Predecessor});
6165 }
6166 }
6167 Predecessor->replaceAllUsesWith(CSI->getUnwindDest());
6168 } else {
6169 // Rewrite all preds to unwind to caller (or from invoke to call).
6170 if (DTU) {
6171 DTU->applyUpdates(Updates);
6172 Updates.clear();
6173 }
6174 SmallVector<BasicBlock *, 8> EHPreds(predecessors(Predecessor));
6175 for (BasicBlock *EHPred : EHPreds)
6176 removeUnwindEdge(EHPred, DTU);
6177 }
6178 // The catchswitch is no longer reachable.
6179 new UnreachableInst(CSI->getContext(), CSI->getIterator());
6180 CSI->eraseFromParent();
6181 Changed = true;
6182 }
6183 } else if (auto *CRI = dyn_cast<CleanupReturnInst>(TI)) {
6184 (void)CRI;
6185 assert(CRI->hasUnwindDest() && CRI->getUnwindDest() == BB &&
6186 "Expected to always have an unwind to BB.");
6187 if (DTU)
6188 Updates.push_back({DominatorTree::Delete, Predecessor, BB});
6189 new UnreachableInst(TI->getContext(), TI->getIterator());
6190 TI->eraseFromParent();
6191 Changed = true;
6192 }
6193 }
6194
6195 if (DTU)
6196 DTU->applyUpdates(Updates);
6197
6198 // If this block is now dead, remove it.
6199 if (pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) {
6200 DeleteDeadBlock(BB, DTU);
6201 return true;
6202 }
6203
6204 return Changed;
6205}
6206
6215
6216static std::optional<ContiguousCasesResult>
6219 BasicBlock *Dest, BasicBlock *OtherDest) {
6220 assert(Cases.size() >= 1);
6221
6223 const APInt &Min = Cases.back()->getValue();
6224 const APInt &Max = Cases.front()->getValue();
6225 APInt Offset = Max - Min;
6226 size_t ContiguousOffset = Cases.size() - 1;
6227 if (Offset == ContiguousOffset) {
6228 return ContiguousCasesResult{
6229 /*Min=*/Cases.back(),
6230 /*Max=*/Cases.front(),
6231 /*Dest=*/Dest,
6232 /*OtherDest=*/OtherDest,
6233 /*Cases=*/&Cases,
6234 /*OtherCases=*/&OtherCases,
6235 };
6236 }
6237 ConstantRange CR = computeConstantRange(Condition, /*ForSigned=*/false,
6238 SimplifyQuery(Dest->getDataLayout()));
6239 // If this is a wrapping contiguous range, that is, [Min, OtherMin] +
6240 // [OtherMax, Max] (also [OtherMax, OtherMin]), [OtherMin+1, OtherMax-1] is a
6241 // contiguous range for the other destination. N.B. If CR is not a full range,
6242 // Max+1 is not equal to Min. It's not continuous in arithmetic.
6243 if (Max == CR.getUnsignedMax() && Min == CR.getUnsignedMin()) {
6244 assert(Cases.size() >= 2);
6245 auto *It =
6246 std::adjacent_find(Cases.begin(), Cases.end(), [](auto L, auto R) {
6247 return L->getValue() != R->getValue() + 1;
6248 });
6249 if (It == Cases.end())
6250 return std::nullopt;
6251 auto [OtherMax, OtherMin] = std::make_pair(*It, *std::next(It));
6252 if ((Max - OtherMax->getValue()) + (OtherMin->getValue() - Min) ==
6253 Cases.size() - 2) {
6254 return ContiguousCasesResult{
6255 /*Min=*/cast<ConstantInt>(
6256 ConstantInt::get(OtherMin->getType(), OtherMin->getValue() + 1)),
6257 /*Max=*/
6259 ConstantInt::get(OtherMax->getType(), OtherMax->getValue() - 1)),
6260 /*Dest=*/OtherDest,
6261 /*OtherDest=*/Dest,
6262 /*Cases=*/&OtherCases,
6263 /*OtherCases=*/&Cases,
6264 };
6265 }
6266 }
6267 return std::nullopt;
6268}
6269
6271 DomTreeUpdater *DTU,
6272 bool RemoveOrigDefaultBlock = true) {
6273 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch default is dead.\n");
6274 auto *BB = Switch->getParent();
6275 auto *OrigDefaultBlock = Switch->getDefaultDest();
6276 if (RemoveOrigDefaultBlock)
6277 OrigDefaultBlock->removePredecessor(BB);
6278 BasicBlock *NewDefaultBlock = BasicBlock::Create(
6279 BB->getContext(), BB->getName() + ".unreachabledefault", BB->getParent(),
6280 OrigDefaultBlock);
6281 auto *UI = new UnreachableInst(Switch->getContext(), NewDefaultBlock);
6283 Switch->setDefaultDest(&*NewDefaultBlock);
6284 if (DTU) {
6286 Updates.push_back({DominatorTree::Insert, BB, &*NewDefaultBlock});
6287 if (RemoveOrigDefaultBlock &&
6288 !is_contained(successors(BB), OrigDefaultBlock))
6289 Updates.push_back({DominatorTree::Delete, BB, &*OrigDefaultBlock});
6290 DTU->applyUpdates(Updates);
6291 }
6292}
6293
6294/// Turn a switch into an integer range comparison and branch.
6295/// Switches with more than 2 destinations are ignored.
6296/// Switches with 1 destination are also ignored.
6297bool SimplifyCFGOpt::turnSwitchRangeIntoICmp(SwitchInst *SI,
6298 IRBuilder<> &Builder) {
6299 assert(SI->getNumCases() > 1 && "Degenerate switch?");
6300
6301 bool HasDefault = !SI->defaultDestUnreachable();
6302
6303 auto *BB = SI->getParent();
6304 // Partition the cases into two sets with different destinations.
6305 BasicBlock *DestA = HasDefault ? SI->getDefaultDest() : nullptr;
6306 BasicBlock *DestB = nullptr;
6309
6310 for (auto Case : SI->cases()) {
6311 BasicBlock *Dest = Case.getCaseSuccessor();
6312 if (!DestA)
6313 DestA = Dest;
6314 if (Dest == DestA) {
6315 CasesA.push_back(Case.getCaseValue());
6316 continue;
6317 }
6318 if (!DestB)
6319 DestB = Dest;
6320 if (Dest == DestB) {
6321 CasesB.push_back(Case.getCaseValue());
6322 continue;
6323 }
6324 return false; // More than two destinations.
6325 }
6326 if (!DestB)
6327 return false; // All destinations are the same and the default is unreachable
6328
6329 assert(DestA && DestB &&
6330 "Single-destination switch should have been folded.");
6331 assert(DestA != DestB);
6332 assert(DestB != SI->getDefaultDest());
6333 assert(!CasesB.empty() && "There must be non-default cases.");
6334 assert(!CasesA.empty() || HasDefault);
6335
6336 // Figure out if one of the sets of cases form a contiguous range.
6337 std::optional<ContiguousCasesResult> ContiguousCases;
6338
6339 // Only one icmp is needed when there is only one case.
6340 if (!HasDefault && CasesA.size() == 1)
6341 ContiguousCases = ContiguousCasesResult{
6342 /*Min=*/CasesA[0],
6343 /*Max=*/CasesA[0],
6344 /*Dest=*/DestA,
6345 /*OtherDest=*/DestB,
6346 /*Cases=*/&CasesA,
6347 /*OtherCases=*/&CasesB,
6348 };
6349 else if (CasesB.size() == 1)
6350 ContiguousCases = ContiguousCasesResult{
6351 /*Min=*/CasesB[0],
6352 /*Max=*/CasesB[0],
6353 /*Dest=*/DestB,
6354 /*OtherDest=*/DestA,
6355 /*Cases=*/&CasesB,
6356 /*OtherCases=*/&CasesA,
6357 };
6358 // Correctness: Cases to the default destination cannot be contiguous cases.
6359 else if (!HasDefault)
6360 ContiguousCases =
6361 findContiguousCases(SI->getCondition(), CasesA, CasesB, DestA, DestB);
6362
6363 if (!ContiguousCases)
6364 ContiguousCases =
6365 findContiguousCases(SI->getCondition(), CasesB, CasesA, DestB, DestA);
6366
6367 if (!ContiguousCases)
6368 return false;
6369
6370 auto [Min, Max, Dest, OtherDest, Cases, OtherCases] = *ContiguousCases;
6371
6372 // Start building the compare and branch.
6373
6375 Constant *NumCases = ConstantInt::get(Offset->getType(),
6376 Max->getValue() - Min->getValue() + 1);
6377 Instruction *NewBI;
6378 if (NumCases->isOneValue()) {
6379 assert(Max->getValue() == Min->getValue());
6380 Value *Cmp = Builder.CreateICmpEQ(SI->getCondition(), Min);
6381 NewBI = Builder.CreateCondBr(Cmp, Dest, OtherDest);
6382 }
6383 // If NumCases overflowed, then all possible values jump to the successor.
6384 else if (NumCases->isNullValue() && !Cases->empty()) {
6385 NewBI = Builder.CreateBr(Dest);
6386 } else {
6387 Value *Sub = SI->getCondition();
6388 if (!Offset->isNullValue())
6389 Sub = Builder.CreateAdd(Sub, Offset, Sub->getName() + ".off");
6390 Value *Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch");
6391 NewBI = Builder.CreateCondBr(Cmp, Dest, OtherDest);
6392 }
6393
6394 // Update weight for the newly-created conditional branch.
6395 if (hasBranchWeightMD(*SI) && isa<CondBrInst>(NewBI)) {
6396 SmallVector<uint64_t, 8> Weights;
6397 getBranchWeights(SI, Weights);
6398 assert(Weights.size() == 1 + SI->getNumCases() &&
6399 "expected number of branch weights to be equal to the number of "
6400 "switch destinations");
6401 uint64_t TrueWeight = 0;
6402 uint64_t FalseWeight = 0;
6403 for (size_t I = 0, E = Weights.size(); I != E; ++I) {
6404 if (SI->getSuccessor(I) == Dest)
6405 TrueWeight += Weights[I];
6406 else
6407 FalseWeight += Weights[I];
6408 }
6409 setFittedBranchWeights(*NewBI, {TrueWeight, FalseWeight},
6410 /*IsExpected=*/false, /*ElideAllZero=*/true);
6411 } else if (isa<CondBrInst>(NewBI)) {
6413 }
6414
6415 // Prune obsolete incoming values off the successors' PHI nodes.
6416 for (auto &PHI : make_early_inc_range(Dest->phis())) {
6417 unsigned PreviousEdges = Cases->size();
6418 if (Dest == SI->getDefaultDest())
6419 ++PreviousEdges;
6420 for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
6421 PHI.removeIncomingValue(SI->getParent());
6422 }
6423 for (auto &PHI : make_early_inc_range(OtherDest->phis())) {
6424 unsigned PreviousEdges = OtherCases->size();
6425 if (OtherDest == SI->getDefaultDest())
6426 ++PreviousEdges;
6427 unsigned E = PreviousEdges - 1;
6428 // Remove all incoming values from OtherDest if OtherDest is unreachable.
6429 if (isa<UncondBrInst>(NewBI))
6430 ++E;
6431 for (unsigned I = 0; I != E; ++I)
6432 PHI.removeIncomingValue(SI->getParent());
6433 }
6434
6435 // Clean up the default block.
6436 SmallVector<DominatorTree::UpdateType, 2> Updates;
6437 if (!HasDefault) {
6438 BasicBlock *OrigDefaultBlock = SI->getDefaultDest();
6439 OrigDefaultBlock->removePredecessor(BB);
6440 Updates.push_back({DominatorTree::Delete, BB, OrigDefaultBlock});
6441 }
6442
6443 // Drop the switch.
6444 SI->eraseFromParent();
6445
6446 if (isa<UncondBrInst>(NewBI))
6447 Updates.push_back({DominatorTree::Delete, BB, OtherDest});
6448
6449 if (DTU)
6450 DTU->applyUpdates(Updates);
6451 return true;
6452}
6453
6454/// Compute masked bits for the condition of a switch
6455/// and use it to remove dead cases.
6457 AssumptionCache *AC,
6458 const DataLayout &DL) {
6459 Value *Cond = SI->getCondition();
6462 bool IsKnownValuesValid = collectPossibleValues(Cond, KnownValues, 4);
6463
6464 // We can also eliminate cases by determining that their values are outside of
6465 // the limited range of the condition based on how many significant (non-sign)
6466 // bits are in the condition value.
6467 unsigned MaxSignificantBitsInCond =
6469
6470 // Gather dead cases.
6472 SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
6473 SmallVector<BasicBlock *, 8> UniqueSuccessors;
6474 for (const auto &Case : SI->cases()) {
6475 auto *Successor = Case.getCaseSuccessor();
6476 if (DTU) {
6477 auto [It, Inserted] = NumPerSuccessorCases.try_emplace(Successor);
6478 if (Inserted)
6479 UniqueSuccessors.push_back(Successor);
6480 ++It->second;
6481 }
6482 ConstantInt *CaseC = Case.getCaseValue();
6483 const APInt &CaseVal = CaseC->getValue();
6484 if (Known.Zero.intersects(CaseVal) || !Known.One.isSubsetOf(CaseVal) ||
6485 (CaseVal.getSignificantBits() > MaxSignificantBitsInCond) ||
6486 (IsKnownValuesValid && !KnownValues.contains(CaseC))) {
6487 DeadCases.push_back(CaseC);
6488 if (DTU)
6489 --NumPerSuccessorCases[Successor];
6490 LLVM_DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseVal
6491 << " is dead.\n");
6492 } else if (IsKnownValuesValid)
6493 KnownValues.erase(CaseC);
6494 }
6495
6496 // If we can prove that the cases must cover all possible values, the
6497 // default destination becomes dead and we can remove it. If we know some
6498 // of the bits in the value, we can use that to more precisely compute the
6499 // number of possible unique case values.
6500 bool HasDefault = !SI->defaultDestUnreachable();
6501 const unsigned NumUnknownBits =
6502 Known.getBitWidth() - (Known.Zero | Known.One).popcount();
6503 assert(NumUnknownBits <= Known.getBitWidth());
6504 if (HasDefault && DeadCases.empty()) {
6505 if (IsKnownValuesValid && all_of(KnownValues, IsaPred<UndefValue>)) {
6507 return true;
6508 }
6509
6510 if (NumUnknownBits < 64 /* avoid overflow */) {
6511 uint64_t AllNumCases = 1ULL << NumUnknownBits;
6512 if (SI->getNumCases() == AllNumCases) {
6514 return true;
6515 }
6516 // When only one case value is missing, replace default with that case.
6517 // Eliminating the default branch will provide more opportunities for
6518 // optimization, such as lookup tables.
6519 if (SI->getNumCases() == AllNumCases - 1) {
6520 assert(NumUnknownBits > 1 && "Should be canonicalized to a branch");
6521 IntegerType *CondTy = cast<IntegerType>(Cond->getType());
6522 if (CondTy->getIntegerBitWidth() > 64 ||
6523 !DL.fitsInLegalInteger(CondTy->getIntegerBitWidth()))
6524 return false;
6525
6526 uint64_t MissingCaseVal = 0;
6527 for (const auto &Case : SI->cases())
6528 MissingCaseVal ^= Case.getCaseValue()->getValue().getLimitedValue();
6529 auto *MissingCase = cast<ConstantInt>(
6530 ConstantInt::get(Cond->getType(), MissingCaseVal));
6532 SIW.addCase(MissingCase, SI->getDefaultDest(),
6533 SIW.getSuccessorWeight(0));
6535 /*RemoveOrigDefaultBlock*/ false);
6536 SIW.setSuccessorWeight(0, 0);
6537 return true;
6538 }
6539 }
6540 }
6541
6542 if (DeadCases.empty())
6543 return false;
6544
6546 for (ConstantInt *DeadCase : DeadCases) {
6547 SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase);
6548 assert(CaseI != SI->case_default() &&
6549 "Case was not found. Probably mistake in DeadCases forming.");
6550 // Prune unused values from PHI nodes.
6551 CaseI->getCaseSuccessor()->removePredecessor(SI->getParent());
6552 SIW.removeCase(CaseI);
6553 }
6554
6555 if (DTU) {
6556 std::vector<DominatorTree::UpdateType> Updates;
6557 for (auto *Successor : UniqueSuccessors)
6558 if (NumPerSuccessorCases[Successor] == 0)
6559 Updates.push_back({DominatorTree::Delete, SI->getParent(), Successor});
6560 DTU->applyUpdates(Updates);
6561 }
6562
6563 return true;
6564}
6565
6566/// If BB would be eligible for simplification by
6567/// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated
6568/// by an unconditional branch), look at the phi node for BB in the successor
6569/// block and see if the incoming value is equal to CaseValue. If so, return
6570/// the phi node, and set PhiIndex to BB's index in the phi node.
6572 BasicBlock *BB, int *PhiIndex) {
6573 if (&*BB->getFirstNonPHIIt() != BB->getTerminator())
6574 return nullptr; // BB must be empty to be a candidate for simplification.
6575 if (!BB->getSinglePredecessor())
6576 return nullptr; // BB must be dominated by the switch.
6577
6579 if (!Branch)
6580 return nullptr; // Terminator must be unconditional branch.
6581
6582 BasicBlock *Succ = Branch->getSuccessor();
6583
6584 for (PHINode &PHI : Succ->phis()) {
6585 int Idx = PHI.getBasicBlockIndex(BB);
6586 assert(Idx >= 0 && "PHI has no entry for predecessor?");
6587
6588 Value *InValue = PHI.getIncomingValue(Idx);
6589 if (InValue != CaseValue)
6590 continue;
6591
6592 *PhiIndex = Idx;
6593 return &PHI;
6594 }
6595
6596 return nullptr;
6597}
6598
6599/// Try to forward the condition of a switch instruction to a phi node
6600/// dominated by the switch, if that would mean that some of the destination
6601/// blocks of the switch can be folded away. Return true if a change is made.
6603 using ForwardingNodesMap = DenseMap<PHINode *, SmallVector<int, 4>>;
6604
6605 ForwardingNodesMap ForwardingNodes;
6606 BasicBlock *SwitchBlock = SI->getParent();
6607 bool Changed = false;
6608 for (const auto &Case : SI->cases()) {
6609 ConstantInt *CaseValue = Case.getCaseValue();
6610 BasicBlock *CaseDest = Case.getCaseSuccessor();
6611
6612 // Replace phi operands in successor blocks that are using the constant case
6613 // value rather than the switch condition variable:
6614 // switchbb:
6615 // switch i32 %x, label %default [
6616 // i32 17, label %succ
6617 // ...
6618 // succ:
6619 // %r = phi i32 ... [ 17, %switchbb ] ...
6620 // -->
6621 // %r = phi i32 ... [ %x, %switchbb ] ...
6622
6623 for (PHINode &Phi : CaseDest->phis()) {
6624 // This only works if there is exactly 1 incoming edge from the switch to
6625 // a phi. If there is >1, that means multiple cases of the switch map to 1
6626 // value in the phi, and that phi value is not the switch condition. Thus,
6627 // this transform would not make sense (the phi would be invalid because
6628 // a phi can't have different incoming values from the same block).
6629 int SwitchBBIdx = Phi.getBasicBlockIndex(SwitchBlock);
6630 if (Phi.getIncomingValue(SwitchBBIdx) == CaseValue &&
6631 count(Phi.blocks(), SwitchBlock) == 1) {
6632 Phi.setIncomingValue(SwitchBBIdx, SI->getCondition());
6633 Changed = true;
6634 }
6635 }
6636
6637 // Collect phi nodes that are indirectly using this switch's case constants.
6638 int PhiIdx;
6639 if (auto *Phi = findPHIForConditionForwarding(CaseValue, CaseDest, &PhiIdx))
6640 ForwardingNodes[Phi].push_back(PhiIdx);
6641 }
6642
6643 for (auto &ForwardingNode : ForwardingNodes) {
6644 PHINode *Phi = ForwardingNode.first;
6645 SmallVectorImpl<int> &Indexes = ForwardingNode.second;
6646 // Check if it helps to fold PHI.
6647 if (Indexes.size() < 2 && !llvm::is_contained(Phi->incoming_values(), SI->getCondition()))
6648 continue;
6649
6650 for (int Index : Indexes)
6651 Phi->setIncomingValue(Index, SI->getCondition());
6652 Changed = true;
6653 }
6654
6655 return Changed;
6656}
6657
6658/// Return true if the backend will be able to handle
6659/// initializing an array of constants like C.
6661 if (C->isThreadDependent())
6662 return false;
6663 if (C->isDLLImportDependent())
6664 return false;
6665
6668 return false;
6669
6670 // Globals cannot contain scalable types.
6671 if (C->getType()->isScalableTy())
6672 return false;
6673
6675 // Pointer casts and in-bounds GEPs will not prohibit the backend from
6676 // materializing the array of constants.
6677 Constant *StrippedC = cast<Constant>(CE->stripInBoundsConstantOffsets());
6678 if (StrippedC == C || !validLookupTableConstant(StrippedC, TTI))
6679 return false;
6680 }
6681
6682 if (!TTI.shouldBuildLookupTablesForConstant(C))
6683 return false;
6684
6685 return true;
6686}
6687
6688/// If V is a Constant, return it. Otherwise, try to look up
6689/// its constant value in ConstantPool, returning 0 if it's not there.
6690static Constant *
6694 return C;
6695 return ConstantPool.lookup(V);
6696}
6697
6698/// Try to fold instruction I into a constant. This works for
6699/// simple instructions such as binary operations where both operands are
6700/// constant or can be replaced by constants from the ConstantPool. Returns the
6701/// resulting constant on success, 0 otherwise.
6702static Constant *
6706 Constant *A = lookupConstant(Select->getCondition(), ConstantPool);
6707 if (!A)
6708 return nullptr;
6709 if (A->isAllOnesValue())
6710 return lookupConstant(Select->getTrueValue(), ConstantPool);
6711 if (A->isNullValue())
6712 return lookupConstant(Select->getFalseValue(), ConstantPool);
6713 return nullptr;
6714 }
6715
6717 for (unsigned N = 0, E = I->getNumOperands(); N != E; ++N) {
6718 if (Constant *A = lookupConstant(I->getOperand(N), ConstantPool))
6719 COps.push_back(A);
6720 else
6721 return nullptr;
6722 }
6723
6724 return ConstantFoldInstOperands(I, COps, DL);
6725}
6726
6727/// Try to determine the resulting constant values in phi nodes
6728/// at the common destination basic block, *CommonDest, for one of the case
6729/// destinations CaseDest corresponding to value CaseVal (nullptr for the
6730/// default case), of a switch instruction SI.
6731static bool
6733 BasicBlock **CommonDest,
6734 SmallVectorImpl<std::pair<PHINode *, Constant *>> &Res,
6735 const DataLayout &DL) {
6736 // The block from which we enter the common destination.
6737 BasicBlock *Pred = SI->getParent();
6738
6739 // If CaseDest is empty except for some side-effect free instructions through
6740 // which we can constant-propagate the CaseVal, continue to its successor.
6742 ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal));
6743 for (Instruction &I : *CaseDest) {
6744 if (I.isTerminator()) {
6745 // If the terminator is a simple branch, continue to the next block.
6746 if (I.getNumSuccessors() != 1 || I.isSpecialTerminator())
6747 return false;
6748 Pred = CaseDest;
6749 CaseDest = I.getSuccessor(0);
6750 } else if (Constant *C = constantFold(&I, DL, ConstantPool)) {
6751 // Instruction is side-effect free and constant.
6752
6753 // If the instruction has uses outside this block or a phi node slot for
6754 // the block, it is not safe to bypass the instruction since it would then
6755 // no longer dominate all its uses.
6756 for (auto &Use : I.uses()) {
6757 User *User = Use.getUser();
6759 if (I->getParent() == CaseDest)
6760 continue;
6761 if (PHINode *Phi = dyn_cast<PHINode>(User))
6762 if (Phi->getIncomingBlock(Use) == CaseDest)
6763 continue;
6764 return false;
6765 }
6766
6767 ConstantPool.insert(std::make_pair(&I, C));
6768 } else {
6769 break;
6770 }
6771 }
6772
6773 // If we did not have a CommonDest before, use the current one.
6774 if (!*CommonDest)
6775 *CommonDest = CaseDest;
6776 // If the destination isn't the common one, abort.
6777 if (CaseDest != *CommonDest)
6778 return false;
6779
6780 // Get the values for this case from phi nodes in the destination block.
6781 for (PHINode &PHI : (*CommonDest)->phis()) {
6782 int Idx = PHI.getBasicBlockIndex(Pred);
6783 if (Idx == -1)
6784 continue;
6785
6786 Constant *ConstVal =
6787 lookupConstant(PHI.getIncomingValue(Idx), ConstantPool);
6788 if (!ConstVal)
6789 return false;
6790
6791 Res.push_back(std::make_pair(&PHI, ConstVal));
6792 }
6793
6794 return Res.size() > 0;
6795}
6796
6797// Helper function used to add CaseVal to the list of cases that generate
6798// Result. Returns the updated number of cases that generate this result.
6799static size_t mapCaseToResult(ConstantInt *CaseVal,
6800 SwitchCaseResultVectorTy &UniqueResults,
6801 Constant *Result) {
6802 for (auto &I : UniqueResults) {
6803 if (I.first == Result) {
6804 I.second.push_back(CaseVal);
6805 return I.second.size();
6806 }
6807 }
6808 UniqueResults.push_back(
6809 std::make_pair(Result, SmallVector<ConstantInt *, 4>(1, CaseVal)));
6810 return 1;
6811}
6812
6813// Helper function that initializes a map containing
6814// results for the PHI node of the common destination block for a switch
6815// instruction. Returns false if multiple PHI nodes have been found or if
6816// there is not a common destination block for the switch.
6818 BasicBlock *&CommonDest,
6819 SwitchCaseResultVectorTy &UniqueResults,
6820 Constant *&DefaultResult,
6821 const DataLayout &DL,
6822 uintptr_t MaxUniqueResults) {
6823 for (const auto &I : SI->cases()) {
6824 ConstantInt *CaseVal = I.getCaseValue();
6825
6826 // Resulting value at phi nodes for this case value.
6827 SwitchCaseResultsTy Results;
6828 if (!getCaseResults(SI, CaseVal, I.getCaseSuccessor(), &CommonDest, Results,
6829 DL))
6830 return false;
6831
6832 // Only one value per case is permitted.
6833 if (Results.size() > 1)
6834 return false;
6835
6836 // Add the case->result mapping to UniqueResults.
6837 const size_t NumCasesForResult =
6838 mapCaseToResult(CaseVal, UniqueResults, Results.begin()->second);
6839
6840 // Early out if there are too many cases for this result.
6841 if (NumCasesForResult > MaxSwitchCasesPerResult)
6842 return false;
6843
6844 // Early out if there are too many unique results.
6845 if (UniqueResults.size() > MaxUniqueResults)
6846 return false;
6847
6848 // Check the PHI consistency.
6849 if (!PHI)
6850 PHI = Results[0].first;
6851 else if (PHI != Results[0].first)
6852 return false;
6853 }
6854 // Find the default result value.
6856 getCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, DefaultResults,
6857 DL);
6858 // If the default value is not found abort unless the default destination
6859 // is unreachable.
6860 DefaultResult =
6861 DefaultResults.size() == 1 ? DefaultResults.begin()->second : nullptr;
6862
6863 return DefaultResult || SI->defaultDestUnreachable();
6864}
6865
6866// Helper function that checks if it is possible to transform a switch with only
6867// two cases (or two cases + default) that produces a result into a select.
6868// TODO: Handle switches with more than 2 cases that map to the same result.
6869// The branch weights correspond to the provided Condition (i.e. if Condition is
6870// modified from the original SwitchInst, the caller must adjust the weights)
6871static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector,
6872 Constant *DefaultResult, Value *Condition,
6873 IRBuilder<> &Builder, const DataLayout &DL,
6874 AssumptionCache *AC, const Instruction *CtxI,
6875 ArrayRef<uint32_t> BranchWeights) {
6876 // If we are selecting between only two cases transform into a simple
6877 // select or a two-way select if default is possible.
6878 // Example:
6879 // switch (a) { %0 = icmp eq i32 %a, 10
6880 // case 10: return 42; %1 = select i1 %0, i32 42, i32 4
6881 // case 20: return 2; ----> %2 = icmp eq i32 %a, 20
6882 // default: return 4; %3 = select i1 %2, i32 2, i32 %1
6883 // }
6884
6885 const bool HasBranchWeights = !BranchWeights.empty();
6886
6887 if (ResultVector.size() == 2 && ResultVector[0].second.size() == 1 &&
6888 ResultVector[1].second.size() == 1) {
6889 ConstantInt *FirstCase = ResultVector[0].second[0];
6890 ConstantInt *SecondCase = ResultVector[1].second[0];
6891 Value *SelectValue = ResultVector[1].first;
6892 if (DefaultResult) {
6893 Value *ValueCompare =
6894 Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp");
6895 SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first,
6896 DefaultResult, "switch.select");
6897 if (auto *SI = dyn_cast<SelectInst>(SelectValue);
6898 SI && HasBranchWeights) {
6899 // We start with 3 probabilities, where the numerator is the
6900 // corresponding BranchWeights[i], and the denominator is the sum over
6901 // BranchWeights. We want the probability and negative probability of
6902 // Condition == SecondCase.
6903 assert(BranchWeights.size() == 3);
6905 *SI, {BranchWeights[2], BranchWeights[0] + BranchWeights[1]},
6906 /*IsExpected=*/false, /*ElideAllZero=*/true);
6907 }
6908 }
6909 Value *ValueCompare =
6910 Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp");
6911 Value *Ret = Builder.CreateSelect(ValueCompare, ResultVector[0].first,
6912 SelectValue, "switch.select");
6913 if (auto *SI = dyn_cast<SelectInst>(Ret); SI && HasBranchWeights) {
6914 // We may have had a DefaultResult. Base the position of the first and
6915 // second's branch weights accordingly. Also the proability that Condition
6916 // != FirstCase needs to take that into account.
6917 assert(BranchWeights.size() >= 2);
6918 size_t FirstCasePos = (Condition != nullptr);
6919 size_t SecondCasePos = FirstCasePos + 1;
6920 uint32_t DefaultCase = (Condition != nullptr) ? BranchWeights[0] : 0;
6922 {BranchWeights[FirstCasePos],
6923 DefaultCase + BranchWeights[SecondCasePos]},
6924 /*IsExpected=*/false, /*ElideAllZero=*/true);
6925 }
6926 return Ret;
6927 }
6928
6929 // Handle the degenerate case where two cases have the same result value.
6930 if (ResultVector.size() == 1 && DefaultResult) {
6931 ArrayRef<ConstantInt *> CaseValues = ResultVector[0].second;
6932 unsigned CaseCount = CaseValues.size();
6933 // n bits group cases map to the same result:
6934 // case 0,4 -> Cond & 0b1..1011 == 0 ? result : default
6935 // case 0,2,4,6 -> Cond & 0b1..1001 == 0 ? result : default
6936 // case 0,2,8,10 -> Cond & 0b1..0101 == 0 ? result : default
6937 if (isPowerOf2_32(CaseCount)) {
6938 ConstantInt *MinCaseVal = CaseValues[0];
6939 // If there are bits that are set exclusively by CaseValues, we
6940 // can transform the switch into a select if the conjunction of
6941 // all the values uniquely identify CaseValues.
6942 APInt AndMask = APInt::getAllOnes(MinCaseVal->getBitWidth());
6943
6944 // Find the minimum value and compute the and of all the case values.
6945 for (auto *Case : CaseValues) {
6946 if (Case->getValue().slt(MinCaseVal->getValue()))
6947 MinCaseVal = Case;
6948 AndMask &= Case->getValue();
6949 }
6950 KnownBits Known = computeKnownBits(Condition, DL, AC, CtxI);
6951
6952 if (!AndMask.isZero() && Known.getMaxValue().uge(AndMask)) {
6953 // Compute the number of bits that are free to vary.
6954 unsigned FreeBits = Known.countMaxActiveBits() - AndMask.popcount();
6955
6956 // Check if the number of values covered by the mask is equal
6957 // to the number of cases.
6958 if (FreeBits == Log2_32(CaseCount)) {
6959 Value *And = Builder.CreateAnd(Condition, AndMask);
6960 Value *Cmp = Builder.CreateICmpEQ(
6961 And, Constant::getIntegerValue(And->getType(), AndMask));
6962 Value *Ret =
6963 Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult);
6964 if (auto *SI = dyn_cast<SelectInst>(Ret); SI && HasBranchWeights) {
6965 // We know there's a Default case. We base the resulting branch
6966 // weights off its probability.
6967 assert(BranchWeights.size() >= 2);
6969 *SI,
6970 {accumulate(drop_begin(BranchWeights), 0U), BranchWeights[0]},
6971 /*IsExpected=*/false, /*ElideAllZero=*/true);
6972 }
6973 return Ret;
6974 }
6975 }
6976
6977 // Mark the bits case number touched.
6978 APInt BitMask = APInt::getZero(MinCaseVal->getBitWidth());
6979 for (auto *Case : CaseValues)
6980 BitMask |= (Case->getValue() - MinCaseVal->getValue());
6981
6982 // Check if cases with the same result can cover all number
6983 // in touched bits.
6984 if (BitMask.popcount() == Log2_32(CaseCount)) {
6985 if (!MinCaseVal->isNullValue())
6986 Condition = Builder.CreateSub(Condition, MinCaseVal);
6987 Value *And = Builder.CreateAnd(Condition, ~BitMask, "switch.and");
6988 Value *Cmp = Builder.CreateICmpEQ(
6989 And, Constant::getNullValue(And->getType()), "switch.selectcmp");
6990 Value *Ret =
6991 Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult);
6992 if (auto *SI = dyn_cast<SelectInst>(Ret); SI && HasBranchWeights) {
6993 assert(BranchWeights.size() >= 2);
6995 *SI,
6996 {accumulate(drop_begin(BranchWeights), 0U), BranchWeights[0]},
6997 /*IsExpected=*/false, /*ElideAllZero=*/true);
6998 }
6999 return Ret;
7000 }
7001 }
7002
7003 // Handle the degenerate case where two cases have the same value.
7004 if (CaseValues.size() == 2) {
7005 Value *Cmp1 = Builder.CreateICmpEQ(Condition, CaseValues[0],
7006 "switch.selectcmp.case1");
7007 Value *Cmp2 = Builder.CreateICmpEQ(Condition, CaseValues[1],
7008 "switch.selectcmp.case2");
7009 Value *Cmp = Builder.CreateOr(Cmp1, Cmp2, "switch.selectcmp");
7010 Value *Ret =
7011 Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult);
7012 if (auto *SI = dyn_cast<SelectInst>(Ret); SI && HasBranchWeights) {
7013 assert(BranchWeights.size() >= 2);
7015 *SI, {accumulate(drop_begin(BranchWeights), 0U), BranchWeights[0]},
7016 /*IsExpected=*/false, /*ElideAllZero=*/true);
7017 }
7018 return Ret;
7019 }
7020 }
7021
7022 return nullptr;
7023}
7024
7025// Helper function to cleanup a switch instruction that has been converted into
7026// a select, fixing up PHI nodes and basic blocks.
7028 Value *SelectValue,
7029 IRBuilder<> &Builder,
7030 DomTreeUpdater *DTU) {
7031 std::vector<DominatorTree::UpdateType> Updates;
7032
7033 BasicBlock *SelectBB = SI->getParent();
7034 BasicBlock *DestBB = PHI->getParent();
7035
7036 if (DTU && !is_contained(predecessors(DestBB), SelectBB))
7037 Updates.push_back({DominatorTree::Insert, SelectBB, DestBB});
7038 Builder.CreateBr(DestBB);
7039
7040 // Remove the switch.
7041
7042 PHI->removeIncomingValueIf(
7043 [&](unsigned Idx) { return PHI->getIncomingBlock(Idx) == SelectBB; });
7044 PHI->addIncoming(SelectValue, SelectBB);
7045
7046 SmallPtrSet<BasicBlock *, 4> RemovedSuccessors;
7047 for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
7048 BasicBlock *Succ = SI->getSuccessor(i);
7049
7050 if (Succ == DestBB)
7051 continue;
7052 Succ->removePredecessor(SelectBB);
7053 if (DTU && RemovedSuccessors.insert(Succ).second)
7054 Updates.push_back({DominatorTree::Delete, SelectBB, Succ});
7055 }
7056 SI->eraseFromParent();
7057 if (DTU)
7058 DTU->applyUpdates(Updates);
7059}
7060
7061/// If a switch is only used to initialize one or more phi nodes in a common
7062/// successor block with only two different constant values, try to replace the
7063/// switch with a select. Returns true if the fold was made.
7066 const DataLayout &DL) {
7067 Value *const Cond = SI->getCondition();
7068 PHINode *PHI = nullptr;
7069 BasicBlock *CommonDest = nullptr;
7070 Constant *DefaultResult;
7071 SwitchCaseResultVectorTy UniqueResults;
7072 // Collect all the cases that will deliver the same value from the switch.
7073 if (!initializeUniqueCases(SI, PHI, CommonDest, UniqueResults, DefaultResult,
7074 DL, /*MaxUniqueResults*/ 2))
7075 return false;
7076
7077 assert(PHI != nullptr && "PHI for value select not found");
7078 Builder.SetInsertPoint(SI);
7079 SmallVector<uint32_t, 4> BranchWeights;
7080 [[maybe_unused]] auto HasWeights =
7082 assert(!HasWeights == (BranchWeights.empty()));
7083 assert(BranchWeights.empty() ||
7084 (BranchWeights.size() >=
7085 UniqueResults.size() + (DefaultResult != nullptr)));
7086
7087 Value *SelectValue = foldSwitchToSelect(UniqueResults, DefaultResult, Cond,
7088 Builder, DL, AC, SI, BranchWeights);
7089 if (!SelectValue)
7090 return false;
7091
7092 removeSwitchAfterSelectFold(SI, PHI, SelectValue, Builder, DTU);
7093 return true;
7094}
7095
7096namespace {
7097
7098/// This class finds alternatives for switches to ultimately
7099/// replace the switch.
7100class SwitchReplacement {
7101public:
7102 /// Create a helper for optimizations to use as a switch replacement.
7103 /// Find a better representation for the content of Values,
7104 /// using DefaultValue to fill any holes in the table.
7105 /// If no representation is possible, isValid() returns false.
7106 SwitchReplacement(
7107 Module &M, uint64_t TableSize, ConstantInt *Offset,
7108 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
7109 Constant *DefaultValue, const DataLayout &DL,
7110 const TargetTransformInfo &TTI, const StringRef &FuncName);
7111
7112 /// Build instructions with Builder to retrieve values using Index
7113 /// and replace the switch.
7114 Value *replaceSwitch(Value *Index, IRBuilder<> &Builder, const DataLayout &DL,
7115 Function *Func);
7116
7117 /// Return true if a table with TableSize elements of
7118 /// type ElementType would fit in a target-legal register.
7119 static bool wouldFitInRegister(const DataLayout &DL, uint64_t TableSize,
7120 Type *ElementType);
7121
7122 /// Return the default value of the switch.
7123 Constant *getDefaultValue();
7124
7125 /// Return true if the replacement is a lookup table.
7126 bool isLookupTable();
7127
7128 /// Return true if the replacement is a bit map.
7129 bool isBitMap();
7130
7131 /// Return true if a suitable switch replacement was found.
7132 bool isValid() const { return Kind != InvalidKind; }
7133
7134private:
7135 // Depending on the switch, there are different alternatives.
7136 enum {
7137 // No suitable replacement was found.
7138 InvalidKind,
7139
7140 // For switches where each case contains the same value, we just have to
7141 // store that single value and return it for each lookup.
7142 SingleValueKind,
7143
7144 // For switches where there is a linear relationship between table index
7145 // and values. We calculate the result with a simple multiplication
7146 // and addition instead of a table lookup.
7147 LinearMapKind,
7148
7149 // For small tables with integer elements, we can pack them into a bitmap
7150 // that fits into a target-legal register. Values are retrieved by
7151 // shift and mask operations.
7152 BitMapKind,
7153
7154 // The table is stored as an array of values. Values are retrieved by load
7155 // instructions from the table.
7156 LookupTableKind
7157 } Kind;
7158
7159 // The default value of the switch.
7160 Constant *DefaultValue;
7161
7162 // The type of the output values.
7163 Type *ValueType;
7164
7165 // For SingleValueKind, this is the single value.
7166 Constant *SingleValue = nullptr;
7167
7168 // For BitMapKind, this is the bitmap.
7169 ConstantInt *BitMap = nullptr;
7170 IntegerType *BitMapElementTy = nullptr;
7171
7172 // For LinearMapKind, these are the constants used to derive the value.
7173 ConstantInt *LinearOffset = nullptr;
7174 ConstantInt *LinearMultiplier = nullptr;
7175 bool LinearMapValWrapped = false;
7176
7177 // For LookupTableKind, this is the table.
7178 Constant *Initializer = nullptr;
7179};
7180
7181} // end anonymous namespace
7182
7183SwitchReplacement::SwitchReplacement(
7184 Module &M, uint64_t TableSize, ConstantInt *Offset,
7185 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
7186 Constant *DefaultValue, const DataLayout &DL,
7187 const TargetTransformInfo &TTI, const StringRef &FuncName)
7188 : DefaultValue(DefaultValue) {
7189 assert(Values.size() && "Can't build lookup table without values!");
7190 assert(TableSize >= Values.size() && "Can't fit values in table!");
7191
7192 // If all values in the table are equal, this is that value.
7193 SingleValue = Values.begin()->second;
7194
7195 ValueType = Values.begin()->second->getType();
7196
7197 // Build up the table contents.
7198 SmallVector<Constant *, 64> TableContents(TableSize);
7199 for (const auto &[CaseVal, CaseRes] : Values) {
7200 assert(CaseRes->getType() == ValueType);
7201
7202 uint64_t Idx = (CaseVal->getValue() - Offset->getValue()).getLimitedValue();
7203 TableContents[Idx] = CaseRes;
7204
7205 if (SingleValue && !isa<PoisonValue>(CaseRes) && CaseRes != SingleValue)
7206 SingleValue = isa<PoisonValue>(SingleValue) ? CaseRes : nullptr;
7207 }
7208
7209 // Fill in any holes in the table with the default result.
7210 if (Values.size() < TableSize) {
7211 assert(DefaultValue &&
7212 "Need a default value to fill the lookup table holes.");
7213 assert(DefaultValue->getType() == ValueType);
7214 for (uint64_t I = 0; I < TableSize; ++I) {
7215 if (!TableContents[I])
7216 TableContents[I] = DefaultValue;
7217 }
7218
7219 // If the default value is poison, all the holes are poison.
7220 bool DefaultValueIsPoison = isa<PoisonValue>(DefaultValue);
7221
7222 if (DefaultValue != SingleValue && !DefaultValueIsPoison)
7223 SingleValue = nullptr;
7224 }
7225
7226 // If each element in the table contains the same value, we only need to store
7227 // that single value.
7228 if (SingleValue) {
7229 Kind = SingleValueKind;
7230 return;
7231 }
7232
7233 // Check if we can derive the value with a linear transformation from the
7234 // table index.
7236 bool LinearMappingPossible = true;
7237 APInt PrevVal;
7238 APInt DistToPrev;
7239 // When linear map is monotonic and signed overflow doesn't happen on
7240 // maximum index, we can attach nsw on Add and Mul.
7241 bool NonMonotonic = false;
7242 assert(TableSize >= 2 && "Should be a SingleValue table.");
7243 // Check if there is the same distance between two consecutive values.
7244 for (uint64_t I = 0; I < TableSize; ++I) {
7245 ConstantInt *ConstVal = dyn_cast<ConstantInt>(TableContents[I]);
7246
7247 if (!ConstVal && isa<PoisonValue>(TableContents[I])) {
7248 // This is an poison, so it's (probably) a lookup table hole.
7249 // To prevent any regressions from before we switched to using poison as
7250 // the default value, holes will fall back to using the first value.
7251 // This can be removed once we add proper handling for poisons in lookup
7252 // tables.
7253 ConstVal = dyn_cast<ConstantInt>(Values[0].second);
7254 }
7255
7256 if (!ConstVal) {
7257 // We only handle literal integers when checking for a linear mapping.
7258 LinearMappingPossible = false;
7259 break;
7260 }
7261 const APInt &Val = ConstVal->getValue();
7262 if (I != 0) {
7263 APInt Dist = Val - PrevVal;
7264 if (I == 1) {
7265 DistToPrev = Dist;
7266 } else if (Dist != DistToPrev) {
7267 LinearMappingPossible = false;
7268 break;
7269 }
7270 NonMonotonic |=
7271 Dist.isStrictlyPositive() ? Val.sle(PrevVal) : Val.sgt(PrevVal);
7272 }
7273 PrevVal = Val;
7274 }
7275 if (LinearMappingPossible) {
7276 LinearOffset = cast<ConstantInt>(TableContents[0]);
7277 LinearMultiplier = ConstantInt::get(M.getContext(), DistToPrev);
7278 APInt M = LinearMultiplier->getValue();
7279 bool MayWrap = true;
7280 if (isIntN(M.getBitWidth(), TableSize - 1))
7281 (void)M.smul_ov(APInt(M.getBitWidth(), TableSize - 1), MayWrap);
7282 LinearMapValWrapped = NonMonotonic || MayWrap;
7283 Kind = LinearMapKind;
7284 return;
7285 }
7286 }
7287
7288 // If the values are integer constants and the table fits in a register,
7289 // build a bitmap.
7290 if (wouldFitInRegister(DL, TableSize, ValueType) &&
7291 all_of(TableContents, IsaPred<ConstantInt, UndefValue>)) {
7293 APInt TableInt(TableSize * IT->getBitWidth(), 0);
7294 for (uint64_t I = TableSize; I > 0; --I) {
7295 TableInt <<= IT->getBitWidth();
7296 // Insert values into the bitmap. Undef values are set to zero.
7297 if (!isa<UndefValue>(TableContents[I - 1])) {
7298 ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]);
7299 TableInt |= Val->getValue().zext(TableInt.getBitWidth());
7300 }
7301 }
7302 BitMap = ConstantInt::get(M.getContext(), TableInt);
7303 BitMapElementTy = IT;
7304 Kind = BitMapKind;
7305 return;
7306 }
7307
7308 // The remaining representation needs a global initializer.
7309 if (!all_of(TableContents,
7310 [&](Constant *C) { return validLookupTableConstant(C, TTI); })) {
7311 Kind = InvalidKind;
7312 return;
7313 }
7314
7315 if (auto *IT = dyn_cast<IntegerType>(ValueType)) {
7316 ConstantRange Range(IT->getBitWidth(), false);
7317 for (Constant *Value : TableContents)
7318 if (!isa<UndefValue>(Value))
7319 Range = Range.unionWith(cast<ConstantInt>(Value)->getValue());
7320 // TODO: handle sign extension as well?
7321 unsigned NeededBitWidth =
7322 std::max(TTI.getMinimumLookupTableEntryBitWidth(),
7323 unsigned(PowerOf2Ceil(Range.getActiveBits())));
7324 if (NeededBitWidth < IT->getBitWidth()) {
7325 IntegerType *DstTy = IntegerType::get(IT->getContext(), NeededBitWidth);
7326 for (Constant *&Value : TableContents)
7327 Value = ConstantFoldCastInstruction(Instruction::Trunc, Value, DstTy);
7328 }
7329 }
7330
7331 // Store the table in an array.
7332 auto *TableTy = ArrayType::get(TableContents[0]->getType(), TableSize);
7333 Initializer = ConstantArray::get(TableTy, TableContents);
7334
7335 Kind = LookupTableKind;
7336}
7337
7338Value *SwitchReplacement::replaceSwitch(Value *Index, IRBuilder<> &Builder,
7339 const DataLayout &DL, Function *Func) {
7340 switch (Kind) {
7341 case InvalidKind:
7342 llvm_unreachable("Cannot use an invalid switch replacement");
7343 case SingleValueKind:
7344 return SingleValue;
7345 case LinearMapKind: {
7346 ++NumLinearMaps;
7347 // Derive the result value from the input value.
7348 Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(),
7349 false, "switch.idx.cast");
7350 if (!LinearMultiplier->isOne())
7351 Result = Builder.CreateMul(Result, LinearMultiplier, "switch.idx.mult",
7352 /*HasNUW = */ false,
7353 /*HasNSW = */ !LinearMapValWrapped);
7354
7355 if (!LinearOffset->isZero())
7356 Result = Builder.CreateAdd(Result, LinearOffset, "switch.offset",
7357 /*HasNUW = */ false,
7358 /*HasNSW = */ !LinearMapValWrapped);
7359 return Result;
7360 }
7361 case BitMapKind: {
7362 ++NumBitMaps;
7363 // Type of the bitmap (e.g. i59).
7364 IntegerType *MapTy = BitMap->getIntegerType();
7365
7366 // Cast Index to the same type as the bitmap.
7367 // Note: The Index is <= the number of elements in the table, so
7368 // truncating it to the width of the bitmask is safe.
7369 Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast");
7370
7371 // Multiply the shift amount by the element width. NUW/NSW can always be
7372 // set, because wouldFitInRegister guarantees Index * ShiftAmt is in
7373 // BitMap's bit width.
7374 ShiftAmt = Builder.CreateMul(
7375 ShiftAmt, ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()),
7376 "switch.shiftamt",/*HasNUW =*/true,/*HasNSW =*/true);
7377
7378 // Shift down.
7379 Value *DownShifted =
7380 Builder.CreateLShr(BitMap, ShiftAmt, "switch.downshift");
7381 // Mask off.
7382 return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked");
7383 }
7384 case LookupTableKind: {
7385 ++NumLookupTables;
7386 auto *Table =
7387 new GlobalVariable(*Func->getParent(), Initializer->getType(),
7388 /*isConstant=*/true, GlobalVariable::PrivateLinkage,
7389 Initializer, "switch.table." + Func->getName());
7390 Table->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
7391 // Set the alignment to that of an array items. We will be only loading one
7392 // value out of it.
7393 Table->setAlignment(DL.getPrefTypeAlign(ValueType));
7394 Type *IndexTy = DL.getIndexType(Table->getType());
7395 auto *ArrayTy = cast<ArrayType>(Table->getValueType());
7396
7397 if (Index->getType() != IndexTy) {
7398 unsigned OldBitWidth = Index->getType()->getIntegerBitWidth();
7399 Index = Builder.CreateZExtOrTrunc(Index, IndexTy);
7400 if (auto *Zext = dyn_cast<ZExtInst>(Index))
7401 Zext->setNonNeg(
7402 isUIntN(OldBitWidth - 1, ArrayTy->getNumElements() - 1));
7403 }
7404
7405 Value *GEPIndices[] = {ConstantInt::get(IndexTy, 0), Index};
7406 Value *GEP =
7407 Builder.CreateInBoundsGEP(ArrayTy, Table, GEPIndices, "switch.gep");
7408 Value *Load =
7409 Builder.CreateLoad(ArrayTy->getElementType(), GEP, "switch.load");
7410 if (Load->getType() == ValueType)
7411 return Load;
7412 return Builder.CreateZExt(Load, ValueType, "switch.ext");
7413 }
7414 }
7415 llvm_unreachable("Unknown helper kind!");
7416}
7417
7418bool SwitchReplacement::wouldFitInRegister(const DataLayout &DL,
7419 uint64_t TableSize,
7420 Type *ElementType) {
7421 auto *IT = dyn_cast<IntegerType>(ElementType);
7422 if (!IT)
7423 return false;
7424 // FIXME: If the type is wider than it needs to be, e.g. i8 but all values
7425 // are <= 15, we could try to narrow the type.
7426
7427 // Avoid overflow, fitsInLegalInteger uses unsigned int for the width.
7428 if (TableSize >= UINT_MAX / IT->getBitWidth())
7429 return false;
7430 return DL.fitsInLegalInteger(TableSize * IT->getBitWidth());
7431}
7432
7434 const DataLayout &DL) {
7435 // Allow any legal type.
7436 if (TTI.isTypeLegal(Ty))
7437 return true;
7438
7439 auto *IT = dyn_cast<IntegerType>(Ty);
7440 if (!IT)
7441 return false;
7442
7443 // Also allow power of 2 integer types that have at least 8 bits and fit in
7444 // a register. These types are common in frontend languages and targets
7445 // usually support loads of these types.
7446 // TODO: We could relax this to any integer that fits in a register and rely
7447 // on ABI alignment and padding in the table to allow the load to be widened.
7448 // Or we could widen the constants and truncate the load.
7449 unsigned BitWidth = IT->getBitWidth();
7450 return BitWidth >= 8 && isPowerOf2_32(BitWidth) &&
7451 DL.fitsInLegalInteger(IT->getBitWidth());
7452}
7453
7454Constant *SwitchReplacement::getDefaultValue() { return DefaultValue; }
7455
7456bool SwitchReplacement::isLookupTable() { return Kind == LookupTableKind; }
7457
7458bool SwitchReplacement::isBitMap() { return Kind == BitMapKind; }
7459
7460static bool isSwitchDense(uint64_t NumCases, uint64_t CaseRange, bool OptSize) {
7461 // 40% is the default density for building a jump table in optsize/minsize
7462 // mode, 10% is the default density for jump tables. See also
7463 // TargetLoweringBase::isSuitableForJumpTable(), which this function was based
7464 // on.
7465 const uint64_t MinDensity = OptSize ? 40 : 10;
7466
7467 if (CaseRange >= UINT64_MAX / 100)
7468 return false; // Avoid multiplication overflows below.
7469
7470 return NumCases * 100 >= CaseRange * MinDensity;
7471}
7472
7473static bool isSwitchDense(ArrayRef<int64_t> Values, bool OptSize) {
7474 uint64_t Diff = (uint64_t)Values.back() - (uint64_t)Values.front();
7475 uint64_t Range = Diff + 1;
7476 if (Range < Diff)
7477 return false; // Overflow.
7478
7479 return isSwitchDense(Values.size(), Range, OptSize);
7480}
7481
7482static std::optional<unsigned>
7484 bool OptSize) {
7485 assert(Values.size() > 1 && "expected multiple switch cases");
7486 if (!llvm::all_of(Values, [Base](int64_t V) { return V >= Base; }))
7487 return std::nullopt;
7488
7489 // First, transform the values by subtracting Base.
7490 SmallVector<int64_t, 4> ReducedValues(Values);
7491 uint64_t ReducedValuesOr = 0;
7492 for (auto &V : ReducedValues) {
7493 uint64_t Reduced = (uint64_t)V - (uint64_t)Base;
7494 ReducedValuesOr |= Reduced;
7495 V = (int64_t)Reduced;
7496 }
7497
7498 // Conceptually, the reduced values are non-negative distances from Base.
7499 // Since the rest of the transform is bitwise only, treat them as unsigned
7500 // bit patterns from here.
7501
7502 // countr_zero(0) returns 64. As Values is guaranteed to have more than
7503 // one element and LLVM disallows duplicate cases, ReducedValuesOr will
7504 // have at least one bit set, so Shift will be less than 64.
7505 unsigned Shift = llvm::countr_zero(ReducedValuesOr);
7506 assert(Shift < 64);
7507 if (Shift > 0)
7508 for (auto &V : ReducedValues)
7509 V = (int64_t)((uint64_t)V >> Shift);
7510
7511 if (!isSwitchDense(ReducedValues, OptSize))
7512 return std::nullopt;
7513
7514 return Shift;
7515}
7516
7517/// Determine whether a lookup table should be built for this switch, based on
7518/// the number of cases, size of the table, and the types of the results.
7519// TODO: We could support larger than legal types by limiting based on the
7520// number of loads required and/or table size. If the constants are small we
7521// could use smaller table entries and extend after the load.
7523 const TargetTransformInfo &TTI,
7524 const DataLayout &DL,
7525 const SmallVector<Type *> &ResultTypes) {
7526 if (SI->getNumCases() > TableSize)
7527 return false; // TableSize overflowed.
7528
7529 bool AllTablesFitInRegister = true;
7530 bool HasIllegalType = false;
7531 for (const auto &Ty : ResultTypes) {
7532 // Saturate this flag to true.
7533 HasIllegalType = HasIllegalType || !isTypeLegalForLookupTable(Ty, TTI, DL);
7534
7535 // Saturate this flag to false.
7536 AllTablesFitInRegister =
7537 AllTablesFitInRegister &&
7538 SwitchReplacement::wouldFitInRegister(DL, TableSize, Ty);
7539
7540 // If both flags saturate, we're done. NOTE: This *only* works with
7541 // saturating flags, and all flags have to saturate first due to the
7542 // non-deterministic behavior of iterating over a dense map.
7543 if (HasIllegalType && !AllTablesFitInRegister)
7544 break;
7545 }
7546
7547 // If each table would fit in a register, we should build it anyway.
7548 if (AllTablesFitInRegister)
7549 return true;
7550
7551 // Don't build a table that doesn't fit in-register if it has illegal types.
7552 if (HasIllegalType)
7553 return false;
7554
7555 return isSwitchDense(SI->getNumCases(), TableSize,
7556 SI->getFunction()->hasOptSize());
7557}
7558
7560 ConstantInt &MinCaseVal, const ConstantInt &MaxCaseVal,
7561 bool HasDefaultResults, const SmallVector<Type *> &ResultTypes,
7562 const DataLayout &DL, const TargetTransformInfo &TTI) {
7563 if (MinCaseVal.isNullValue())
7564 return true;
7565 if (MinCaseVal.isNegative() ||
7566 MaxCaseVal.getLimitedValue() == std::numeric_limits<uint64_t>::max() ||
7567 !HasDefaultResults)
7568 return false;
7569 return all_of(ResultTypes, [&](const auto &ResultType) {
7570 return SwitchReplacement::wouldFitInRegister(
7571 DL, MaxCaseVal.getLimitedValue() + 1 /* TableSize */, ResultType);
7572 });
7573}
7574
7575/// Try to reuse the switch table index compare. Following pattern:
7576/// \code
7577/// if (idx < tablesize)
7578/// r = table[idx]; // table does not contain default_value
7579/// else
7580/// r = default_value;
7581/// if (r != default_value)
7582/// ...
7583/// \endcode
7584/// Is optimized to:
7585/// \code
7586/// cond = idx < tablesize;
7587/// if (cond)
7588/// r = table[idx];
7589/// else
7590/// r = default_value;
7591/// if (cond)
7592/// ...
7593/// \endcode
7594/// Jump threading will then eliminate the second if(cond).
7596 User *PhiUser, BasicBlock *PhiBlock, CondBrInst *RangeCheckBranch,
7597 Constant *DefaultValue,
7598 const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values) {
7600 if (!CmpInst)
7601 return;
7602
7603 // We require that the compare is in the same block as the phi so that jump
7604 // threading can do its work afterwards.
7605 if (CmpInst->getParent() != PhiBlock)
7606 return;
7607
7609 if (!CmpOp1)
7610 return;
7611
7612 Value *RangeCmp = RangeCheckBranch->getCondition();
7613 Constant *TrueConst = ConstantInt::getTrue(RangeCmp->getType());
7614 Constant *FalseConst = ConstantInt::getFalse(RangeCmp->getType());
7615
7616 // Check if the compare with the default value is constant true or false.
7617 const DataLayout &DL = PhiBlock->getDataLayout();
7619 CmpInst->getPredicate(), DefaultValue, CmpOp1, DL);
7620 if (DefaultConst != TrueConst && DefaultConst != FalseConst)
7621 return;
7622
7623 // Check if the compare with the case values is distinct from the default
7624 // compare result.
7625 for (auto ValuePair : Values) {
7627 CmpInst->getPredicate(), ValuePair.second, CmpOp1, DL);
7628 if (!CaseConst || CaseConst == DefaultConst ||
7629 (CaseConst != TrueConst && CaseConst != FalseConst))
7630 return;
7631 }
7632
7633 // Check if the branch instruction dominates the phi node. It's a simple
7634 // dominance check, but sufficient for our needs.
7635 // Although this check is invariant in the calling loops, it's better to do it
7636 // at this late stage. Practically we do it at most once for a switch.
7637 BasicBlock *BranchBlock = RangeCheckBranch->getParent();
7638 for (BasicBlock *Pred : predecessors(PhiBlock)) {
7639 if (Pred != BranchBlock && Pred->getUniquePredecessor() != BranchBlock)
7640 return;
7641 }
7642
7643 if (DefaultConst == FalseConst) {
7644 // The compare yields the same result. We can replace it.
7645 CmpInst->replaceAllUsesWith(RangeCmp);
7646 ++NumTableCmpReuses;
7647 } else {
7648 // The compare yields the same result, just inverted. We can replace it.
7649 Value *InvertedTableCmp = BinaryOperator::CreateXor(
7650 RangeCmp, ConstantInt::get(RangeCmp->getType(), 1), "inverted.cmp",
7651 RangeCheckBranch->getIterator());
7652 CmpInst->replaceAllUsesWith(InvertedTableCmp);
7653 ++NumTableCmpReuses;
7654 }
7655}
7656
7657/// If the switch is only used to initialize one or more phi nodes in a common
7658/// successor block with different constant values, replace the switch with
7659/// lookup tables.
7661 DomTreeUpdater *DTU, const DataLayout &DL,
7662 const TargetTransformInfo &TTI,
7663 bool ConvertSwitchToLookupTable) {
7664 assert(SI->getNumCases() > 1 && "Degenerate switch?");
7665
7666 BasicBlock *BB = SI->getParent();
7667 Function *Fn = BB->getParent();
7668
7669 // FIXME: If the switch is too sparse for a lookup table, perhaps we could
7670 // split off a dense part and build a lookup table for that.
7671
7672 // FIXME: This creates arrays of GEPs to constant strings, which means each
7673 // GEP needs a runtime relocation in PIC code. We should just build one big
7674 // string and lookup indices into that.
7675
7676 // Ignore switches with less than three cases. Lookup tables will not make
7677 // them faster, so we don't analyze them.
7678 if (SI->getNumCases() < 3)
7679 return false;
7680
7681 // Figure out the corresponding result for each case value and phi node in the
7682 // common destination, as well as the min and max case values.
7683 assert(!SI->cases().empty());
7684 SwitchInst::CaseIt CI = SI->case_begin();
7685 ConstantInt *MinCaseVal = CI->getCaseValue();
7686 ConstantInt *MaxCaseVal = CI->getCaseValue();
7687
7688 BasicBlock *CommonDest = nullptr;
7689
7690 using ResultListTy = SmallVector<std::pair<ConstantInt *, Constant *>, 4>;
7692
7694 SmallVector<Type *> ResultTypes;
7696
7697 for (SwitchInst::CaseIt E = SI->case_end(); CI != E; ++CI) {
7698 ConstantInt *CaseVal = CI->getCaseValue();
7699 if (CaseVal->getValue().slt(MinCaseVal->getValue()))
7700 MinCaseVal = CaseVal;
7701 if (CaseVal->getValue().sgt(MaxCaseVal->getValue()))
7702 MaxCaseVal = CaseVal;
7703
7704 // Resulting value at phi nodes for this case value.
7706 ResultsTy Results;
7707 if (!getCaseResults(SI, CaseVal, CI->getCaseSuccessor(), &CommonDest,
7708 Results, DL))
7709 return false;
7710
7711 // Append the result and result types from this case to the list for each
7712 // phi.
7713 for (const auto &I : Results) {
7714 PHINode *PHI = I.first;
7715 Constant *Value = I.second;
7716 auto [It, Inserted] = ResultLists.try_emplace(PHI);
7717 if (Inserted)
7718 PHIs.push_back(PHI);
7719 It->second.push_back(std::make_pair(CaseVal, Value));
7720 ResultTypes.push_back(PHI->getType());
7721 }
7722 }
7723
7724 // If the table has holes, we need a constant result for the default case
7725 // or a bitmask that fits in a register.
7726 SmallVector<std::pair<PHINode *, Constant *>, 4> DefaultResultsList;
7727 bool HasDefaultResults = getCaseResults(SI, nullptr, SI->getDefaultDest(),
7728 &CommonDest, DefaultResultsList, DL);
7729 for (const auto &I : DefaultResultsList) {
7730 PHINode *PHI = I.first;
7731 Constant *Result = I.second;
7732 DefaultResults[PHI] = Result;
7733 }
7734
7735 bool UseSwitchConditionAsTableIndex = shouldUseSwitchConditionAsTableIndex(
7736 *MinCaseVal, *MaxCaseVal, HasDefaultResults, ResultTypes, DL, TTI);
7737 uint64_t TableSize;
7738 ConstantInt *TableIndexOffset;
7739 if (UseSwitchConditionAsTableIndex) {
7740 TableSize = MaxCaseVal->getLimitedValue() + 1;
7741 TableIndexOffset = ConstantInt::get(MaxCaseVal->getIntegerType(), 0);
7742 } else {
7743 TableSize =
7744 (MaxCaseVal->getValue() - MinCaseVal->getValue()).getLimitedValue() + 1;
7745
7746 TableIndexOffset = MinCaseVal;
7747 }
7748
7749 // If the default destination is unreachable, or if the lookup table covers
7750 // all values of the conditional variable, branch directly to the lookup table
7751 // BB. Otherwise, check that the condition is within the case range.
7752 uint64_t NumResults = ResultLists[PHIs[0]].size();
7753 bool DefaultIsReachable = !SI->defaultDestUnreachable();
7754
7755 bool TableHasHoles = (NumResults < TableSize);
7756
7757 // If the table has holes but the default destination doesn't produce any
7758 // constant results, the lookup table entries corresponding to the holes will
7759 // contain poison.
7760 bool AllHolesArePoison = TableHasHoles && !HasDefaultResults;
7761
7762 // If the default destination doesn't produce a constant result but is still
7763 // reachable, and the lookup table has holes, we need to use a mask to
7764 // determine if the current index should load from the lookup table or jump
7765 // to the default case.
7766 // The mask is unnecessary if the table has holes but the default destination
7767 // is unreachable, as in that case the holes must also be unreachable.
7768 bool NeedMask = AllHolesArePoison && DefaultIsReachable;
7769 if (NeedMask) {
7770 // As an extra penalty for the validity test we require more cases.
7771 if (SI->getNumCases() < 4) // FIXME: Find best threshold value (benchmark).
7772 return false;
7773 if (!DL.fitsInLegalInteger(TableSize))
7774 return false;
7775 }
7776
7777 if (!shouldBuildLookupTable(SI, TableSize, TTI, DL, ResultTypes))
7778 return false;
7779
7780 // Compute the table index value.
7781 Value *TableIndex;
7782 if (UseSwitchConditionAsTableIndex) {
7783 TableIndex = SI->getCondition();
7784 if (HasDefaultResults) {
7785 // Grow the table to cover all possible index values to avoid the range
7786 // check. It will use the default result to fill in the table hole later,
7787 // so make sure it exist.
7788 ConstantRange CR = computeConstantRange(TableIndex, /*ForSigned=*/false,
7789 SimplifyQuery(DL));
7790 // Grow the table shouldn't have any size impact by checking
7791 // wouldFitInRegister.
7792 // TODO: Consider growing the table also when it doesn't fit in a register
7793 // if no optsize is specified.
7794 const uint64_t UpperBound = CR.getUpper().getLimitedValue();
7795 if (!CR.isUpperWrapped() &&
7796 all_of(ResultTypes, [&](const auto &ResultType) {
7797 return SwitchReplacement::wouldFitInRegister(DL, UpperBound,
7798 ResultType);
7799 })) {
7800 // There may be some case index larger than the UpperBound (unreachable
7801 // case), so make sure the table size does not get smaller.
7802 TableSize = std::max(UpperBound, TableSize);
7803 // The default branch is unreachable after we enlarge the lookup table.
7804 // Adjust DefaultIsReachable to reuse code path.
7805 DefaultIsReachable = false;
7806 }
7807 }
7808 }
7809
7810 // Keep track of the switch replacement for each phi
7812 for (PHINode *PHI : PHIs) {
7813 const auto &ResultList = ResultLists[PHI];
7814
7815 Type *ResultType = ResultList.begin()->second->getType();
7816 // Use any value to fill the lookup table holes.
7817 Constant *DefaultVal =
7818 AllHolesArePoison ? PoisonValue::get(ResultType) : DefaultResults[PHI];
7819 StringRef FuncName = Fn->getName();
7820 SwitchReplacement Replacement(*Fn->getParent(), TableSize, TableIndexOffset,
7821 ResultList, DefaultVal, DL, TTI, FuncName);
7822 if (!Replacement.isValid())
7823 return false;
7824 PhiToReplacementMap.insert({PHI, Replacement});
7825 }
7826
7827 bool AnyLookupTables = any_of(
7828 PhiToReplacementMap, [](auto &KV) { return KV.second.isLookupTable(); });
7829 bool AnyBitMaps = any_of(PhiToReplacementMap,
7830 [](auto &KV) { return KV.second.isBitMap(); });
7831
7832 // A few conditions prevent the generation of lookup tables:
7833 // 1. The target does not support lookup tables.
7834 // 2. The "no-jump-tables" function attribute is set.
7835 // However, these objections do not apply to other switch replacements, like
7836 // the bitmap, so we only stop here if any of these conditions are met and we
7837 // want to create a LUT. Otherwise, continue with the switch replacement.
7838 if (AnyLookupTables &&
7839 (!TTI.shouldBuildLookupTables() ||
7840 Fn->getFnAttribute("no-jump-tables").getValueAsBool()))
7841 return false;
7842
7843 // In the early optimization pipeline, disable formation of lookup tables,
7844 // bit maps and mask checks, as they may inhibit further optimization.
7845 if (!ConvertSwitchToLookupTable &&
7846 (AnyLookupTables || AnyBitMaps || NeedMask))
7847 return false;
7848
7849 Builder.SetInsertPoint(SI);
7850 // TableIndex is the switch condition - TableIndexOffset if we don't
7851 // use the condition directly
7852 if (!UseSwitchConditionAsTableIndex) {
7853 // If the default is unreachable, all case values are s>= MinCaseVal. Then
7854 // we can try to attach nsw.
7855 bool MayWrap = true;
7856 if (!DefaultIsReachable) {
7857 APInt Res =
7858 MaxCaseVal->getValue().ssub_ov(MinCaseVal->getValue(), MayWrap);
7859 (void)Res;
7860 }
7861 TableIndex = Builder.CreateSub(SI->getCondition(), TableIndexOffset,
7862 "switch.tableidx", /*HasNUW =*/false,
7863 /*HasNSW =*/!MayWrap);
7864 }
7865
7866 std::vector<DominatorTree::UpdateType> Updates;
7867
7868 // Compute the maximum table size representable by the integer type we are
7869 // switching upon.
7870 unsigned CaseSize = MinCaseVal->getType()->getPrimitiveSizeInBits();
7871 uint64_t MaxTableSize = CaseSize > 63 ? UINT64_MAX : 1ULL << CaseSize;
7872 assert(MaxTableSize >= TableSize &&
7873 "It is impossible for a switch to have more entries than the max "
7874 "representable value of its input integer type's size.");
7875
7876 // Create the BB that does the lookups.
7877 Module &Mod = *CommonDest->getParent()->getParent();
7878 BasicBlock *LookupBB = BasicBlock::Create(
7879 Mod.getContext(), "switch.lookup", CommonDest->getParent(), CommonDest);
7880
7881 CondBrInst *RangeCheckBranch = nullptr;
7882 CondBrInst *CondBranch = nullptr;
7883
7884 Builder.SetInsertPoint(SI);
7885 const bool GeneratingCoveredLookupTable = (MaxTableSize == TableSize);
7886 const bool IsFullyCoveredLookupTable =
7887 !DefaultIsReachable || GeneratingCoveredLookupTable;
7888 if (IsFullyCoveredLookupTable) {
7889 Builder.CreateBr(LookupBB);
7890 if (DTU)
7891 Updates.push_back({DominatorTree::Insert, BB, LookupBB});
7892 // Note: We call removeProdecessor later since we need to be able to get the
7893 // PHI value for the default case in case we're using a bit mask.
7894 } else {
7895 Value *Cmp = Builder.CreateICmpULT(
7896 TableIndex, ConstantInt::get(MinCaseVal->getType(), TableSize));
7897 RangeCheckBranch =
7898 Builder.CreateCondBr(Cmp, LookupBB, SI->getDefaultDest());
7899 CondBranch = RangeCheckBranch;
7900 if (DTU)
7901 Updates.push_back({DominatorTree::Insert, BB, LookupBB});
7902 }
7903
7904 // Populate the BB that does the lookups.
7905 Builder.SetInsertPoint(LookupBB);
7906
7907 if (NeedMask) {
7908 // Before doing the lookup, we do the hole check. The LookupBB is therefore
7909 // re-purposed to do the hole check, and we create a new LookupBB.
7910 BasicBlock *MaskBB = LookupBB;
7911 MaskBB->setName("switch.hole_check");
7912 LookupBB = BasicBlock::Create(Mod.getContext(), "switch.lookup",
7913 CommonDest->getParent(), CommonDest);
7914
7915 // Make the mask's bitwidth at least 8-bit and a power-of-2 to avoid
7916 // unnecessary illegal types.
7917 uint64_t TableSizePowOf2 = NextPowerOf2(std::max(7ULL, TableSize - 1ULL));
7918 APInt MaskInt(TableSizePowOf2, 0);
7919 APInt One(TableSizePowOf2, 1);
7920 // Build bitmask; fill in a 1 bit for every case.
7921 const ResultListTy &ResultList = ResultLists[PHIs[0]];
7922 for (const auto &Result : ResultList) {
7923 uint64_t Idx = (Result.first->getValue() - TableIndexOffset->getValue())
7924 .getLimitedValue();
7925 MaskInt |= One << Idx;
7926 }
7927 ConstantInt *TableMask = ConstantInt::get(Mod.getContext(), MaskInt);
7928
7929 // Get the TableIndex'th bit of the bitmask.
7930 // If this bit is 0 (meaning hole) jump to the default destination,
7931 // else continue with table lookup.
7932 IntegerType *MapTy = TableMask->getIntegerType();
7933 Value *MaskIndex =
7934 Builder.CreateZExtOrTrunc(TableIndex, MapTy, "switch.maskindex");
7935 Value *Shifted = Builder.CreateLShr(TableMask, MaskIndex, "switch.shifted");
7936 Value *LoBit = Builder.CreateTrunc(
7937 Shifted, Type::getInt1Ty(Mod.getContext()), "switch.lobit");
7938 if (!IsFullyCoveredLookupTable)
7940 CondBranch = Builder.CreateCondBr(LoBit, LookupBB, SI->getDefaultDest());
7941 if (DTU) {
7942 Updates.push_back({DominatorTree::Insert, MaskBB, LookupBB});
7943 Updates.push_back({DominatorTree::Insert, MaskBB, SI->getDefaultDest()});
7944 }
7945 Builder.SetInsertPoint(LookupBB);
7946 addPredecessorToBlock(SI->getDefaultDest(), MaskBB, BB);
7947 }
7948
7949 if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
7950 // We cached PHINodes in PHIs. To avoid accessing deleted PHINodes later,
7951 // do not delete PHINodes here.
7952 SI->getDefaultDest()->removePredecessor(BB,
7953 /*KeepOneInputPHIs=*/true);
7954 if (DTU)
7955 Updates.push_back({DominatorTree::Delete, BB, SI->getDefaultDest()});
7956 }
7957
7958 for (PHINode *PHI : PHIs) {
7959 const ResultListTy &ResultList = ResultLists[PHI];
7960 auto Replacement = PhiToReplacementMap.at(PHI);
7961 auto *Result = Replacement.replaceSwitch(TableIndex, Builder, DL, Fn);
7962 // Do a small peephole optimization: re-use the switch table compare if
7963 // possible.
7964 if (!TableHasHoles && HasDefaultResults && RangeCheckBranch) {
7965 BasicBlock *PhiBlock = PHI->getParent();
7966 // Search for compare instructions which use the phi.
7967 for (auto *User : PHI->users()) {
7968 reuseTableCompare(User, PhiBlock, RangeCheckBranch,
7969 Replacement.getDefaultValue(), ResultList);
7970 }
7971 }
7972
7973 PHI->addIncoming(Result, LookupBB);
7974 }
7975
7976 Builder.CreateBr(CommonDest);
7977 if (DTU)
7978 Updates.push_back({DominatorTree::Insert, LookupBB, CommonDest});
7979
7980 SmallVector<uint32_t> BranchWeights;
7981 const bool HasBranchWeights =
7982 CondBranch && extractBranchWeights(*SI, BranchWeights);
7983 uint64_t ToLookupWeight = 0;
7984 uint64_t ToDefaultWeight = 0;
7985
7986 // Remove the switch.
7987 SmallPtrSet<BasicBlock *, 8> RemovedSuccessors;
7988 for (unsigned I = 0, E = SI->getNumSuccessors(); I < E; ++I) {
7989 BasicBlock *Succ = SI->getSuccessor(I);
7990
7991 if (Succ == SI->getDefaultDest()) {
7992 if (HasBranchWeights)
7993 ToDefaultWeight += BranchWeights[I];
7994 continue;
7995 }
7996 Succ->removePredecessor(BB);
7997 if (DTU && RemovedSuccessors.insert(Succ).second)
7998 Updates.push_back({DominatorTree::Delete, BB, Succ});
7999 if (HasBranchWeights)
8000 ToLookupWeight += BranchWeights[I];
8001 }
8002 SI->eraseFromParent();
8003 // In the case where we do not have a fully covered lookup table (i.e., we
8004 // need a range check), and we need a mask check, we split the default case
8005 // into two different cases for which we cannot infer branch weights.
8006 // Otherwise we have a single branch for which we can infer branch weights.
8007 if ((NeedMask && !IsFullyCoveredLookupTable) &&
8010 else if (HasBranchWeights)
8011 setFittedBranchWeights(*CondBranch, {ToLookupWeight, ToDefaultWeight},
8012 /*IsExpected=*/false);
8013 if (DTU)
8014 DTU->applyUpdates(Updates);
8015
8016 if (NeedMask)
8017 ++NumLookupTablesHoles;
8018 return true;
8019}
8020
8021/// Try to transform a switch that has "holes" in it to a contiguous sequence
8022/// of cases.
8023///
8024/// A switch such as: switch(i) {case 5: case 9: case 13: case 17:} can be
8025/// range-reduced to: switch ((i-5) / 4) {case 0: case 1: case 2: case 3:}.
8026///
8027/// This converts a sparse switch into a dense switch which allows better
8028/// lowering and could also allow transforming into a lookup table.
8030 const DataLayout &DL,
8031 const TargetTransformInfo &TTI) {
8032 auto *CondTy = cast<IntegerType>(SI->getCondition()->getType());
8033 if (CondTy->getIntegerBitWidth() > 64 ||
8034 !DL.fitsInLegalInteger(CondTy->getIntegerBitWidth()))
8035 return false;
8036 // Only bother with this optimization if there are more than 3 switch cases;
8037 // SDAG will only bother creating jump tables for 4 or more cases.
8038 if (SI->getNumCases() < 4)
8039 return false;
8040
8041 // This transform is agnostic to the signedness of the input or case values. We
8042 // can treat the case values as signed or unsigned. We can optimize more common
8043 // cases such as a sequence crossing zero {-4,0,4,8} if we interpret case values
8044 // as signed.
8046 for (const auto &C : SI->cases())
8047 Values.push_back(C.getCaseValue()->getValue().getSExtValue());
8049
8050 // If the switch is already dense, there's nothing useful to do here.
8051 bool OptSize = SI->getFunction()->hasOptSize();
8052 if (isSwitchDense(Values, OptSize))
8053 return false;
8054
8055 // Find a Base and corresponding Shift that results in a dense switch range.
8056 // Values[0] is the local minimum.
8057 int64_t Base = Values[0];
8058 std::optional<unsigned> Shift;
8059 // Prefer Base=0 when shifting out common low zero bits still produces a dense
8060 // range, as this avoids an unnecessary `(condition - local_min)` expression.
8061 // However, avoiding the subtract can leave a wider reduced range than using
8062 // the local minimum, so require Base=0 to satisfy the stricter optsize
8063 // density threshold before falling back to the normal density policy for
8064 // local-min.
8065 if ((Shift = getDenseSwitchRangeReductionShift(Values, /*Base=*/0,
8066 /*OptSize=*/true)))
8067 Base = 0;
8068 else if (Base != 0)
8070
8071 if (!Shift)
8072 return false;
8073
8074 // The obvious transform is to shift the switch condition right and emit a
8075 // check that the condition actually cleanly divided by GCD, i.e.
8076 // C & (1 << Shift - 1) == 0
8077 // inserting a new CFG edge to handle the case where it didn't divide cleanly.
8078 //
8079 // A cheaper way of doing this is a simple ROTR(C, Shift). This performs the
8080 // shift and puts the shifted-off bits in the uppermost bits. If any of these
8081 // are nonzero then the switch condition will be very large and will hit the
8082 // default case.
8083 //
8084 // This transform can be done speculatively because it is so cheap - it
8085 // results in a single rotate operation being inserted.
8086
8087 auto *Ty = cast<IntegerType>(SI->getCondition()->getType());
8088 Builder.SetInsertPoint(SI);
8089 Value *Sub = SI->getCondition();
8090 if (Base != 0)
8091 Sub = Builder.CreateSub(Sub, ConstantInt::getSigned(Ty, Base));
8092 Value *Rot = Builder.CreateIntrinsic(
8093 Ty, Intrinsic::fshl,
8094 {Sub, Sub, ConstantInt::get(Ty, Ty->getBitWidth() - *Shift)});
8095 SI->replaceUsesOfWith(SI->getCondition(), Rot);
8096
8097 for (auto Case : SI->cases()) {
8098 auto *Orig = Case.getCaseValue();
8099 auto Sub = Orig->getValue() - APInt(Ty->getBitWidth(), Base, true);
8100 Case.setValue(cast<ConstantInt>(ConstantInt::get(Ty, Sub.lshr(*Shift))));
8101 }
8102 return true;
8103}
8104
8105/// Tries to transform the switch when the condition is umin with a constant.
8106/// In that case, the default branch can be replaced by the constant's branch.
8107/// This method also removes dead cases when the simplification cannot replace
8108/// the default branch.
8109///
8110/// For example:
8111/// switch(umin(a, 3)) {
8112/// case 0:
8113/// case 1:
8114/// case 2:
8115/// case 3:
8116/// case 4:
8117/// // ...
8118/// default:
8119/// unreachable
8120/// }
8121///
8122/// Transforms into:
8123///
8124/// switch(a) {
8125/// case 0:
8126/// case 1:
8127/// case 2:
8128/// default:
8129/// // This is case 3
8130/// }
8132 Value *A;
8134
8135 if (!match(SI->getCondition(), m_UMin(m_Value(A), m_ConstantInt(Constant))))
8136 return false;
8137
8140 BasicBlock *BB = SIW->getParent();
8141
8142 // Dead cases are removed even when the simplification fails.
8143 // A case is dead when its value is higher than the Constant.
8144 for (auto I = SI->case_begin(), E = SI->case_end(); I != E;) {
8145 if (!I->getCaseValue()->getValue().ugt(Constant->getValue())) {
8146 ++I;
8147 continue;
8148 }
8149 BasicBlock *DeadCaseBB = I->getCaseSuccessor();
8150 DeadCaseBB->removePredecessor(BB);
8151 I = SIW.removeCase(I);
8152 E = SIW->case_end();
8153 if (!is_contained(successors(BB), DeadCaseBB))
8154 Updates.push_back({DominatorTree::Delete, BB, DeadCaseBB});
8155 }
8156
8157 auto Case = SI->findCaseValue(Constant);
8158 // If the case value is not found, `findCaseValue` returns the default case.
8159 // In this scenario, since there is no explicit `case 3:`, the simplification
8160 // fails. The simplification also fails when the switch’s default destination
8161 // is reachable.
8162 if (!SI->defaultDestUnreachable() || Case == SI->case_default()) {
8163 if (DTU)
8164 DTU->applyUpdates(Updates);
8165 return !Updates.empty();
8166 }
8167
8168 BasicBlock *Unreachable = SI->getDefaultDest();
8169 SIW.replaceDefaultDest(Case);
8170 SIW.removeCase(Case);
8171 SIW->setCondition(A);
8172
8173 Updates.push_back({DominatorTree::Delete, BB, Unreachable});
8174
8175 if (DTU)
8176 DTU->applyUpdates(Updates);
8177
8178 return true;
8179}
8180
8182 AssumptionCache *AC,
8183 const DataLayout &DL) {
8184 assert(SI);
8185 if (SI->defaultDestUnreachable())
8186 return false;
8187
8188 // If it can be proved that the switch condition takes some concrete value
8189 // in the default block, we can make some nice simplifications to the
8190 // switch.
8191 BasicBlock *Default = SI->getDefaultDest();
8192 const Instruction *CtxI = &*Default->getFirstNonPHIIt();
8194 SI->getCondition(),
8195 SimplifyQuery(DL, /*DT=*/nullptr, AC, CtxI).allowEphemerals(true));
8196 if (!Known.isConstant())
8197 return false;
8198
8199 // At this point, we know that only one value can be mapped to the
8200 // default block. So, if a case doesn't exist for it already, we
8201 // can create one pointing to the default block.
8202 ConstantInt *CaseVal =
8203 ConstantInt::get(SI->getContext(), Known.getConstant());
8204 const llvm::SwitchInst::CaseIt CaseIt = SI->findCaseValue(CaseVal);
8205 if (CaseIt == SI->case_default()) {
8207 SIW.addCase(CaseVal, Default, SIW.getSuccessorWeight(0));
8208 SIW.setSuccessorWeight(0, 0);
8209 }
8210 // If there is a pre-existing case for the constant, the default branch
8211 // will be removed rather than being moved. Thus, we are removing an edge
8212 // in the CFG, and need to update any PHIs in the default block.
8213 createUnreachableSwitchDefault(SI, DTU, /*RemoveOrigDefaultBlock=*/CaseIt !=
8214 SI->case_default());
8215
8216 assert(SI->getNumCases() > 0 && "Switch should have at least one case");
8217 assert(SI->findCaseValue(CaseVal) != SI->case_default() &&
8218 "Proven value should have a dedicated case");
8219 assert(SI->defaultDestUnreachable());
8220 return true;
8221}
8222
8223/// Tries to transform switch of powers of two to reduce switch range.
8224/// For example, switch like:
8225/// switch (C) { case 1: case 2: case 64: case 128: }
8226/// will be transformed to:
8227/// switch (count_trailing_zeros(C)) { case 0: case 1: case 6: case 7: }
8228///
8229/// This transformation allows better lowering and may transform the switch
8230/// instruction into a sequence of bit manipulation and a smaller
8231/// log2(C)-indexed value table (instead of traditionally emitting a load of the
8232/// address of the jump target, and indirectly jump to it).
8234 DomTreeUpdater *DTU,
8235 const DataLayout &DL,
8236 const TargetTransformInfo &TTI) {
8237 Value *Condition = SI->getCondition();
8238 LLVMContext &Context = SI->getContext();
8239 auto *CondTy = cast<IntegerType>(Condition->getType());
8240
8241 if (CondTy->getIntegerBitWidth() > 64 ||
8242 !DL.fitsInLegalInteger(CondTy->getIntegerBitWidth()))
8243 return false;
8244
8245 // Ensure trailing zeroes count intrinsic emission is not too expensive.
8246 IntrinsicCostAttributes Attrs(Intrinsic::cttz, CondTy,
8247 {Condition, ConstantInt::getTrue(Context)});
8248 if (TTI.getIntrinsicInstrCost(Attrs, TTI::TCK_SizeAndLatency) >
8249 TTI::TCC_Basic * 2)
8250 return false;
8251
8252 // Only bother with this optimization if there are more than 3 switch cases.
8253 // SDAG will start emitting jump tables for 4 or more cases.
8254 if (SI->getNumCases() < 4)
8255 return false;
8256
8257 // Check that switch cases are powers of two.
8259 for (const auto &Case : SI->cases()) {
8260 uint64_t CaseValue = Case.getCaseValue()->getValue().getZExtValue();
8261 if (llvm::has_single_bit(CaseValue))
8262 Values.push_back(CaseValue);
8263 else
8264 return false;
8265 }
8266
8267 // isSwichDense requires case values to be sorted.
8269 if (!isSwitchDense(Values.size(),
8270 llvm::countr_zero(Values.back()) -
8271 llvm::countr_zero(Values.front()) + 1,
8272 SI->getFunction()->hasOptSize()))
8273 // Transform is unable to generate dense switch.
8274 return false;
8275
8276 Builder.SetInsertPoint(SI);
8277
8278 if (!SI->defaultDestUnreachable()) {
8279 // Let non-power-of-two inputs jump to the default case, when the latter is
8280 // reachable.
8281 auto *PopC = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, Condition);
8282 auto *IsPow2 = Builder.CreateICmpEQ(PopC, ConstantInt::get(CondTy, 1));
8283
8284 auto *OrigBB = SI->getParent();
8285 auto *DefaultCaseBB = SI->getDefaultDest();
8286 BasicBlock *SplitBB = SplitBlock(OrigBB, SI, DTU);
8287 auto It = OrigBB->getTerminator()->getIterator();
8288 SmallVector<uint32_t> Weights;
8289 auto HasWeights = extractBranchWeights(*SI, Weights);
8290 auto *BI = CondBrInst::Create(IsPow2, SplitBB, DefaultCaseBB, It);
8291 if (HasWeights && any_of(Weights, not_equal_to(0))) {
8292 // IsPow2 covers a subset of the cases in which we'd go to the default
8293 // label. The other is those powers of 2 that don't appear in the case
8294 // statement. We don't know the distribution of the values coming in, so
8295 // the safest is to split 50-50 the original probability to `default`.
8296 uint64_t OrigDenominator =
8298 SmallVector<uint64_t> NewWeights(2);
8299 NewWeights[1] = Weights[0] / 2;
8300 NewWeights[0] = OrigDenominator - NewWeights[1];
8301 setFittedBranchWeights(*BI, NewWeights, /*IsExpected=*/false);
8302 // The probability of executing the default block stays constant. It was
8303 // p_d = Weights[0] / OrigDenominator
8304 // we rewrite as W/D
8305 // We want to find the probability of the default branch of the switch
8306 // statement. Let's call it X. We have W/D = W/2D + X * (1-W/2D)
8307 // i.e. the original probability is the probability we go to the default
8308 // branch from the BI branch, or we take the default branch on the SI.
8309 // Meaning X = W / (2D - W), or (W/2) / (D - W/2)
8310 // This matches using W/2 for the default branch probability numerator and
8311 // D-W/2 as the denominator.
8312 Weights[0] = NewWeights[1];
8313 uint64_t CasesDenominator = OrigDenominator - Weights[0];
8314 for (auto &W : drop_begin(Weights))
8315 W = NewWeights[0] * static_cast<double>(W) / CasesDenominator;
8316
8317 setBranchWeights(*SI, Weights, /*IsExpected=*/false);
8318 }
8319 // BI is handling the default case for SI, and so should share its DebugLoc.
8320 BI->setDebugLoc(SI->getDebugLoc());
8321 It->eraseFromParent();
8322
8323 addPredecessorToBlock(DefaultCaseBB, OrigBB, SplitBB);
8324 if (DTU)
8325 DTU->applyUpdates({{DominatorTree::Insert, OrigBB, DefaultCaseBB}});
8326 }
8327
8328 // Replace each case with its trailing zeros number.
8329 for (auto &Case : SI->cases()) {
8330 auto *OrigValue = Case.getCaseValue();
8331 Case.setValue(ConstantInt::get(OrigValue->getIntegerType(),
8332 OrigValue->getValue().countr_zero()));
8333 }
8334
8335 // Replace condition with its trailing zeros number.
8336 auto *ConditionTrailingZeros = Builder.CreateIntrinsic(
8337 Intrinsic::cttz, {CondTy}, {Condition, ConstantInt::getTrue(Context)});
8338
8339 SI->setCondition(ConditionTrailingZeros);
8340
8341 return true;
8342}
8343
8344/// Fold switch over ucmp/scmp intrinsic to br if two of the switch arms have
8345/// the same destination.
8347 DomTreeUpdater *DTU) {
8348 auto *Cmp = dyn_cast<CmpIntrinsic>(SI->getCondition());
8349 if (!Cmp || !Cmp->hasOneUse())
8350 return false;
8351
8353 bool HasWeights = extractBranchWeights(getBranchWeightMDNode(*SI), Weights);
8354 if (!HasWeights)
8355 Weights.resize(4); // Avoid checking HasWeights everywhere.
8356
8357 // Normalize to [us]cmp == Res ? Succ : OtherSucc.
8358 int64_t Res;
8359 BasicBlock *Succ, *OtherSucc;
8360 uint32_t SuccWeight = 0, OtherSuccWeight = 0;
8361 BasicBlock *Unreachable = nullptr;
8362
8363 if (SI->getNumCases() == 2) {
8364 // Find which of 1, 0 or -1 is missing (handled by default dest).
8365 SmallSet<int64_t, 3> Missing;
8366 Missing.insert(1);
8367 Missing.insert(0);
8368 Missing.insert(-1);
8369
8370 Succ = SI->getDefaultDest();
8371 SuccWeight = Weights[0];
8372 OtherSucc = nullptr;
8373 for (auto &Case : SI->cases()) {
8374 std::optional<int64_t> Val =
8375 Case.getCaseValue()->getValue().trySExtValue();
8376 if (!Val)
8377 return false;
8378 if (!Missing.erase(*Val))
8379 return false;
8380 if (OtherSucc && OtherSucc != Case.getCaseSuccessor())
8381 return false;
8382 OtherSucc = Case.getCaseSuccessor();
8383 OtherSuccWeight += Weights[Case.getSuccessorIndex()];
8384 }
8385
8386 assert(Missing.size() == 1 && "Should have one case left");
8387 Res = *Missing.begin();
8388 } else if (SI->getNumCases() == 3 && SI->defaultDestUnreachable()) {
8389 // Normalize so that Succ is taken once and OtherSucc twice.
8390 Unreachable = SI->getDefaultDest();
8391 Succ = OtherSucc = nullptr;
8392 for (auto &Case : SI->cases()) {
8393 BasicBlock *NewSucc = Case.getCaseSuccessor();
8394 uint32_t Weight = Weights[Case.getSuccessorIndex()];
8395 if (!OtherSucc || OtherSucc == NewSucc) {
8396 OtherSucc = NewSucc;
8397 OtherSuccWeight += Weight;
8398 } else if (!Succ) {
8399 Succ = NewSucc;
8400 SuccWeight = Weight;
8401 } else if (Succ == NewSucc) {
8402 std::swap(Succ, OtherSucc);
8403 std::swap(SuccWeight, OtherSuccWeight);
8404 } else
8405 return false;
8406 }
8407 for (auto &Case : SI->cases()) {
8408 std::optional<int64_t> Val =
8409 Case.getCaseValue()->getValue().trySExtValue();
8410 if (!Val || (Val != 1 && Val != 0 && Val != -1))
8411 return false;
8412 if (Case.getCaseSuccessor() == Succ) {
8413 Res = *Val;
8414 break;
8415 }
8416 }
8417 } else {
8418 return false;
8419 }
8420
8421 // Determine predicate for the missing case.
8423 switch (Res) {
8424 case 1:
8426 break;
8427 case 0:
8429 break;
8430 case -1:
8432 break;
8433 }
8434 if (Cmp->isSigned())
8436
8437 MDNode *NewWeights = nullptr;
8438 if (HasWeights)
8439 NewWeights = MDBuilder(SI->getContext())
8440 .createBranchWeights(SuccWeight, OtherSuccWeight);
8441
8442 BasicBlock *BB = SI->getParent();
8443 Builder.SetInsertPoint(SI->getIterator());
8444 Value *ICmp = Builder.CreateICmp(Pred, Cmp->getLHS(), Cmp->getRHS());
8445 Builder.CreateCondBr(ICmp, Succ, OtherSucc, NewWeights,
8446 SI->getMetadata(LLVMContext::MD_unpredictable));
8447 OtherSucc->removePredecessor(BB);
8448 if (Unreachable)
8449 Unreachable->removePredecessor(BB);
8450 SI->eraseFromParent();
8451 Cmp->eraseFromParent();
8452 if (DTU && Unreachable)
8453 DTU->applyUpdates({{DominatorTree::Delete, BB, Unreachable}});
8454 return true;
8455}
8456
8457/// Checking whether two BBs are equal depends on the contents of the
8458/// BasicBlock and the incoming values of their successor PHINodes.
8459/// PHINode::getIncomingValueForBlock is O(|Preds|), so we'd like to avoid
8460/// calling this function on each BasicBlock every time isEqual is called,
8461/// especially since the same BasicBlock may be passed as an argument multiple
8462/// times. To do this, we can precompute a map of PHINode -> Pred BasicBlock ->
8463/// IncomingValue and add it in the Wrapper so isEqual can do O(1) checking
8464/// of the incoming values.
8467
8468 // One Phi usually has < 8 incoming values.
8472
8473 // We only merge the identical non-entry BBs with
8474 // - terminator unconditional br to Succ (pending relaxation),
8475 // - does not have address taken / weird control.
8476 static bool canBeMerged(const BasicBlock *BB) {
8477 assert(BB && "Expected non-null BB");
8478 // Entry block cannot be eliminated or have predecessors.
8479 if (BB->isEntryBlock())
8480 return false;
8481
8482 // Single successor and must be Succ.
8483 // FIXME: Relax that the terminator is a BranchInst by checking for equality
8484 // on other kinds of terminators. We decide to only support unconditional
8485 // branches for now for compile time reasons.
8486 auto *BI = dyn_cast<UncondBrInst>(BB->getTerminator());
8487 if (!BI)
8488 return false;
8489
8490 // Avoid blocks that are "address-taken" (blockaddress) or have unusual
8491 // uses.
8492 if (BB->hasAddressTaken() || BB->isEHPad())
8493 return false;
8494
8495 // TODO: relax this condition to merge equal blocks with >1 instructions?
8496 // Here, we use a O(1) form of the O(n) comparison of `size() != 1`.
8497 if (&BB->front() != &BB->back())
8498 return false;
8499
8500 // The BB must have at least one predecessor.
8501 if (pred_empty(BB))
8502 return false;
8503
8504 return true;
8505 }
8506};
8507
8509 static unsigned getHashValue(const EqualBBWrapper *EBW) {
8510 BasicBlock *BB = EBW->BB;
8512 assert(BB->size() == 1 && "Expected just a single branch in the BB");
8513
8514 // Since we assume the BB is just a single UncondBrInst with a single
8515 // successor, we hash as the BB and the incoming Values of its successor
8516 // PHIs. Initially, we tried to just use the successor BB as the hash, but
8517 // including the incoming PHI values leads to better performance.
8518 // We also tried to build a map from BB -> Succs.IncomingValues ahead of
8519 // time and passing it in EqualBBWrapper, but this slowed down the average
8520 // compile time without having any impact on the worst case compile time.
8521 BasicBlock *Succ = BI->getSuccessor();
8522 auto PhiValsForBB = map_range(Succ->phis(), [&](PHINode &Phi) {
8523 return (*EBW->PhiPredIVs)[&Phi][BB];
8524 });
8525 return hash_combine(Succ, hash_combine_range(PhiValsForBB));
8526 }
8527 static bool isEqual(const EqualBBWrapper *LHS, const EqualBBWrapper *RHS) {
8528 BasicBlock *A = LHS->BB;
8529 BasicBlock *B = RHS->BB;
8530
8531 // FIXME: we checked that the size of A and B are both 1 in
8532 // mergeIdenticalUncondBBs to make the Case list smaller to
8533 // improve performance. If we decide to support BasicBlocks with more
8534 // than just a single instruction, we need to check that A.size() ==
8535 // B.size() here, and we need to check more than just the BranchInsts
8536 // for equality.
8537
8538 UncondBrInst *ABI = cast<UncondBrInst>(A->getTerminator());
8539 UncondBrInst *BBI = cast<UncondBrInst>(B->getTerminator());
8540 if (ABI->getSuccessor() != BBI->getSuccessor())
8541 return false;
8542
8543 // Need to check that PHIs in successor have matching values.
8544 BasicBlock *Succ = ABI->getSuccessor();
8545 auto IfPhiIVMatch = [&](PHINode &Phi) {
8546 // Replace O(|Pred|) Phi.getIncomingValueForBlock with this O(1) hashmap
8547 // query.
8548 auto &PredIVs = (*LHS->PhiPredIVs)[&Phi];
8549 return PredIVs[A] == PredIVs[B];
8550 };
8551 return all_of(Succ->phis(), IfPhiIVMatch);
8552 }
8553};
8554
8555// Merge identical BBs into one of them.
8557 DomTreeUpdater *DTU) {
8558 if (Candidates.size() < 2)
8559 return false;
8560
8561 // Build Cases. Skip BBs that are not candidates for simplification. Mark
8562 // PHINodes which need to be processed into PhiPredIVs. We decide to process
8563 // an entire PHI at once after the loop, opposed to calling
8564 // getIncomingValueForBlock inside this loop, since each call to
8565 // getIncomingValueForBlock is O(|Preds|).
8566 EqualBBWrapper::Phi2IVsMap PhiPredIVs;
8568 BBs2Merge.reserve(Candidates.size());
8570
8571 for (BasicBlock *BB : Candidates) {
8572 BasicBlock *Succ = BB->getSingleSuccessor();
8573 assert(Succ && "Expected unconditional BB");
8574 BBs2Merge.emplace_back(EqualBBWrapper{BB, &PhiPredIVs});
8575 Phis.insert_range(make_pointer_range(Succ->phis()));
8576 }
8577
8578 // Precompute a data structure to improve performance of isEqual for
8579 // EqualBBWrapper.
8580 PhiPredIVs.reserve(Phis.size());
8581 for (PHINode *Phi : Phis) {
8582 auto &IVs =
8583 PhiPredIVs.try_emplace(Phi, Phi->getNumIncomingValues()).first->second;
8584 // Pre-fill all incoming for O(1) lookup as Phi.getIncomingValueForBlock is
8585 // O(|Pred|).
8586 for (auto &IV : Phi->incoming_values())
8587 IVs.insert({Phi->getIncomingBlock(IV), IV.get()});
8588 }
8589
8590 // Group duplicates using DenseSet with custom equality/hashing.
8591 // Build a set such that if the EqualBBWrapper exists in the set and another
8592 // EqualBBWrapper isEqual, then the equivalent EqualBBWrapper which is not in
8593 // the set should be replaced with the one in the set. If the EqualBBWrapper
8594 // is not in the set, then it should be added to the set so other
8595 // EqualBBWrapper can check against it in the same manner. We use
8596 // EqualBBWrapper instead of just BasicBlock because we'd like to pass around
8597 // information to isEquality, getHashValue, and when doing the replacement
8598 // with better performance.
8600 Keep.reserve(BBs2Merge.size());
8601
8603 Updates.reserve(BBs2Merge.size() * 2);
8604
8605 bool MadeChange = false;
8606
8607 // Helper: redirect all edges X -> DeadPred to X -> LivePred.
8608 auto RedirectIncomingEdges = [&](BasicBlock *Dead, BasicBlock *Live) {
8611 if (DTU) {
8612 // All predecessors of DeadPred (except the common predecessor) will be
8613 // moved to LivePred.
8614 Updates.reserve(Updates.size() + DeadPreds.size() * 2);
8616 predecessors(Live));
8617 for (BasicBlock *PredOfDead : DeadPreds) {
8618 // Do not modify those common predecessors of DeadPred and LivePred.
8619 if (!LivePreds.contains(PredOfDead))
8620 Updates.push_back({DominatorTree::Insert, PredOfDead, Live});
8621 Updates.push_back({DominatorTree::Delete, PredOfDead, Dead});
8622 }
8623 }
8624 LLVM_DEBUG(dbgs() << "Replacing duplicate pred BB ";
8625 Dead->printAsOperand(dbgs()); dbgs() << " with pred ";
8626 Live->printAsOperand(dbgs()); dbgs() << " for ";
8627 Live->getSingleSuccessor()->printAsOperand(dbgs());
8628 dbgs() << "\n");
8629 // Replace successors in all predecessors of DeadPred.
8630 for (BasicBlock *PredOfDead : DeadPreds) {
8631 Instruction *T = PredOfDead->getTerminator();
8632 T->replaceSuccessorWith(Dead, Live);
8633 }
8634 };
8635
8636 // Try to eliminate duplicate predecessors.
8637 for (const auto &EBW : BBs2Merge) {
8638 // EBW is a candidate for simplification. If we find a duplicate BB,
8639 // replace it.
8640 const auto &[It, Inserted] = Keep.insert(&EBW);
8641 if (Inserted)
8642 continue;
8643
8644 // Found duplicate: merge P into canonical predecessor It->Pred.
8645 BasicBlock *KeepBB = (*It)->BB;
8646 BasicBlock *DeadBB = EBW.BB;
8647
8648 // Avoid merging a BB with itself.
8649 if (KeepBB == DeadBB)
8650 continue;
8651
8652 // Redirect all edges into DeadPred to KeepPred.
8653 RedirectIncomingEdges(DeadBB, KeepBB);
8654
8655 // Now DeadBB should become unreachable; leave DCE to later,
8656 // but we can try to simplify it if it only branches to Succ.
8657 // (We won't erase here to keep the routine simple and DT-safe.)
8658 assert(pred_empty(DeadBB) && "DeadBB should be unreachable.");
8659 MadeChange = true;
8660 }
8661
8662 if (DTU && !Updates.empty())
8663 DTU->applyUpdates(Updates);
8664
8665 return MadeChange;
8666}
8667
8668bool SimplifyCFGOpt::simplifyDuplicateSwitchArms(SwitchInst *SI,
8669 DomTreeUpdater *DTU) {
8670 // Collect candidate switch-arms top-down.
8671 SmallSetVector<BasicBlock *, 16> FilteredArms(
8674 return mergeIdenticalBBs(FilteredArms.getArrayRef(), DTU);
8675}
8676
8677bool SimplifyCFGOpt::simplifyDuplicatePredecessors(BasicBlock *BB,
8678 DomTreeUpdater *DTU) {
8679 // Need at least 2 predecessors to do anything.
8680 if (!BB || !BB->hasNPredecessorsOrMore(2))
8681 return false;
8682
8683 // Compilation time consideration: retain the canonical loop, otherwise, we
8684 // require more time in the later loop canonicalization.
8685 if (Options.NeedCanonicalLoop && is_contained(LoopHeaders, BB))
8686 return false;
8687
8688 // Collect candidate predecessors bottom-up.
8689 SmallSetVector<BasicBlock *, 8> FilteredPreds(
8692 return mergeIdenticalBBs(FilteredPreds.getArrayRef(), DTU);
8693}
8694
8695bool SimplifyCFGOpt::simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
8696 BasicBlock *BB = SI->getParent();
8697
8698 if (isValueEqualityComparison(SI)) {
8699 // If we only have one predecessor, and the predecessor branches on the
8700 // same value as this switch, try to simplify the switch based on the
8701 // conditions under which the predecessor jumps to this block.
8702 if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
8703 if (simplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder))
8704 return requestResimplify();
8705
8706 Value *Cond = SI->getCondition();
8707 if (SelectInst *Select = dyn_cast<SelectInst>(Cond))
8708 if (simplifySwitchOnSelect(SI, Select))
8709 return requestResimplify();
8710
8711 // If the block only contains the switch, see if we can fold the block
8712 // away into any preds.
8713 if (SI == &*BB->begin())
8714 if (foldValueComparisonIntoPredecessors(SI, Builder))
8715 return requestResimplify();
8716 }
8717
8718 // Try to transform the switch into an icmp and a branch.
8719 // The conversion from switch to comparison may lose information on
8720 // impossible switch values, so disable it early in the pipeline.
8721 if (Options.ConvertSwitchRangeToICmp && turnSwitchRangeIntoICmp(SI, Builder))
8722 return requestResimplify();
8723
8724 // Remove unreachable cases.
8725 if (eliminateDeadSwitchCases(SI, DTU, Options.AC, DL))
8726 return requestResimplify();
8727
8728 if (simplifySwitchOfCmpIntrinsic(SI, Builder, DTU))
8729 return requestResimplify();
8730
8731 if (trySwitchToSelect(SI, Builder, DTU, Options.AC, DL))
8732 return requestResimplify();
8733
8734 if (Options.ForwardSwitchCondToPhi && forwardSwitchConditionToPHI(SI))
8735 return requestResimplify();
8736
8737 // The conversion of switches to arithmetic or lookup table is disabled in
8738 // the early optimization pipeline, as it may lose information or make the
8739 // resulting code harder to analyze.
8740 if (Options.ConvertSwitchToArithmetic || Options.ConvertSwitchToLookupTable)
8741 if (simplifySwitchLookup(SI, Builder, DTU, DL, TTI,
8742 Options.ConvertSwitchToLookupTable))
8743 return requestResimplify();
8744
8745 if (simplifySwitchOfPowersOfTwo(SI, Builder, DTU, DL, TTI))
8746 return requestResimplify();
8747
8748 if (reduceSwitchRange(SI, Builder, DL, TTI))
8749 return requestResimplify();
8750
8751 if (HoistCommon &&
8752 hoistCommonCodeFromSuccessors(SI, !Options.HoistCommonInsts))
8753 return requestResimplify();
8754
8755 // We can merge identical switch arms early to enhance more aggressive
8756 // optimization on switch.
8757 if (simplifyDuplicateSwitchArms(SI, DTU))
8758 return requestResimplify();
8759
8760 if (simplifySwitchWhenUMin(SI, DTU))
8761 return requestResimplify();
8762
8763 if (simplifySwitchDefaultBranch(SI, DTU, Options.AC, DL))
8764 return requestResimplify();
8765
8766 return false;
8767}
8768
8769bool SimplifyCFGOpt::simplifyIndirectBr(IndirectBrInst *IBI) {
8770 BasicBlock *BB = IBI->getParent();
8771 bool Changed = false;
8772 SmallVector<uint32_t> BranchWeights;
8773 const bool HasBranchWeights = extractBranchWeights(*IBI, BranchWeights);
8774
8775 DenseMap<const BasicBlock *, uint64_t> TargetWeight;
8776 if (HasBranchWeights)
8777 for (size_t I = 0, E = IBI->getNumDestinations(); I < E; ++I)
8778 TargetWeight[IBI->getDestination(I)] += BranchWeights[I];
8779
8780 // Eliminate redundant destinations.
8781 SmallPtrSet<Value *, 8> Succs;
8782 SmallSetVector<BasicBlock *, 8> RemovedSuccs;
8783 for (unsigned I = 0, E = IBI->getNumDestinations(); I != E; ++I) {
8784 BasicBlock *Dest = IBI->getDestination(I);
8785 if (!Dest->hasAddressTaken() || !Succs.insert(Dest).second) {
8786 if (!Dest->hasAddressTaken())
8787 RemovedSuccs.insert(Dest);
8788 Dest->removePredecessor(BB);
8789 IBI->removeDestination(I);
8790 --I;
8791 --E;
8792 Changed = true;
8793 }
8794 }
8795
8796 if (DTU) {
8797 std::vector<DominatorTree::UpdateType> Updates;
8798 Updates.reserve(RemovedSuccs.size());
8799 for (auto *RemovedSucc : RemovedSuccs)
8800 Updates.push_back({DominatorTree::Delete, BB, RemovedSucc});
8801 DTU->applyUpdates(Updates);
8802 }
8803
8804 if (IBI->getNumDestinations() == 0) {
8805 // If the indirectbr has no successors, change it to unreachable.
8806 new UnreachableInst(IBI->getContext(), IBI->getIterator());
8808 return true;
8809 }
8810
8811 if (IBI->getNumDestinations() == 1) {
8812 // If the indirectbr has one successor, change it to a direct branch.
8815 return true;
8816 }
8817 if (HasBranchWeights) {
8818 SmallVector<uint64_t> NewBranchWeights(IBI->getNumDestinations());
8819 for (size_t I = 0, E = IBI->getNumDestinations(); I < E; ++I)
8820 NewBranchWeights[I] += TargetWeight.find(IBI->getDestination(I))->second;
8821 setFittedBranchWeights(*IBI, NewBranchWeights, /*IsExpected=*/false);
8822 }
8823 if (SelectInst *SI = dyn_cast<SelectInst>(IBI->getAddress())) {
8824 if (simplifyIndirectBrOnSelect(IBI, SI))
8825 return requestResimplify();
8826 }
8827 return Changed;
8828}
8829
8830/// Given an block with only a single landing pad and a unconditional branch
8831/// try to find another basic block which this one can be merged with. This
8832/// handles cases where we have multiple invokes with unique landing pads, but
8833/// a shared handler.
8834///
8835/// We specifically choose to not worry about merging non-empty blocks
8836/// here. That is a PRE/scheduling problem and is best solved elsewhere. In
8837/// practice, the optimizer produces empty landing pad blocks quite frequently
8838/// when dealing with exception dense code. (see: instcombine, gvn, if-else
8839/// sinking in this file)
8840///
8841/// This is primarily a code size optimization. We need to avoid performing
8842/// any transform which might inhibit optimization (such as our ability to
8843/// specialize a particular handler via tail commoning). We do this by not
8844/// merging any blocks which require us to introduce a phi. Since the same
8845/// values are flowing through both blocks, we don't lose any ability to
8846/// specialize. If anything, we make such specialization more likely.
8847///
8848/// Phis in the successor are allowed if both blocks pass the same value, or
8849/// each passes its own landing pad value and that is its only use: the merged
8850/// block's landing pad then carries the value for both edges.
8852 BasicBlock *BB, DomTreeUpdater *DTU) {
8853 auto Succ = BB->getUniqueSuccessor();
8854 assert(Succ);
8855
8856 for (BasicBlock *OtherPred : predecessors(Succ)) {
8857 if (BB == OtherPred)
8858 continue;
8859 BasicBlock::iterator I = OtherPred->begin();
8861 if (!LPad2 || !LPad2->isIdenticalTo(LPad))
8862 continue;
8863 ++I;
8865 if (!BI2 || !BI2->isIdenticalTo(BI))
8866 continue;
8867
8868 // Merging must not require a new phi in the merged landing pad block.
8869 if (!all_of(Succ->phis(), [&](PHINode &PN) {
8870 Value *V = PN.getIncomingValueForBlock(BB);
8871 Value *V2 = PN.getIncomingValueForBlock(OtherPred);
8872 return V == V2 || (V == LPad && V2 == LPad2);
8873 }))
8874 continue;
8875
8876 // Those phis must be the only uses of the landing pads. Otherwise, with a
8877 // loop through Succ, a use would be clobbered when the merged pad is
8878 // re-entered.
8879 unsigned NumLPadPhis = count_if(Succ->phis(), [&](PHINode &PN) {
8880 return PN.getIncomingValueForBlock(BB) == LPad;
8881 });
8882 if (!LPad->hasNUses(NumLPadPhis) || !LPad2->hasNUses(NumLPadPhis))
8883 continue;
8884
8885 std::vector<DominatorTree::UpdateType> Updates;
8886
8887 // We've found an identical block. Update our predecessors to take that
8888 // path instead and make ourselves dead.
8890 for (BasicBlock *Pred : UniquePreds) {
8891 InvokeInst *II = cast<InvokeInst>(Pred->getTerminator());
8892 assert(II->getNormalDest() != BB && II->getUnwindDest() == BB &&
8893 "unexpected successor");
8894 II->setUnwindDest(OtherPred);
8895 if (DTU) {
8896 Updates.push_back({DominatorTree::Insert, Pred, OtherPred});
8897 Updates.push_back({DominatorTree::Delete, Pred, BB});
8898 }
8899 }
8900
8902 for (BasicBlock *Succ : UniqueSuccs) {
8903 Succ->removePredecessor(BB);
8904 if (DTU)
8905 Updates.push_back({DominatorTree::Delete, BB, Succ});
8906 }
8907
8908 IRBuilder<> Builder(BI);
8909 Builder.CreateUnreachable();
8910 BI->eraseFromParent();
8911 if (DTU)
8912 DTU->applyUpdates(Updates);
8913 return true;
8914 }
8915 return false;
8916}
8917
8918bool SimplifyCFGOpt::simplifyUncondBranch(UncondBrInst *BI,
8919 IRBuilder<> &Builder) {
8920 BasicBlock *BB = BI->getParent();
8921 BasicBlock *Succ = BI->getSuccessor(0);
8922
8923 // If the Terminator is the only non-phi instruction, simplify the block.
8924 // If LoopHeader is provided, check if the block or its successor is a loop
8925 // header. (This is for early invocations before loop simplify and
8926 // vectorization to keep canonical loop forms for nested loops. These blocks
8927 // can be eliminated when the pass is invoked later in the back-end.)
8928 // Note that if BB has only one predecessor then we do not introduce new
8929 // backedge, so we can eliminate BB.
8930 bool NeedCanonicalLoop =
8931 Options.NeedCanonicalLoop &&
8932 (!LoopHeaders.empty() && BB->hasNPredecessorsOrMore(2) &&
8933 (is_contained(LoopHeaders, BB) || is_contained(LoopHeaders, Succ)));
8935 if (I->isTerminator() && BB != &BB->getParent()->getEntryBlock() &&
8936 !NeedCanonicalLoop && TryToSimplifyUncondBranchFromEmptyBlock(BB, DTU))
8937 return true;
8938
8939 // If the only instruction in the block is a seteq/setne comparison against a
8940 // constant, try to simplify the block.
8941 if (ICmpInst *ICI = dyn_cast<ICmpInst>(I)) {
8942 if (ICI->isEquality() && isa<ConstantInt>(ICI->getOperand(1))) {
8943 ++I;
8944 if (I->isTerminator() &&
8945 tryToSimplifyUncondBranchWithICmpInIt(ICI, Builder))
8946 return true;
8947 if (isa<SelectInst>(I) && I->getNextNode()->isTerminator() &&
8948 tryToSimplifyUncondBranchWithICmpSelectInIt(ICI, cast<SelectInst>(I),
8949 Builder))
8950 return true;
8951 }
8952 }
8953
8954 // See if we can merge an empty landing pad block with another which is
8955 // equivalent.
8956 if (LandingPadInst *LPad = dyn_cast<LandingPadInst>(I)) {
8957 ++I;
8958 if (I->isTerminator() && tryToMergeLandingPad(LPad, BI, BB, DTU))
8959 return true;
8960 }
8961
8962 return false;
8963}
8964
8966 BasicBlock *PredPred = nullptr;
8967 for (auto *P : predecessors(BB)) {
8968 BasicBlock *PPred = P->getSinglePredecessor();
8969 if (!PPred || (PredPred && PredPred != PPred))
8970 return nullptr;
8971 PredPred = PPred;
8972 }
8973 return PredPred;
8974}
8975
8976/// Fold the following pattern:
8977/// bb0:
8978/// br i1 %cond1, label %bb1, label %bb2
8979/// bb1:
8980/// br i1 %cond2, label %bb3, label %bb4
8981/// bb2:
8982/// br i1 %cond2, label %bb4, label %bb3
8983/// bb3:
8984/// ...
8985/// bb4:
8986/// ...
8987/// into
8988/// bb0:
8989/// %cond = xor i1 %cond1, %cond2
8990/// br i1 %cond, label %bb4, label %bb3
8991/// bb3:
8992/// ...
8993/// bb4:
8994/// ...
8995/// NOTE: %cond2 always dominates the terminator of bb0.
8997 BasicBlock *BB = BI->getParent();
8998 BasicBlock *BB1 = BI->getSuccessor(0);
8999 BasicBlock *BB2 = BI->getSuccessor(1);
9000 auto IsSimpleSuccessor = [BB](BasicBlock *Succ, CondBrInst *&SuccBI) {
9001 if (Succ == BB)
9002 return false;
9003 if (&Succ->front() != Succ->getTerminator())
9004 return false;
9005 SuccBI = dyn_cast<CondBrInst>(Succ->getTerminator());
9006 if (!SuccBI)
9007 return false;
9008 BasicBlock *Succ1 = SuccBI->getSuccessor(0);
9009 BasicBlock *Succ2 = SuccBI->getSuccessor(1);
9010 return Succ1 != Succ && Succ2 != Succ && Succ1 != BB && Succ2 != BB &&
9011 !isa<PHINode>(Succ1->front()) && !isa<PHINode>(Succ2->front());
9012 };
9013 CondBrInst *BB1BI, *BB2BI;
9014 if (!IsSimpleSuccessor(BB1, BB1BI) || !IsSimpleSuccessor(BB2, BB2BI))
9015 return false;
9016
9017 if (BB1BI->getCondition() != BB2BI->getCondition() ||
9018 BB1BI->getSuccessor(0) != BB2BI->getSuccessor(1) ||
9019 BB1BI->getSuccessor(1) != BB2BI->getSuccessor(0))
9020 return false;
9021
9022 BasicBlock *BB3 = BB1BI->getSuccessor(0);
9023 BasicBlock *BB4 = BB1BI->getSuccessor(1);
9024 // Bail out on trivial cases to avoid bothering to handle the special case in
9025 // the code below.
9026 if (BB3 == BB4)
9027 return false;
9028 IRBuilder<> Builder(BI);
9029 BI->setCondition(
9030 Builder.CreateXor(BI->getCondition(), BB1BI->getCondition()));
9031 BB1->removePredecessor(BB);
9032 BI->setSuccessor(0, BB4);
9033 BB2->removePredecessor(BB);
9034 BI->setSuccessor(1, BB3);
9035 if (DTU) {
9037 Updates.push_back({DominatorTree::Delete, BB, BB1});
9038 Updates.push_back({DominatorTree::Insert, BB, BB4});
9039 Updates.push_back({DominatorTree::Delete, BB, BB2});
9040 Updates.push_back({DominatorTree::Insert, BB, BB3});
9041
9042 DTU->applyUpdates(Updates);
9043 }
9044 bool HasWeight = false;
9045 uint64_t BBTWeight, BBFWeight;
9046 if (extractBranchWeights(*BI, BBTWeight, BBFWeight))
9047 HasWeight = true;
9048 else
9049 BBTWeight = BBFWeight = 1;
9050 uint64_t BB1TWeight, BB1FWeight;
9051 if (extractBranchWeights(*BB1BI, BB1TWeight, BB1FWeight))
9052 HasWeight = true;
9053 else
9054 BB1TWeight = BB1FWeight = 1;
9055 uint64_t BB2TWeight, BB2FWeight;
9056 if (extractBranchWeights(*BB2BI, BB2TWeight, BB2FWeight))
9057 HasWeight = true;
9058 else
9059 BB2TWeight = BB2FWeight = 1;
9060 if (HasWeight) {
9061 uint64_t Weights[2] = {BBTWeight * BB1FWeight + BBFWeight * BB2TWeight,
9062 BBTWeight * BB1TWeight + BBFWeight * BB2FWeight};
9063 setFittedBranchWeights(*BI, Weights, /*IsExpected=*/false,
9064 /*ElideAllZero=*/true);
9065 }
9066 return true;
9067}
9068
9069bool SimplifyCFGOpt::simplifyCondBranch(CondBrInst *BI, IRBuilder<> &Builder) {
9070 assert(
9072 BI->getSuccessor(0) != BI->getSuccessor(1) &&
9073 "Tautological conditional branch should have been eliminated already.");
9074
9075 BasicBlock *BB = BI->getParent();
9076 if (!Options.SimplifyCondBranch ||
9077 BI->getFunction()->hasFnAttribute(Attribute::OptForFuzzing))
9078 return false;
9079
9080 // Conditional branch
9081 if (isValueEqualityComparison(BI)) {
9082 // If we only have one predecessor, and the predecessor branches on the
9083 // same value as this branch, try to simplify the branch based on the
9084 // conditions under which the predecessor jumps to this block.
9085 if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
9086 if (simplifyEqualityComparisonWithOnlyPredecessor(BI, OnlyPred, Builder))
9087 return requestResimplify();
9088
9089 // This block must be empty, except for the setcond inst, if it exists.
9090 // Ignore pseudo intrinsics.
9091 for (auto &I : *BB) {
9092 if (isa<PseudoProbeInst>(I) ||
9093 &I == cast<Instruction>(BI->getCondition()))
9094 continue;
9095 if (&I == BI)
9096 if (foldValueComparisonIntoPredecessors(BI, Builder))
9097 return requestResimplify();
9098 break;
9099 }
9100 }
9101
9102 // Try to turn "br (X == 0 | X == 1), T, F" into a switch instruction.
9103 if (simplifyBranchOnICmpChain(BI, Builder, DL))
9104 return true;
9105
9106 // If this basic block has dominating predecessor blocks and the dominating
9107 // blocks' conditions imply BI's condition, we know the direction of BI.
9108 std::optional<bool> Imp = isImpliedByDomCondition(BI->getCondition(), BI, DL);
9109 if (Imp) {
9110 // Turn this into a branch on constant.
9111 auto *OldCond = BI->getCondition();
9112 ConstantInt *TorF = *Imp ? ConstantInt::getTrue(BB->getContext())
9113 : ConstantInt::getFalse(BB->getContext());
9114 BI->setCondition(TorF);
9116 return requestResimplify();
9117 }
9118
9119 // If this basic block is ONLY a compare and a branch, and if a predecessor
9120 // branches to us and one of our successors, fold the comparison into the
9121 // predecessor and use logical operations to pick the right destination.
9122 if (Options.SpeculateBlocks &&
9123 foldBranchToCommonDest(BI, DTU, /*MSSAU=*/nullptr, &TTI, Options.AC,
9124 Options.BonusInstThreshold))
9125 return requestResimplify();
9126
9127 // We have a conditional branch to two blocks that are only reachable
9128 // from BI. We know that the condbr dominates the two blocks, so see if
9129 // there is any identical code in the "then" and "else" blocks. If so, we
9130 // can hoist it up to the branching block.
9131 if (BI->getSuccessor(0)->getSinglePredecessor()) {
9132 if (BI->getSuccessor(1)->getSinglePredecessor()) {
9133 if (HoistCommon &&
9134 hoistCommonCodeFromSuccessors(BI, !Options.HoistCommonInsts))
9135 return requestResimplify();
9136
9137 if (BI && Options.HoistLoadsStoresWithCondFaulting &&
9138 isProfitableToSpeculate(BI, std::nullopt, TTI)) {
9139 SmallVector<Instruction *, 2> SpeculatedConditionalLoadsStores;
9140 auto CanSpeculateConditionalLoadsStores = [&]() {
9141 for (auto *Succ : successors(BB)) {
9142 for (Instruction &I : *Succ) {
9143 if (I.isTerminator()) {
9144 if (I.getNumSuccessors() > 1)
9145 return false;
9146 continue;
9147 } else if (!isSafeCheapLoadStore(&I, TTI) ||
9148 SpeculatedConditionalLoadsStores.size() ==
9150 return false;
9151 }
9152 SpeculatedConditionalLoadsStores.push_back(&I);
9153 }
9154 }
9155 return !SpeculatedConditionalLoadsStores.empty();
9156 };
9157
9158 if (CanSpeculateConditionalLoadsStores()) {
9159 hoistConditionalLoadsStores(BI, SpeculatedConditionalLoadsStores,
9160 std::nullopt, nullptr);
9161 return requestResimplify();
9162 }
9163 }
9164 } else {
9165 // If Successor #1 has multiple preds, we may be able to conditionally
9166 // execute Successor #0 if it branches to Successor #1.
9167 Instruction *Succ0TI = BI->getSuccessor(0)->getTerminator();
9168 if (Succ0TI->getNumSuccessors() == 1 &&
9169 Succ0TI->getSuccessor(0) == BI->getSuccessor(1))
9170 if (speculativelyExecuteBB(BI, BI->getSuccessor(0)))
9171 return requestResimplify();
9172 }
9173 } else if (BI->getSuccessor(1)->getSinglePredecessor()) {
9174 // If Successor #0 has multiple preds, we may be able to conditionally
9175 // execute Successor #1 if it branches to Successor #0.
9176 Instruction *Succ1TI = BI->getSuccessor(1)->getTerminator();
9177 if (Succ1TI->getNumSuccessors() == 1 &&
9178 Succ1TI->getSuccessor(0) == BI->getSuccessor(0))
9179 if (speculativelyExecuteBB(BI, BI->getSuccessor(1)))
9180 return requestResimplify();
9181 }
9182
9183 // If this is a branch on something for which we know the constant value in
9184 // predecessors (e.g. a phi node in the current block), thread control
9185 // through this block.
9186 if (foldCondBranchOnValueKnownInPredecessor(BI))
9187 return requestResimplify();
9188
9189 // Scan predecessor blocks for conditional branches.
9190 for (BasicBlock *Pred : predecessors(BB))
9191 if (CondBrInst *PBI = dyn_cast<CondBrInst>(Pred->getTerminator()))
9192 if (PBI != BI)
9193 if (SimplifyCondBranchToCondBranch(PBI, BI, DTU, DL, TTI))
9194 return requestResimplify();
9195
9196 // Look for diamond patterns.
9197 if (MergeCondStores)
9198 if (BasicBlock *PrevBB = allPredecessorsComeFromSameSource(BB))
9199 if (CondBrInst *PBI = dyn_cast<CondBrInst>(PrevBB->getTerminator()))
9200 if (PBI != BI)
9201 if (mergeConditionalStores(PBI, BI, DTU, DL, TTI))
9202 return requestResimplify();
9203
9204 // Look for nested conditional branches.
9205 if (mergeNestedCondBranch(BI, DTU))
9206 return requestResimplify();
9207
9208 return false;
9209}
9210
9211/// Check if passing a value to an instruction will cause undefined behavior.
9212static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified) {
9213 assert(V->getType() == I->getType() && "Mismatched types");
9215 if (!C)
9216 return false;
9217
9218 if (I->use_empty())
9219 return false;
9220
9221 if (C->isNullValue() || isa<UndefValue>(C)) {
9222 // Find the first same-block use with a UB-triggering opcode, skipping
9223 // cross-block or before-I uses.
9224 auto FindUse = llvm::find_if(I->uses(), [I](auto &U) {
9225 auto *Use = cast<Instruction>(U.getUser());
9226 // Only same-block uses after I can witness UB at I's program point.
9227 // Self-uses and before-I uses can occur when I is a PHI node.
9228 if (Use->getParent() != I->getParent() || Use == I || Use->comesBefore(I))
9229 return false;
9230 // Change this list when we want to add new instructions.
9231 switch (Use->getOpcode()) {
9232 default:
9233 return false;
9234 case Instruction::GetElementPtr:
9235 case Instruction::Ret:
9236 case Instruction::BitCast:
9237 case Instruction::Load:
9238 case Instruction::Store:
9239 case Instruction::Call:
9240 case Instruction::CallBr:
9241 case Instruction::Invoke:
9242 case Instruction::UDiv:
9243 case Instruction::URem:
9244 // Note: signed div/rem of INT_MIN / -1 is also immediate UB, not
9245 // implemented to avoid code complexity as it is unclear how useful such
9246 // logic is.
9247 case Instruction::SDiv:
9248 case Instruction::SRem:
9249 return true;
9250 }
9251 });
9252 if (FindUse == I->use_end())
9253 return false;
9254 auto &Use = *FindUse;
9255 auto *User = cast<Instruction>(Use.getUser());
9256
9257 // Now make sure that there are no instructions in between that can alter
9258 // control flow (eg. calls)
9259 auto InstrRange =
9260 make_range(std::next(I->getIterator()), User->getIterator());
9261 if (any_of(InstrRange, [](Instruction &I) {
9263 }))
9264 return false;
9265
9266 // Look through GEPs. A load from a GEP derived from NULL is still undefined
9268 if (GEP->getPointerOperand() == I) {
9269 // The type of GEP may differ from the type of base pointer.
9270 // Bail out on vector GEPs, as they are not handled by other checks.
9271 if (GEP->getType()->isVectorTy())
9272 return false;
9273 // The current base address is null, there are four cases to consider:
9274 // getelementptr (TY, null, 0) -> null
9275 // getelementptr (TY, null, not zero) -> may be modified
9276 // getelementptr inbounds (TY, null, 0) -> null
9277 // getelementptr inbounds (TY, null, not zero) -> poison iff null is
9278 // undefined?
9279 if (!GEP->hasAllZeroIndices() &&
9280 (!GEP->isInBounds() ||
9281 NullPointerIsDefined(GEP->getFunction(),
9282 GEP->getPointerAddressSpace())))
9283 PtrValueMayBeModified = true;
9284 return passingValueIsAlwaysUndefined(V, GEP, PtrValueMayBeModified);
9285 }
9286
9287 // Look through return.
9288 if (ReturnInst *Ret = dyn_cast<ReturnInst>(User)) {
9289 bool HasNoUndefAttr =
9290 Ret->getFunction()->hasRetAttribute(Attribute::NoUndef);
9291 // Return undefined to a noundef return value is undefined.
9292 if (isa<UndefValue>(C) && HasNoUndefAttr)
9293 return true;
9294 // Return null to a nonnull+noundef return value is undefined.
9295 if (C->isNullValue() && HasNoUndefAttr &&
9296 Ret->getFunction()->hasRetAttribute(Attribute::NonNull)) {
9297 return !PtrValueMayBeModified;
9298 }
9299 }
9300
9301 // Load from null is undefined.
9302 if (LoadInst *LI = dyn_cast<LoadInst>(User))
9303 if (!LI->isVolatile())
9304 return !NullPointerIsDefined(LI->getFunction(),
9305 LI->getPointerAddressSpace());
9306
9307 // Store to null is undefined.
9309 if (!SI->isVolatile())
9310 return (!NullPointerIsDefined(SI->getFunction(),
9311 SI->getPointerAddressSpace())) &&
9312 SI->getPointerOperand() == I;
9313
9314 // llvm.assume(false/undef) always triggers immediate UB.
9315 if (auto *Assume = dyn_cast<AssumeInst>(User)) {
9316 // Ignore assume operand bundles.
9317 if (I == Assume->getArgOperand(0))
9318 return true;
9319 }
9320
9321 if (auto *CB = dyn_cast<CallBase>(User)) {
9322 if (C->isNullValue() && NullPointerIsDefined(CB->getFunction()))
9323 return false;
9324 // A call to null is undefined.
9325 if (CB->getCalledOperand() == I)
9326 return true;
9327
9328 if (CB->isArgOperand(&Use)) {
9329 unsigned ArgIdx = CB->getArgOperandNo(&Use);
9330 // Passing null to a nonnnull+noundef argument is undefined.
9331 if (isa<ConstantPointerNull>(C) && C->getType()->isPointerTy() &&
9332 CB->paramHasNonNullAttr(ArgIdx, /*AllowUndefOrPoison=*/false))
9333 return !PtrValueMayBeModified;
9334 // Passing undef to a noundef argument is undefined.
9335 if (isa<UndefValue>(C) && CB->isPassingUndefUB(ArgIdx))
9336 return true;
9337 }
9338 }
9339 // Div/Rem by zero is immediate UB
9340 if (match(User, m_BinOp(m_Value(), m_Specific(I))) && User->isIntDivRem())
9341 return true;
9342 }
9343 return false;
9344}
9345
9346/// If BB has an incoming value that will always trigger undefined behavior
9347/// (eg. null pointer dereference), remove the branch leading here.
9349 DomTreeUpdater *DTU,
9350 AssumptionCache *AC) {
9351 for (PHINode &PHI : BB->phis())
9352 for (unsigned i = 0, e = PHI.getNumIncomingValues(); i != e; ++i)
9353 if (passingValueIsAlwaysUndefined(PHI.getIncomingValue(i), &PHI)) {
9354 BasicBlock *Predecessor = PHI.getIncomingBlock(i);
9355 Instruction *T = Predecessor->getTerminator();
9356 IRBuilder<> Builder(T);
9357 if (isa<UncondBrInst>(T)) {
9358 BB->removePredecessor(Predecessor);
9359 // Turn unconditional branches into unreachables.
9360 Builder.CreateUnreachable();
9361 T->eraseFromParent();
9362 if (DTU)
9363 DTU->applyUpdates({{DominatorTree::Delete, Predecessor, BB}});
9364 return true;
9365 } else if (CondBrInst *BI = dyn_cast<CondBrInst>(T)) {
9366 BB->removePredecessor(Predecessor);
9367 // Handle degenerate conditional branches.
9368 if (BI->getSuccessor(0) == BI->getSuccessor(1)) {
9369 // The only difference from the UncondBrInst path above is that it
9370 // has two edges in CFG.
9371 BB->removePredecessor(Predecessor);
9372 // Turn unconditional branches into unreachables.
9373 Builder.CreateUnreachable();
9374 } else {
9375 // Preserve guarding condition in assume, because it might not be
9376 // inferrable from any dominating condition.
9377 Value *Cond = BI->getCondition();
9378 CallInst *Assumption;
9379 if (BI->getSuccessor(0) == BB)
9380 Assumption = Builder.CreateAssumption(Builder.CreateNot(Cond));
9381 else
9382 Assumption = Builder.CreateAssumption(Cond);
9383 if (AC)
9384 AC->registerAssumption(cast<AssumeInst>(Assumption));
9385 Builder.CreateBr(BI->getSuccessor(0) == BB ? BI->getSuccessor(1)
9386 : BI->getSuccessor(0));
9387 }
9388 BI->eraseFromParent();
9389 if (DTU)
9390 DTU->applyUpdates({{DominatorTree::Delete, Predecessor, BB}});
9391 return true;
9392 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(T)) {
9393 // Redirect all branches leading to UB into
9394 // a newly created unreachable block.
9395 BasicBlock *Unreachable = BasicBlock::Create(
9396 Predecessor->getContext(), "unreachable", BB->getParent(), BB);
9397 Builder.SetInsertPoint(Unreachable);
9398 // The new block contains only one instruction: Unreachable
9399 Builder.CreateUnreachable();
9400 for (const auto &Case : SI->cases())
9401 if (Case.getCaseSuccessor() == BB) {
9402 BB->removePredecessor(Predecessor);
9403 Case.setSuccessor(Unreachable);
9404 }
9405 if (SI->getDefaultDest() == BB) {
9406 BB->removePredecessor(Predecessor);
9407 SI->setDefaultDest(Unreachable);
9408 }
9409
9410 if (DTU)
9411 DTU->applyUpdates(
9412 { { DominatorTree::Insert, Predecessor, Unreachable },
9413 { DominatorTree::Delete, Predecessor, BB } });
9414 return true;
9415 }
9416 }
9417
9418 return false;
9419}
9420
9421bool SimplifyCFGOpt::simplifyOnce(BasicBlock *BB) {
9422 bool Changed = false;
9423
9424 assert(BB && BB->getParent() && "Block not embedded in function!");
9425 assert(BB->hasTerminator() && "Degenerate basic block encountered!");
9426
9427 // Remove basic blocks that have no predecessors (except the entry block)...
9428 // or that just have themself as a predecessor. These are unreachable.
9429 if ((pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) ||
9430 BB->getSinglePredecessor() == BB) {
9431 LLVM_DEBUG(dbgs() << "Removing BB: \n" << *BB);
9432 DeleteDeadBlock(BB, DTU);
9433 return true;
9434 }
9435
9436 // Check to see if we can constant propagate this terminator instruction
9437 // away...
9438 Changed |= ConstantFoldTerminator(BB, /*DeleteDeadConditions=*/true,
9439 /*TLI=*/nullptr, DTU);
9440
9441 // Check for and eliminate duplicate PHI nodes in this block.
9443
9444 // Check for and remove branches that will always cause undefined behavior.
9446 return requestResimplify();
9447
9448 // Merge basic blocks into their predecessor if there is only one distinct
9449 // pred, and if there is only one distinct successor of the predecessor, and
9450 // if there are no PHI nodes.
9451 if (MergeBlockIntoPredecessor(BB, DTU))
9452 return true;
9453
9454 if (SinkCommon && Options.SinkCommonInsts) {
9455 if (sinkCommonCodeFromPredecessors(BB, DTU) ||
9456 mergeCompatibleInvokes(BB, DTU)) {
9457 // sinkCommonCodeFromPredecessors() does not automatically CSE PHI's,
9458 // so we may now have duplicate PHI's.
9459 // Let's rerun EliminateDuplicatePHINodes() first,
9460 // before foldTwoEntryPHINode() potentially converts them into select's,
9461 // after which we'd need a whole EarlyCSE pass run to cleanup them.
9462 return true;
9463 }
9464 // Merge identical predecessors of this block.
9465 if (simplifyDuplicatePredecessors(BB, DTU))
9466 return true;
9467 }
9468
9469 if (Options.SpeculateBlocks &&
9470 !BB->getParent()->hasFnAttribute(Attribute::OptForFuzzing)) {
9471 // If there is a trivial two-entry PHI node in this basic block, and we can
9472 // eliminate it, do so now.
9473 if (auto *PN = dyn_cast<PHINode>(BB->begin()))
9474 if (PN->getNumIncomingValues() == 2)
9475 if (foldTwoEntryPHINode(PN, TTI, DTU, Options.AC, DL,
9476 Options.SpeculateUnpredictables))
9477 return true;
9478 }
9479
9480 IRBuilder<> Builder(BB);
9482 Builder.SetInsertPoint(Terminator);
9483 switch (Terminator->getOpcode()) {
9484 case Instruction::UncondBr:
9485 Changed |= simplifyUncondBranch(cast<UncondBrInst>(Terminator), Builder);
9486 break;
9487 case Instruction::CondBr:
9488 Changed |= simplifyCondBranch(cast<CondBrInst>(Terminator), Builder);
9489 break;
9490 case Instruction::Resume:
9491 Changed |= simplifyResume(cast<ResumeInst>(Terminator), Builder);
9492 break;
9493 case Instruction::CleanupRet:
9494 Changed |= simplifyCleanupReturn(cast<CleanupReturnInst>(Terminator));
9495 break;
9496 case Instruction::Switch:
9497 Changed |= simplifySwitch(cast<SwitchInst>(Terminator), Builder);
9498 break;
9499 case Instruction::Unreachable:
9500 Changed |= simplifyUnreachable(cast<UnreachableInst>(Terminator));
9501 break;
9502 case Instruction::IndirectBr:
9503 Changed |= simplifyIndirectBr(cast<IndirectBrInst>(Terminator));
9504 break;
9505 }
9506
9507 return Changed;
9508}
9509
9510bool SimplifyCFGOpt::run(BasicBlock *BB) {
9511 bool Changed = false;
9512
9513 // Repeated simplify BB as long as resimplification is requested.
9514 do {
9515 Resimplify = false;
9516
9517 // Perform one round of simplifcation. Resimplify flag will be set if
9518 // another iteration is requested.
9519 Changed |= simplifyOnce(BB);
9520 } while (Resimplify);
9521
9522 return Changed;
9523}
9524
9527 ArrayRef<WeakVH> LoopHeaders) {
9528 return SimplifyCFGOpt(TTI, DTU, BB->getDataLayout(), LoopHeaders,
9529 Options)
9530 .run(BB);
9531}
#define Fail
#define Success
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
uint64_t uint64_t
AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
static MachineBasicBlock * OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
Function Alias Analysis Results
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseMap class.
@ Default
#define DEBUG_TYPE
Hexagon Common GEP
static bool IsIndirectCall(const MachineInstr *MI)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
static Constant * getFalse(Type *Ty)
For a boolean type or a vector of boolean type, return false or a vector with every element false.
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static LVOptions Options
Definition LVOptions.cpp:25
#define I(x, y, z)
Definition MD5.cpp:57
Machine Check Debug Module
This file implements a map that provides insertion order iteration.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
This file exposes an interface to building/using memory SSA to walk memory instructions using a use/d...
This file contains the declarations for metadata subclasses.
#define T
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define P(N)
if(auto Err=PB.parsePassPipeline(MPM, Passes)) return wrap(std MPM run * Mod
This file contains the declarations for profiling metadata utility functions.
static cl::opt< uint32_t > SelectFalseWeight("profcheck-default-select-false-weight", cl::init(3U), cl::desc("When annotating `select` instructions, this value will be used " "for the second ('false') case."))
static cl::opt< uint32_t > SelectTrueWeight("profcheck-default-select-true-weight", cl::init(2U), cl::desc("When annotating `select` instructions, this value will be used " "for the first ('true') case."))
SmallVector< MachineOperand, 4 > Cond
int64_t C1
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
This file contains some templates that are useful if you are working with the STL at all.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
Provides some synthesis utilities to produce sequences of values.
This file defines generic set operations that may be used on set's of different types,...
This file implements a set that has insertion order iteration characteristics.
static std::optional< ContiguousCasesResult > findContiguousCases(Value *Condition, SmallVectorImpl< ConstantInt * > &Cases, SmallVectorImpl< ConstantInt * > &OtherCases, BasicBlock *Dest, BasicBlock *OtherDest)
static void addPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred, BasicBlock *ExistPred, MemorySSAUpdater *MSSAU=nullptr)
Update PHI nodes in Succ to indicate that there will now be entries in it from the 'NewPred' block.
static bool validLookupTableConstant(Constant *C, const TargetTransformInfo &TTI)
Return true if the backend will be able to handle initializing an array of constants like C.
static StoreInst * findUniqueStoreInBlocks(BasicBlock *BB1, BasicBlock *BB2)
static bool isSwitchDense(uint64_t NumCases, uint64_t CaseRange, bool OptSize)
static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB, BasicBlock *EndBB, unsigned &SpeculatedInstructions, InstructionCost &Cost, const TargetTransformInfo &TTI)
Estimate the cost of the insertion(s) and check that the PHI nodes can be converted to selects.
static bool simplifySwitchLookup(SwitchInst *SI, IRBuilder<> &Builder, DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI, bool ConvertSwitchToLookupTable)
If the switch is only used to initialize one or more phi nodes in a common successor block with diffe...
static void removeSwitchAfterSelectFold(SwitchInst *SI, PHINode *PHI, Value *SelectValue, IRBuilder<> &Builder, DomTreeUpdater *DTU)
static bool valuesOverlap(std::vector< ValueEqualityComparisonCase > &C1, std::vector< ValueEqualityComparisonCase > &C2)
Return true if there are any keys in C1 that exist in C2 as well.
static bool isProfitableToSpeculate(const CondBrInst *BI, std::optional< bool > Invert, const TargetTransformInfo &TTI)
static bool mergeConditionalStoreToAddress(BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB, BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond, DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI)
static bool mergeCleanupPad(CleanupReturnInst *RI)
static bool isVectorOp(Instruction &I)
Return if an instruction's type or any of its operands' types are a vector type.
static BasicBlock * allPredecessorsComeFromSameSource(BasicBlock *BB)
static bool trySwitchToSelect(SwitchInst *SI, IRBuilder<> &Builder, DomTreeUpdater *DTU, AssumptionCache *AC, const DataLayout &DL)
If a switch is only used to initialize one or more phi nodes in a common successor block with only tw...
static void cloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(BasicBlock *BB, BasicBlock *PredBlock, ValueToValueMapTy &VMap)
static int constantIntSortPredicate(ConstantInt *const *P1, ConstantInt *const *P2)
static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified=false)
Check if passing a value to an instruction will cause undefined behavior.
static std::optional< std::tuple< BasicBlock *, Instruction::BinaryOps, bool > > shouldFoldCondBranchesToCommonDestination(CondBrInst *BI, CondBrInst *PBI, const TargetTransformInfo *TTI)
Determine if the two branches share a common destination and deduce a glue that joins the branches' c...
static bool isSafeToHoistInstr(Instruction *I, unsigned Flags)
static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2, Instruction *I1, Instruction *I2)
static ConstantInt * getConstantInt(Value *V, const DataLayout &DL)
Extract ConstantInt from value, looking through IntToPtr and PointerNullValue.
static bool simplifySwitchOfCmpIntrinsic(SwitchInst *SI, IRBuilderBase &Builder, DomTreeUpdater *DTU)
Fold switch over ucmp/scmp intrinsic to br if two of the switch arms have the same destination.
static bool shouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize, const TargetTransformInfo &TTI, const DataLayout &DL, const SmallVector< Type * > &ResultTypes)
Determine whether a lookup table should be built for this switch, based on the number of cases,...
static Constant * constantFold(Instruction *I, const DataLayout &DL, const SmallDenseMap< Value *, Constant * > &ConstantPool)
Try to fold instruction I into a constant.
static bool simplifySwitchDefaultBranch(SwitchInst *SI, DomTreeUpdater *DTU, AssumptionCache *AC, const DataLayout &DL)
static bool getCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest, BasicBlock **CommonDest, SmallVectorImpl< std::pair< PHINode *, Constant * > > &Res, const DataLayout &DL)
Try to determine the resulting constant values in phi nodes at the common destination basic block,...
static bool areIdenticalUpToCommutativity(const Instruction *I1, const Instruction *I2)
static bool forwardSwitchConditionToPHI(SwitchInst *SI)
Try to forward the condition of a switch instruction to a phi node dominated by the switch,...
static PHINode * findPHIForConditionForwarding(ConstantInt *CaseValue, BasicBlock *BB, int *PhiIndex)
If BB would be eligible for simplification by TryToSimplifyUncondBranchFromEmptyBlock (i....
static bool reachesUncontrolledConvergentCallBeforeBlock(BasicBlock *From, BasicBlock *StopBB)
static bool simplifySwitchOfPowersOfTwo(SwitchInst *SI, IRBuilder<> &Builder, DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI)
Tries to transform switch of powers of two to reduce switch range.
static bool isCleanupBlockEmpty(iterator_range< BasicBlock::iterator > R)
static bool isSafeToThreadThroughLoopHeader(BasicBlock *Header, ArrayRef< BasicBlock * > PredBBs, BasicBlock *RealDest)
Return true if threading PredBBs through Header to RealDest eliminates the loop's single backedge.
static Value * ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB, Value *AlternativeV=nullptr)
static Value * createLogicalOp(IRBuilderBase &Builder, Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="")
static void hoistConditionalLoadsStores(CondBrInst *BI, SmallVectorImpl< Instruction * > &SpeculatedConditionalLoadsStores, std::optional< bool > Invert, Instruction *Sel)
If the target supports conditional faulting, we look for the following pattern:
static bool shouldHoistCommonInstructions(Instruction *I1, Instruction *I2, const TargetTransformInfo &TTI)
Helper function for hoistCommonCodeFromSuccessors.
static bool reduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder, const DataLayout &DL, const TargetTransformInfo &TTI)
Try to transform a switch that has "holes" in it to a contiguous sequence of cases.
static bool safeToMergeTerminators(Instruction *SI1, Instruction *SI2, SmallSetVector< BasicBlock *, 4 > *FailBlocks=nullptr)
Return true if it is safe to merge these two terminator instructions together.
SkipFlags
@ SkipReadMem
@ SkipSideEffect
@ SkipImplicitControlFlow
static bool incomingValuesAreCompatible(BasicBlock *BB, ArrayRef< BasicBlock * > IncomingBlocks, SmallPtrSetImpl< Value * > *EquivalenceSet=nullptr)
Return true if all the PHI nodes in the basic block BB receive compatible (identical) incoming values...
static void createUnreachableSwitchDefault(SwitchInst *Switch, DomTreeUpdater *DTU, bool RemoveOrigDefaultBlock=true)
static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, DomTreeUpdater *DTU)
Check whether BB's predecessors end with unconditional branches.
static bool isTypeLegalForLookupTable(Type *Ty, const TargetTransformInfo &TTI, const DataLayout &DL)
static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU, AssumptionCache *AC, const DataLayout &DL)
Compute masked bits for the condition of a switch and use it to remove dead cases.
static bool blockIsSimpleEnoughToThreadThrough(BasicBlock *BB, BlocksSet &NonLocalUseBlocks)
Return true if we can thread a branch across this block.
static bool foldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI, DomTreeUpdater *DTU, AssumptionCache *AC, const DataLayout &DL, bool SpeculateUnpredictables)
Given a BB that starts with the specified two-entry PHI node, see if we can eliminate it.
static bool findReaching(BasicBlock *BB, BasicBlock *DefBB, BlocksSet &ReachesNonLocalUses)
static bool extractPredSuccWeights(CondBrInst *PBI, CondBrInst *BI, uint64_t &PredTrueWeight, uint64_t &PredFalseWeight, uint64_t &SuccTrueWeight, uint64_t &SuccFalseWeight)
Return true if either PBI or BI has branch weight available, and store the weights in {Pred|Succ}...
static bool shouldUseSwitchConditionAsTableIndex(ConstantInt &MinCaseVal, const ConstantInt &MaxCaseVal, bool HasDefaultResults, const SmallVector< Type * > &ResultTypes, const DataLayout &DL, const TargetTransformInfo &TTI)
static InstructionCost computeSpeculationCost(const User *I, const TargetTransformInfo &TTI)
Compute an abstract "cost" of speculating the given instruction, which is assumed to be safe to specu...
static bool performBranchToCommonDestFolding(CondBrInst *BI, CondBrInst *PBI, DomTreeUpdater *DTU, MemorySSAUpdater *MSSAU, const TargetTransformInfo *TTI)
static std::optional< unsigned > getDenseSwitchRangeReductionShift(ArrayRef< int64_t > Values, int64_t Base, bool OptSize)
SmallPtrSet< BasicBlock *, 8 > BlocksSet
static unsigned skippedInstrFlags(Instruction *I)
static bool mergeCompatibleInvokes(BasicBlock *BB, DomTreeUpdater *DTU)
If this block is a landingpad exception handling block, categorize all the predecessor invokes into s...
static bool replacingOperandWithVariableIsCheap(const Instruction *I, int OpIdx)
static void eraseTerminatorAndDCECond(Instruction *TI, MemorySSAUpdater *MSSAU=nullptr)
static void eliminateBlockCases(BasicBlock *BB, std::vector< ValueEqualityComparisonCase > &Cases)
Given a vector of bb/value pairs, remove any entries in the list that match the specified block.
static bool mergeConditionalStores(CondBrInst *PBI, CondBrInst *QBI, DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI)
static bool mergeNestedCondBranch(CondBrInst *BI, DomTreeUpdater *DTU)
Fold the following pattern: bb0: br i1 cond1, label bb1, label bb2 bb1: br i1 cond2,...
static void sinkLastInstruction(ArrayRef< BasicBlock * > Blocks)
static size_t mapCaseToResult(ConstantInt *CaseVal, SwitchCaseResultVectorTy &UniqueResults, Constant *Result)
static Value * foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector, Constant *DefaultResult, Value *Condition, IRBuilder<> &Builder, const DataLayout &DL, AssumptionCache *AC, const Instruction *CtxI, ArrayRef< uint32_t > BranchWeights)
static bool tryWidenCondBranchToCondBranch(CondBrInst *PBI, CondBrInst *BI, DomTreeUpdater *DTU)
If the previous block ended with a widenable branch, determine if reusing the target block is profita...
static void mergeCompatibleInvokesImpl(ArrayRef< InvokeInst * > Invokes, DomTreeUpdater *DTU)
static bool mergeIdenticalBBs(ArrayRef< BasicBlock * > Candidates, DomTreeUpdater *DTU)
static void getBranchWeights(Instruction *TI, SmallVectorImpl< uint64_t > &Weights)
Get Weights of a given terminator, the default weight is at the front of the vector.
static bool tryToMergeLandingPad(LandingPadInst *LPad, UncondBrInst *BI, BasicBlock *BB, DomTreeUpdater *DTU)
Given an block with only a single landing pad and a unconditional branch try to find another basic bl...
static bool initializeUniqueCases(SwitchInst *SI, PHINode *&PHI, BasicBlock *&CommonDest, SwitchCaseResultVectorTy &UniqueResults, Constant *&DefaultResult, const DataLayout &DL, uintptr_t MaxUniqueResults)
static Value * isSafeToSpeculateStore(Instruction *I, BasicBlock *BrBB, BasicBlock *StoreBB, BasicBlock *EndBB, Instruction *&PreviousAccess)
Determine if we can hoist sink a sole store instruction out of a conditional block.
static Constant * lookupConstant(Value *V, const SmallDenseMap< Value *, Constant * > &ConstantPool)
If V is a Constant, return it.
static bool SimplifyCondBranchToCondBranch(CondBrInst *PBI, CondBrInst *BI, DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI)
If we have a conditional branch as a predecessor of another block, this function tries to simplify it...
static bool canSinkInstructions(ArrayRef< Instruction * > Insts, DenseMap< const Use *, SmallVector< Value *, 4 > > &PHIOperands)
static void hoistLockstepIdenticalDbgVariableRecords(Instruction *TI, Instruction *I1, SmallVectorImpl< Instruction * > &OtherInsts)
Hoists DbgVariableRecords from I1 and OtherInstrs that are identical in lock-step to TI.
static bool removeEmptyCleanup(CleanupReturnInst *RI, DomTreeUpdater *DTU)
static bool removeUndefIntroducingPredecessor(BasicBlock *BB, DomTreeUpdater *DTU, AssumptionCache *AC)
If BB has an incoming value that will always trigger undefined behavior (eg.
static bool isUncontrolledConvergentCall(CallBase *CB)
static bool simplifySwitchWhenUMin(SwitchInst *SI, DomTreeUpdater *DTU)
Tries to transform the switch when the condition is umin with a constant.
static bool isSafeCheapLoadStore(const Instruction *I, const TargetTransformInfo &TTI)
static ConstantInt * getKnownValueOnEdge(Value *V, BasicBlock *From, BasicBlock *To)
static std::optional< bool > foldCondBranchOnValueKnownInPredecessorImpl(CondBrInst *BI, const TargetTransformInfo &TTI, DomTreeUpdater *DTU, AssumptionCache *AC, const DataLayout &DL, bool KeepCanonicalLoop)
If we have a conditional branch on something for which we know the constant value in predecessors (e....
static bool dominatesMergePoint(Value *V, BasicBlock *BB, Instruction *InsertPt, SmallPtrSetImpl< Instruction * > &AggressiveInsts, InstructionCost &Cost, InstructionCost Budget, const TargetTransformInfo &TTI, AssumptionCache *AC, SmallPtrSetImpl< Instruction * > &ZeroCostInstructions, unsigned Depth=0)
If we have a merge point of an "if condition" as accepted above, return true if the specified value d...
static void reuseTableCompare(User *PhiUser, BasicBlock *PhiBlock, CondBrInst *RangeCheckBranch, Constant *DefaultValue, const SmallVectorImpl< std::pair< ConstantInt *, Constant * > > &Values)
Try to reuse the switch table index compare.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:119
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition blake3_impl.h:83
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
unsigned popcount() const
Count the number of bits set.
Definition APInt.h:1690
bool sgt(const APInt &RHS) const
Signed greater than comparison.
Definition APInt.h:1205
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
bool sle(const APInt &RHS) const
Signed less or equal comparison.
Definition APInt.h:1170
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
Definition APInt.h:1551
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition APInt.h:352
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition APInt.h:471
LLVM_ABI APInt smul_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1998
bool slt(const APInt &RHS) const
Signed less than comparison.
Definition APInt.h:1134
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:196
std::optional< int64_t > trySExtValue() const
Get sign extended value if possible.
Definition APInt.h:1594
LLVM_ABI APInt ssub_ov(const APInt &RHS, bool &Overflow) const
Definition APInt.cpp:1979
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
const T & front() const
Get the first element.
Definition ArrayRef.h:144
size_t size() const
Get the array size.
Definition ArrayRef.h:141
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
A cache of @llvm.assume calls within a function.
LLVM_ABI void registerAssumption(AssumeInst *CI)
Add an @llvm.assume intrinsic to this function's cache.
LLVM_ABI bool getValueAsBool() const
Return the attribute's value as a boolean.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator end()
Definition BasicBlock.h:453
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:440
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition BasicBlock.h:509
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
bool hasTerminator() const LLVM_READONLY
Returns whether the block has a terminator.
Definition BasicBlock.h:232
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
Definition BasicBlock.h:666
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition BasicBlock.h:206
LLVM_ABI bool hasNPredecessors(unsigned N) const
Return true if this block has exactly N predecessors.
LLVM_ABI const BasicBlock * getUniqueSuccessor() const
Return the successor of this block if it has a unique successor.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction & front() const
Definition BasicBlock.h:463
LLVM_ABI const CallInst * getTerminatingDeoptimizeCall() const
Returns the call instruction calling @llvm.experimental.deoptimize prior to the terminating return in...
LLVM_ABI const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI void flushTerminatorDbgRecords()
Eject any debug-info trailing at the end of a block.
LLVM_ABI InstListType::const_iterator getFirstNonPHIOrPseudoProbe() const
Returns an iterator to the first instruction in this block that is not a PHINode or a pseudo operatio...
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
size_t size() const
Definition BasicBlock.h:461
LLVM_ABI bool isLandingPad() const
Return true if this basic block is a landing pad.
LLVM_ABI bool hasNPredecessorsOrMore(unsigned N) const
Return true if this block has N predecessors or more.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
void splice(BasicBlock::iterator ToIt, BasicBlock *FromBB)
Transfer all instructions from FromBB to this basic block at ToIt.
Definition BasicBlock.h:638
LLVM_ABI const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
LLVM_ABI void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
BasicBlock * getBasicBlock() const
Definition Constants.h:1125
static LLVM_ABI BranchProbability getBranchProbability(uint64_t Numerator, uint64_t Denominator)
BranchProbability getCompl() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void addRangeRetAttr(const ConstantRange &CR)
adds the range attribute to the list of attributes.
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
bool isConvergent() const
Determine if the invoke is convergent.
Value * getConvergenceControlToken() const
Return the convergence control token for this call, if it exists.
bool isDataOperand(const Use *U) const
bool tryIntersectAttributes(const CallBase *Other)
Try to intersect the attributes from 'this' CallBase and the 'Other' CallBase.
This class represents a function call, abstracting a target machine's calling convention.
mapped_iterator< op_iterator, DerefFnTy > handler_iterator
CleanupPadInst * getCleanupPad() const
Convenience accessor.
BasicBlock * getUnwindDest() const
This class is the base class for the comparison instructions.
Definition InstrTypes.h:728
bool isEquality() const
Determine if this is an equals/not equals predicate.
Definition InstrTypes.h:978
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
Predicate getPredicate() const
Return the predicate for this instruction.
Definition InstrTypes.h:828
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
Conditional Branch instruction.
static CondBrInst * Create(Value *Cond, BasicBlock *IfTrue, BasicBlock *IfFalse, InsertPosition InsertBefore=nullptr)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
void setCondition(Value *V)
Value * getCondition() const
BasicBlock * getSuccessor(unsigned i) const
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
A vector constant whose element type is a simple 1/2/4/8-byte integer or float/double,...
Definition Constants.h:951
A constant value that is initialized with an expression using other constant values.
Definition Constants.h:1316
static LLVM_ABI Constant * getNeg(Constant *C, bool HasNSW=false)
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
ConstantFolder - Create constants with minimum, target independent, folding.
This is the shared class of boolean and integer constants.
Definition Constants.h:87
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition Constants.h:225
bool isNegative() const
Definition Constants.h:214
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
Definition Constants.h:269
IntegerType * getIntegerType() const
Variant of the getType() method to always return an IntegerType, which reduces the amount of casting ...
Definition Constants.h:198
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
Definition Constants.h:135
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition Constants.h:162
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
A constant pointer value that points to null.
Definition Constants.h:716
This class represents a range of values.
LLVM_ABI bool getEquivalentICmp(CmpInst::Predicate &Pred, APInt &RHS) const
Set up Pred and RHS such that ConstantRange::makeExactICmpRegion(Pred, RHS) == *this.
LLVM_ABI ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
const APInt & getLower() const
Return the lower value for this range.
LLVM_ABI APInt getUnsignedMin() const
Return the smallest unsigned value contained in the ConstantRange.
LLVM_ABI bool isEmptySet() const
Return true if this set contains no members.
LLVM_ABI bool isSizeLargerThan(uint64_t MaxSize) const
Compare set size of this range with Value.
const APInt & getUpper() const
Return the upper value for this range.
LLVM_ABI bool isUpperWrapped() const
Return true if the exclusive upper bound wraps around the unsigned domain.
LLVM_ABI ConstantRange unionWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the union of this range with another range.
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI ConstantRange inverse() const
Return a new range that is the logical not of the current set.
LLVM_ABI APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
static ConstantRange getNonEmpty(APInt Lower, APInt Upper)
Create non-empty constant range with the given bounds.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition Constant.h:65
LLVM_ABI bool isOneValue() const
Returns true if the value is one.
Definition Constants.cpp:83
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
Base class for non-instruction debug metadata records that have positions within IR.
LLVM_ABI void removeFromParent()
simple_ilist< DbgRecord >::iterator self_iterator
Record of a variable value-assignment, aka a non instruction representation of the dbg....
A debug info location.
Definition DebugLoc.h:126
bool isSameSourceLocation(const DebugLoc &Other) const
Return true if the source locations match, ignoring isImplicitCode and source atom info.
Definition DebugLoc.h:250
static DebugLoc getTemporary()
Definition DebugLoc.h:152
static LLVM_ABI DebugLoc getMergedLocation(DebugLoc LocA, DebugLoc LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
Definition DebugLoc.cpp:186
static LLVM_ABI DebugLoc getMergedLocations(ArrayRef< DebugLoc > Locs)
Try to combine the vector of locations passed as input in a single one.
Definition DebugLoc.cpp:173
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
ValueT & at(const_arg_type_t< KeyT > Val)
Return the entry for the specified key, or abort if no such entry exists.
Definition DenseMap.h:811
void reserve(size_type NumEntries)
Grow the DenseMap so that it can contain at least NumEntries items before resizing again.
Definition DenseMap.h:721
iterator end()
Definition DenseMap.h:686
unsigned size() const
Definition DenseMap.h:717
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:856
Implements a dense probed hash-table based set.
Definition DenseSet.h:283
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
const BasicBlock & getEntryBlock() const
Definition Function.h:803
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:766
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:705
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
void applyUpdates(ArrayRef< UpdateT > Updates)
Submit updates to all available trees.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Module * getParent()
Get the module that this global value is contained inside of...
This instruction compares its operands according to the predicate given to the constructor.
Predicate getSignedPredicate() const
For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
bool isEquality() const
Return true if this predicate is either EQ or NE.
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:2474
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Definition IRBuilder.h:2215
CondBrInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Definition IRBuilder.h:1218
LLVM_ABI Value * CreateSelectFMF(Value *C, Value *True, Value *False, FMFSource FMFSource, const Twine &Name="", Instruction *MDFrom=nullptr)
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition IRBuilder.h:451
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Definition IRBuilder.h:177
Value * CreateFreeze(Value *V, const Twine &Name="")
Definition IRBuilder.h:2814
void SetCurrentDebugLocation(const DebugLoc &L)
Set location information used by debugging information.
Definition IRBuilder.h:229
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition IRBuilder.h:1534
LLVM_ABI CallInst * CreateAssumption(Value *Cond)
Create an assume intrinsic call that allows the optimizer to assume that the provided condition will ...
Value * CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Definition IRBuilder.h:2021
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", bool HasSameSign=false)
Definition IRBuilder.h:2568
UncondBrInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
Definition IRBuilder.h:1212
Value * CreateNot(Value *V, const Twine &Name="")
Definition IRBuilder.h:1856
SwitchInst * CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases=10, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a switch instruction with the specified value, default dest, and with a hint for the number of...
Definition IRBuilder.h:1241
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition IRBuilder.h:2458
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition IRBuilder.h:1913
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Definition IRBuilder.h:2197
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition IRBuilder.h:1931
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1424
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2309
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition IRBuilder.h:456
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Definition IRBuilder.h:2183
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition IRBuilder.h:2399
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition IRBuilder.h:197
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
Definition IRBuilder.h:1594
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition IRBuilder.h:1458
Provides an 'InsertHelper' that calls a user-provided callback after performing the default insertion...
Definition IRBuilder.h:74
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
Indirect Branch Instruction.
BasicBlock * getDestination(unsigned i)
Return the specified destination.
unsigned getNumDestinations() const
return the number of possible destinations in this indirectbr instruction.
LLVM_ABI void removeDestination(unsigned i)
This method removes the specified successor from the indirectbr instruction.
LLVM_ABI void dropUBImplyingAttrsAndMetadata(ArrayRef< unsigned > Keep={})
Drop any attributes or metadata that can cause immediate undefined behavior.
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI iterator_range< simple_ilist< DbgRecord >::iterator > cloneDebugInfoFrom(const Instruction *From, std::optional< simple_ilist< DbgRecord >::iterator > FromHere=std::nullopt, bool InsertAtHead=false)
Clone any debug-info attached to From onto this instruction.
LLVM_ABI unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
iterator_range< simple_ilist< DbgRecord >::iterator > getDbgRecordRange() const
Return a range over the DbgRecords attached to this instruction.
LLVM_ABI void dropLocation()
Drop the instruction's debug location.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void andIRFlags(const Value *V)
Logical 'and' of any supported wrapping, exact, and fast-math flags of V and this instruction.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
bool isTerminator() const
iterator_range< user_iterator > users()
LLVM_ABI bool isUsedOutsideOfBlock(const BasicBlock *BB) const LLVM_READONLY
Return true if there are any uses of this instruction in blocks other than the specified block.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
@ CompareUsingIntersectedAttrs
Check for equivalence with intersected callbase attrs.
LLVM_ABI AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
LLVM_ABI bool isIdenticalTo(const Instruction *I) const LLVM_READONLY
Return true if the specified instruction is exactly identical to the current one.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
LLVM_ABI void applyMergedLocation(DebugLoc LocA, DebugLoc LocB)
Merge 2 debug locations and apply it to the Instruction.
LLVM_ABI void dropDbgRecords()
Erase any DbgRecords attached to this instruction.
LLVM_ABI InstListType::iterator insertInto(BasicBlock *ParentBB, InstListType::iterator It)
Inserts an unlinked instruction into ParentBB at position It and returns the iterator of the inserted...
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
Invoke instruction.
void setNormalDest(BasicBlock *B)
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
static unsigned getPointerOperandIndex()
Iterates through instructions in a set of blocks in reverse order from the first non-terminator.
LLVM_ABI MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)
Return metadata containing two branch weights.
Definition MDBuilder.cpp:38
Metadata node.
Definition Metadata.h:1087
Helper class to manipulate !mmra metadata nodes.
bool empty() const
Definition MapVector.h:80
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition MapVector.h:127
size_type size() const
Definition MapVector.h:59
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
iterator_range< const_block_iterator > blocks() const
op_range incoming_values()
void setIncomingValue(unsigned i, Value *V)
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Value * getValue() const
Convenience accessor.
Return a value (possibly void), from a function.
This class represents the LLVM 'select' instruction.
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
void insert_range(Range &&R)
Definition SetVector.h:182
bool empty() const
Determine if the SetVector is empty or not.
Definition SetVector.h:100
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
size_type size() const
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
void insert_range(Range &&R)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:346
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition SmallSet.h:134
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void resize(size_type N)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this store instruction.
bool isAtomic() const
Return true if this store has an atomic ordering.
Align getAlign() const
bool isSimple() const
Value * getValueOperand()
bool isUnordered() const
static unsigned getPointerOperandIndex()
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this store instruction.
Value * getPointerOperand()
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
A wrapper class to simplify modification of SwitchInst cases along with their prof branch_weights met...
LLVM_ABI void setSuccessorWeight(unsigned idx, CaseWeightOpt W)
LLVM_ABI void addCase(ConstantInt *OnVal, BasicBlock *Dest, CaseWeightOpt W)
Delegate the call to the underlying SwitchInst::addCase() and set the specified branch weight for the...
LLVM_ABI CaseWeightOpt getSuccessorWeight(unsigned idx)
LLVM_ABI void replaceDefaultDest(SwitchInst::CaseIt I)
Replace the default destination by given case.
std::optional< uint32_t > CaseWeightOpt
LLVM_ABI SwitchInst::CaseIt removeCase(SwitchInst::CaseIt I)
Delegate the call to the underlying SwitchInst::removeCase() and remove correspondent branch weight.
Multiway switch.
CaseIt case_end()
Returns a read/write iterator that points one past the last in the SwitchInst.
BasicBlock * getSuccessor(unsigned idx) const
void setCondition(Value *V)
LLVM_ABI void addCase(ConstantInt *OnVal, BasicBlock *Dest)
Add an entry to the switch instruction.
CaseIteratorImpl< CaseHandle > CaseIt
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
unsigned getNumSuccessors() const
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
TargetCostKind
The kind of cost model.
@ TCK_CodeSize
Instruction code size.
@ TCK_SizeAndLatency
The weighted sum of size and latency.
@ TCC_Free
Expected to fold away in lowering.
@ TCC_Basic
The cost of a typical 'add' instruction.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:291
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
Unconditional Branch instruction.
void setSuccessor(BasicBlock *NewSucc)
static UncondBrInst * Create(BasicBlock *Target, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i=0) const
'undef' values are things that do not have specified contents.
Definition Constants.h:1657
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
Definition Use.cpp:35
LLVM_ABI void set(Value *Val)
Definition Value.h:876
User * getUser() const
Returns the User that contains this Use.
Definition Use.h:61
op_range operands()
Definition User.h:267
const Use & getOperandUse(unsigned i) const
Definition User.h:220
void setOperand(unsigned i, Value *Val)
Definition User.h:212
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition User.cpp:25
Value * getOperand(unsigned i) const
Definition User.h:207
unsigned getNumOperands() const
Definition User.h:229
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
static constexpr uint64_t MaximumAlignment
Definition Value.h:801
LLVM_ABI Value(Type *Ty, unsigned scid)
Definition Value.cpp:49
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:389
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
LLVM_ABI bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
Definition Value.cpp:142
bool use_empty() const
Definition Value.h:348
iterator_range< use_iterator > uses()
Definition Value.h:382
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
Represents an op.with.overflow intrinsic.
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
A range adaptor for a pair of iterators.
#define UINT64_MAX
Definition DataTypes.h:77
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
auto m_Cmp()
Matches any compare instruction and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
bool match(Val *V, const Pattern &P)
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
auto m_UMin(const Opnd0 &Op0, const Opnd1 &Op1)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
ExtractValue_match< Ind, Val_t > m_ExtractValue(const Val_t &V)
Match a single index ExtractValue instruction.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
ThreeOps_match< decltype(m_Value()), LHS, RHS, Instruction::Select, true > m_c_Select(const LHS &L, const RHS &R)
Match Select(C, LHS, RHS) or Select(C, RHS, LHS)
match_bind< WithOverflowInst > m_WithOverflowInst(WithOverflowInst *&I)
Match a with overflow intrinsic, capturing it if we match.
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
NoWrapTrunc_match< OpTy, TruncInst::NoUnsignedWrap > m_NUWTrunc(const OpTy &Op)
Matches trunc nuw.
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
SmallVector< DbgVariableRecord * > getDVRAssignmentMarkers(const Instruction *Inst)
Return a range of dbg_assign records for which Inst performs the assignment they encode.
Definition DebugInfo.h:209
LLVM_ABI void deleteAssignmentMarkers(const Instruction *Inst)
Delete the llvm.dbg.assign intrinsics linked to Inst.
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
constexpr double e
@ User
could "use" a pointer
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition STLExtras.h:846
bool operator<(int64_t V1, const APSInt &V2)
Definition APSInt.h:360
constexpr auto not_equal_to(T &&Arg)
Functor variant of std::not_equal_to that can be used as a UnaryPredicate in functional algorithms li...
Definition STLExtras.h:2196
LLVM_ABI bool foldBranchToCommonDest(CondBrInst *BI, llvm::DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr, const TargetTransformInfo *TTI=nullptr, AssumptionCache *AC=nullptr, unsigned BonusInstThreshold=1)
If this basic block is ONLY a setcc and a branch, and if a predecessor branches to us and one of our ...
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1781
static cl::opt< unsigned > MaxLoopHeaderThreadingSearchBlocks("simplifycfg-max-loop-header-threading-search", cl::Hidden, cl::init(1024), cl::desc("Limit number of blocks searched to prove that threading through " "a loop header keeps the loop canonical"))
LLVM_ABI cl::opt< bool > ProfcheckDisableMetadataFixes
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
If the specified value is a trivially dead instruction, delete it.
Definition Local.cpp:522
bool succ_empty(const Instruction *I)
Definition CFG.h:141
LLVM_ABI bool IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB)
Check if we can prove that all paths starting from this block converge to a block that either has a @...
LLVM_ABI bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions=false, const TargetLibraryInfo *TLI=nullptr, DomTreeUpdater *DTU=nullptr)
If a terminator instruction is predicated on a constant value, convert it into an unconditional branc...
Definition Local.cpp:133
static cl::opt< unsigned > MaxSwitchCasesPerResult("max-switch-cases-per-result", cl::Hidden, cl::init(16), cl::desc("Limit cases to analyze when converting a switch to select"))
InstructionCost Cost
RelativeUniformCounterPtr Values
Definition InstrProf.h:91
static cl::opt< bool > SpeculateOneExpensiveInst("speculate-one-expensive-inst", cl::Hidden, cl::init(true), cl::desc("Allow exactly one expensive instruction to be speculatively " "executed"))
@ Known
Known to have no common set bits.
@ Dead
Unused definition.
auto pred_end(const MachineBasicBlock *BB)
void set_intersect(S1Ty &S1, const S2Ty &S2)
set_intersect(A, B) - Compute A := A ^ B Identical to set_intersection, except that it works on set<>...
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
auto successors(const MachineBasicBlock *BB)
@ Load
The value being inserted comes from a load (InsertElement only).
auto accumulate(R &&Range, E &&Init)
Wrapper for std::accumulate.
Definition STLExtras.h:1718
constexpr from_range_t from_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
LLVM_ABI MDNode * getBranchWeightMDNode(const Instruction &I)
Get the branch weights metadata node.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2229
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:244
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
LLVM_ABI void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, bool KeepOneInputPHIs=false)
Delete the specified block, which must have no predecessors.
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
auto unique(Range &&R, Predicate P)
Definition STLExtras.h:2150
static cl::opt< unsigned > MaxSpeculationDepth("max-speculation-depth", cl::Hidden, cl::init(10), cl::desc("Limit maximum recursion depth when calculating costs of " "speculatively executed instructions"))
OutputIt copy_if(R &&Range, OutputIt Out, UnaryPredicate P)
Provide wrappers to std::copy_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1807
static cl::opt< unsigned > PHINodeFoldingThreshold("phi-node-folding-threshold", cl::Hidden, cl::init(2), cl::desc("Control the amount of phi node folding to perform (default = 2)"))
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
static cl::opt< bool > MergeCondStoresAggressively("simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false), cl::desc("When merging conditional stores, do so even if the resultant " "basic blocks are unlikely to be if-converted as a result"))
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
LLVM_ABI unsigned ComputeMaxSignificantBits(const Value *Op, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Get the upper bound on bit size for this Value Op as a signed integer.
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
Definition STLExtras.h:366
static cl::opt< unsigned > BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden, cl::init(2), cl::desc("Maximum cost of combining conditions when " "folding branches"))
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Definition MathExtras.h:380
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
static cl::opt< bool > SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true), cl::desc("Sink common instructions down to the end block"))
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Function *CtxF=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition STLExtras.h:2220
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:149
static cl::opt< bool > HoistStoresWithCondFaulting("simplifycfg-hoist-stores-with-cond-faulting", cl::Hidden, cl::init(true), cl::desc("Hoist stores if the target supports conditional faulting"))
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
constexpr detail::StaticCastFunc< To > StaticCastTo
Function objects corresponding to the Cast types defined above.
Definition Casting.h:882
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
LLVM_ABI CondBrInst * GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue, BasicBlock *&IfFalse)
Check whether BB is the merge point of a if-region.
LLVM_ABI bool TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB, DomTreeUpdater *DTU=nullptr)
BB is known to contain an unconditional branch, and contains no instructions other than PHI nodes,...
Definition Local.cpp:1185
void RemapDbgRecordRange(Module *M, iterator_range< DbgRecordIterator > Range, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataPredicate *IdentityMD=nullptr)
Remap the Values used in the DbgRecords Range using the value map VM.
LLVM_ABI void InvertBranch(CondBrInst *PBI, IRBuilderBase &Builder)
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI bool impliesPoison(const Value *ValAssumedPoison, const Value *V)
Return true if V is poison given that ValAssumedPoison is already poison.
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
static cl::opt< bool > EnableMergeCompatibleInvokes("simplifycfg-merge-compatible-invokes", cl::Hidden, cl::init(true), cl::desc("Allow SimplifyCFG to merge invokes together when appropriate"))
@ RF_IgnoreMissingLocals
If this flag is set, the remapper ignores missing function-local entries (Argument,...
Definition ValueMapper.h:98
@ RF_NoModuleLevelChanges
If this flag is set, the remapper knows that only local values within a function (such as an instruct...
Definition ValueMapper.h:80
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
auto make_first_range(ContainerTy &&c)
Given a container of pairs, return a range over the first elements.
Definition STLExtras.h:1415
LLVM_ABI bool collectPossibleValues(const Value *V, SmallPtrSetImpl< const Constant * > &Constants, unsigned MaxCount, bool AllowUndefOrPoison=true)
Enumerates all possible immediate values of V and inserts them into the set Constants.
LLVM_ABI Instruction * removeUnwindEdge(BasicBlock *BB, DomTreeUpdater *DTU=nullptr)
Replace 'BB's terminator with one that does not have an unwind successor block.
Definition Local.cpp:2923
auto succ_size(const MachineBasicBlock *BB)
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
Definition STLExtras.h:552
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
static cl::opt< unsigned > MaxJumpThreadingLiveBlocks("max-jump-threading-live-blocks", cl::Hidden, cl::init(24), cl::desc("Limit number of blocks a define in a threaded block is allowed " "to be live in"))
RNSuccIterator< NodeRef, BlockT, RegionT > succ_begin(NodeRef Node)
LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J, bool DoesKMove)
Combine the metadata of two instructions so that K can replace J.
Definition Local.cpp:3186
iterator_range(Container &&) -> iterator_range< llvm::detail::IterOfRange< Container > >
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
static cl::opt< int > MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden, cl::init(10), cl::desc("Max size of a block which is still considered " "small enough to thread through"))
LLVM_ABI BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
LLVM_ABI bool isWidenableBranch(const User *U)
Returns true iff U is a widenable branch (that is, extractWidenableCondition returns widenable condit...
@ Other
Any other memory.
Definition ModRef.h:68
TargetTransformInfo TTI
static cl::opt< unsigned > HoistCommonSkipLimit("simplifycfg-hoist-common-skip-limit", cl::Hidden, cl::init(20), cl::desc("Allow reordering across at most this many " "instructions when hoisting"))
LLVM_ABI cl::opt< bool > RequireAndPreserveDomTree
This function is used to do simplification of a CFG.
static cl::opt< bool > MergeCondStores("simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true), cl::desc("Hoist conditional stores even if an unconditional store does not " "precede - hoist multiple conditional stores into a single " "predicated store"))
static cl::opt< unsigned > BranchFoldToCommonDestVectorMultiplier("simplifycfg-branch-fold-common-dest-vector-multiplier", cl::Hidden, cl::init(2), cl::desc("Multiplier to apply to threshold when determining whether or not " "to fold branch to common destination when vector operations are " "present"))
RNSuccIterator< NodeRef, BlockT, RegionT > succ_end(NodeRef Node)
LLVM_ABI bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, MemoryDependenceResults *MemDep=nullptr, bool PredecessorWithTwoSuccessors=false, DominatorTree *DT=nullptr)
Attempts to merge a block into its predecessor, if possible.
LLVM_ABI void hoistAllInstructionsInto(BasicBlock *DomBlock, Instruction *InsertPt, BasicBlock *BB)
Hoist all of the instructions in the IfBlock to the dominant block DomBlock, by moving its instructio...
Definition Local.cpp:3467
@ Sub
Subtraction of integers.
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the specified block at the specified instruction.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition STLExtras.h:2028
IntPtrTy
Definition InstrProf.h:82
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataPredicate *IdentityMD=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM.
LLVM_ABI bool canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx)
Given an instruction, is it legal to set operand OpIdx to a non-constant value?
Definition Local.cpp:3973
DWARFExpression::Operation Op
LLVM_ABI bool PointerMayBeCaptured(const Value *V, bool ReturnCaptures, unsigned MaxUsesToExplore=0)
PointerMayBeCaptured - Return true if this pointer value may be captured by the enclosing function (w...
LLVM_ABI bool FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceResults *MemDep=nullptr)
We know that BB has one predecessor.
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
void RemapDbgRecord(Module *M, DbgRecord *DR, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataPredicate *IdentityMD=nullptr)
Remap the Values used in the DbgRecord DR using the value map VM.
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
auto sum_of(R &&Range, E Init=E{0})
Returns the sum of all values in Range with Init initial value.
Definition STLExtras.h:1733
ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy
LLVM_ABI bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
static cl::opt< bool > HoistCondStores("simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true), cl::desc("Hoist conditional stores if an unconditional store precedes"))
LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)
Extract branch weights from MD_prof metadata.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:2035
LLVM_ABI bool simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI, DomTreeUpdater *DTU=nullptr, const SimplifyCFGOptions &Options={}, ArrayRef< WeakVH > LoopHeaders={})
auto pred_begin(const MachineBasicBlock *BB)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2210
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Definition MathExtras.h:249
auto predecessors(const MachineBasicBlock *BB)
static cl::opt< unsigned > HoistLoadsStoresWithCondFaultingThreshold("hoist-loads-stores-with-cond-faulting-threshold", cl::Hidden, cl::init(6), cl::desc("Control the maximal conditional load/store that we are willing " "to speculatively execute to eliminate conditional branch " "(default = 6)"))
static cl::opt< bool > HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true), cl::desc("Hoist common instructions up to the parent block"))
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
Definition iterator.h:368
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
static cl::opt< unsigned > TwoEntryPHINodeFoldingThreshold("two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4), cl::desc("Control the maximal total instruction cost that we are willing " "to speculatively execute to fold a 2-entry PHI node into a " "select (default = 4)"))
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
PointerUnion< const Value *, const PseudoSourceValue * > ValueType
SmallVector< uint64_t, 2 > getDisjunctionWeights(const SmallVector< T1, 2 > &B1, const SmallVector< T2, 2 > &B2)
Get the branch weights of a branch conditioned on b1 || b2, where b1 and b2 are 2 booleans that are t...
bool pred_empty(const BasicBlock *BB)
Definition CFG.h:107
LLVM_ABI Constant * ConstantFoldCastInstruction(unsigned opcode, Constant *V, Type *DestTy)
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
LLVM_ABI std::optional< bool > isImpliedByDomCondition(const Value *Cond, const Instruction *ContextI, const DataLayout &DL)
Return the boolean condition value in the context of the given instruction if it is known based on do...
void array_pod_sort(IteratorTy Start, IteratorTy End)
array_pod_sort - This sorts an array with the specified start and end extent.
Definition STLExtras.h:1612
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
Definition Hashing.h:307
LLVM_ABI bool isDereferenceablePointer(const Value *V, Type *Ty, const SimplifyQuery &Q, bool IgnoreFree=false)
Equivalent to isDereferenceableAndAlignedPointer with an alignment of 1.
Definition Loads.cpp:266
bool equal(L &&LRange, R &&RRange)
Wrapper function around std::equal to detect if pair-wise elements between two ranges are the same.
Definition STLExtras.h:2162
static cl::opt< bool > HoistLoadsWithCondFaulting("simplifycfg-hoist-loads-with-cond-faulting", cl::Hidden, cl::init(true), cl::desc("Hoist loads if the target supports conditional faulting"))
LLVM_ABI Constant * ConstantFoldInstOperands(const Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands.
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef< uint64_t > Weights, bool IsExpected, bool ElideAllZero=false)
Variant of setBranchWeights where the Weights will be fit first to uint32_t by shifting right.
LLVM_ABI Constant * ConstantFoldIntegerCast(Constant *C, Type *DestTy, bool IsSigned, const DataLayout &DL)
Constant fold a zext, sext or trunc, depending on IsSigned and whether the DestTy is wider or narrowe...
bool capturesNothing(CaptureComponents CC)
Definition ModRef.h:375
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
LLVM_ABI bool EliminateDuplicatePHINodes(BasicBlock *BB)
Check for and eliminate duplicate PHI nodes in this block.
Definition Local.cpp:1550
@ Keep
No function return thunk.
Definition CodeGen.h:370
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
Definition Casting.h:866
LLVM_ABI void RemapSourceAtom(Instruction *I, ValueToValueMapTy &VM)
Remap source location atom.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition Hashing.h:287
LLVM_ABI bool isWritableObject(const Value *Object, bool &ExplicitlyDereferenceableOnly)
Return true if the Object is writable, in the sense that any location based on this pointer that can ...
LLVM_ABI void mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap)
Mark a cloned instruction as a new instance so that its source loc can be updated when remapped.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition MathExtras.h:368
LLVM_ABI void extractFromBranchWeightMD64(const MDNode *ProfileData, SmallVectorImpl< uint64_t > &Weights)
Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...
LLVM_ABI ConstantRange computeConstantRange(const Value *V, bool ForSigned, const SimplifyQuery &SQ, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
SmallVectorImpl< ConstantInt * > * Cases
SmallVectorImpl< ConstantInt * > * OtherCases
Checking whether two BBs are equal depends on the contents of the BasicBlock and the incoming values ...
SmallDenseMap< BasicBlock *, Value *, 8 > BB2ValueMap
Phi2IVsMap * PhiPredIVs
DenseMap< PHINode *, BB2ValueMap > Phi2IVsMap
static bool canBeMerged(const BasicBlock *BB)
BasicBlock * BB
LLVM_ABI AAMDNodes merge(const AAMDNodes &Other) const
Given two sets of AAMDNodes applying to potentially different locations, determine the best AAMDNodes...
static bool isEqual(const EqualBBWrapper *LHS, const EqualBBWrapper *RHS)
static unsigned getHashValue(const EqualBBWrapper *EBW)
An information struct used to provide DenseMap with the various necessary components for a given valu...
Matching combinators.
A MapVector that performs no allocations if smaller than a certain size.
Definition MapVector.h:344