LLVM 19.0.0git
SimplifyCFG.cpp
Go to the documentation of this file.
1//===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Peephole optimize the CFG.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APInt.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/Sequence.h"
20#include "llvm/ADT/SetVector.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/ADT/StringRef.h"
35#include "llvm/IR/Attributes.h"
36#include "llvm/IR/BasicBlock.h"
37#include "llvm/IR/CFG.h"
38#include "llvm/IR/Constant.h"
40#include "llvm/IR/Constants.h"
41#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/DebugInfo.h"
44#include "llvm/IR/Function.h"
45#include "llvm/IR/GlobalValue.h"
47#include "llvm/IR/IRBuilder.h"
48#include "llvm/IR/InstrTypes.h"
49#include "llvm/IR/Instruction.h"
52#include "llvm/IR/LLVMContext.h"
53#include "llvm/IR/MDBuilder.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/NoFolder.h"
58#include "llvm/IR/Operator.h"
61#include "llvm/IR/Type.h"
62#include "llvm/IR/Use.h"
63#include "llvm/IR/User.h"
64#include "llvm/IR/Value.h"
65#include "llvm/IR/ValueHandle.h"
69#include "llvm/Support/Debug.h"
77#include <algorithm>
78#include <cassert>
79#include <climits>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <optional>
85#include <set>
86#include <tuple>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace PatternMatch;
92
93#define DEBUG_TYPE "simplifycfg"
94
96 "simplifycfg-require-and-preserve-domtree", cl::Hidden,
97
98 cl::desc("Temorary development switch used to gradually uplift SimplifyCFG "
99 "into preserving DomTree,"));
100
101// Chosen as 2 so as to be cheap, but still to have enough power to fold
102// a select, so the "clamp" idiom (of a min followed by a max) will be caught.
103// To catch this, we need to fold a compare and a select, hence '2' being the
104// minimum reasonable default.
106 "phi-node-folding-threshold", cl::Hidden, cl::init(2),
107 cl::desc(
108 "Control the amount of phi node folding to perform (default = 2)"));
109
111 "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4),
112 cl::desc("Control the maximal total instruction cost that we are willing "
113 "to speculatively execute to fold a 2-entry PHI node into a "
114 "select (default = 4)"));
115
116static cl::opt<bool>
117 HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true),
118 cl::desc("Hoist common instructions up to the parent block"));
119
121 HoistCommonSkipLimit("simplifycfg-hoist-common-skip-limit", cl::Hidden,
122 cl::init(20),
123 cl::desc("Allow reordering across at most this many "
124 "instructions when hoisting"));
125
126static cl::opt<bool>
127 SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
128 cl::desc("Sink common instructions down to the end block"));
129
131 "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true),
132 cl::desc("Hoist conditional stores if an unconditional store precedes"));
133
135 "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true),
136 cl::desc("Hoist conditional stores even if an unconditional store does not "
137 "precede - hoist multiple conditional stores into a single "
138 "predicated store"));
139
141 "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false),
142 cl::desc("When merging conditional stores, do so even if the resultant "
143 "basic blocks are unlikely to be if-converted as a result"));
144
146 "speculate-one-expensive-inst", cl::Hidden, cl::init(true),
147 cl::desc("Allow exactly one expensive instruction to be speculatively "
148 "executed"));
149
151 "max-speculation-depth", cl::Hidden, cl::init(10),
152 cl::desc("Limit maximum recursion depth when calculating costs of "
153 "speculatively executed instructions"));
154
155static cl::opt<int>
156 MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden,
157 cl::init(10),
158 cl::desc("Max size of a block which is still considered "
159 "small enough to thread through"));
160
161// Two is chosen to allow one negation and a logical combine.
163 BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden,
164 cl::init(2),
165 cl::desc("Maximum cost of combining conditions when "
166 "folding branches"));
167
169 "simplifycfg-branch-fold-common-dest-vector-multiplier", cl::Hidden,
170 cl::init(2),
171 cl::desc("Multiplier to apply to threshold when determining whether or not "
172 "to fold branch to common destination when vector operations are "
173 "present"));
174
176 "simplifycfg-merge-compatible-invokes", cl::Hidden, cl::init(true),
177 cl::desc("Allow SimplifyCFG to merge invokes together when appropriate"));
178
180 "max-switch-cases-per-result", cl::Hidden, cl::init(16),
181 cl::desc("Limit cases to analyze when converting a switch to select"));
182
183STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps");
184STATISTIC(NumLinearMaps,
185 "Number of switch instructions turned into linear mapping");
186STATISTIC(NumLookupTables,
187 "Number of switch instructions turned into lookup tables");
189 NumLookupTablesHoles,
190 "Number of switch instructions turned into lookup tables (holes checked)");
191STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares");
192STATISTIC(NumFoldValueComparisonIntoPredecessors,
193 "Number of value comparisons folded into predecessor basic blocks");
194STATISTIC(NumFoldBranchToCommonDest,
195 "Number of branches folded into predecessor basic block");
197 NumHoistCommonCode,
198 "Number of common instruction 'blocks' hoisted up to the begin block");
199STATISTIC(NumHoistCommonInstrs,
200 "Number of common instructions hoisted up to the begin block");
201STATISTIC(NumSinkCommonCode,
202 "Number of common instruction 'blocks' sunk down to the end block");
203STATISTIC(NumSinkCommonInstrs,
204 "Number of common instructions sunk down to the end block");
205STATISTIC(NumSpeculations, "Number of speculative executed instructions");
206STATISTIC(NumInvokes,
207 "Number of invokes with empty resume blocks simplified into calls");
208STATISTIC(NumInvokesMerged, "Number of invokes that were merged together");
209STATISTIC(NumInvokeSetsFormed, "Number of invoke sets that were formed");
210
211namespace {
212
213// The first field contains the value that the switch produces when a certain
214// case group is selected, and the second field is a vector containing the
215// cases composing the case group.
216using SwitchCaseResultVectorTy =
218
219// The first field contains the phi node that generates a result of the switch
220// and the second field contains the value generated for a certain case in the
221// switch for that PHI.
222using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
223
224/// ValueEqualityComparisonCase - Represents a case of a switch.
225struct ValueEqualityComparisonCase {
227 BasicBlock *Dest;
228
229 ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest)
230 : Value(Value), Dest(Dest) {}
231
232 bool operator<(ValueEqualityComparisonCase RHS) const {
233 // Comparing pointers is ok as we only rely on the order for uniquing.
234 return Value < RHS.Value;
235 }
236
237 bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; }
238};
239
240class SimplifyCFGOpt {
242 DomTreeUpdater *DTU;
243 const DataLayout &DL;
244 ArrayRef<WeakVH> LoopHeaders;
246 bool Resimplify;
247
248 Value *isValueEqualityComparison(Instruction *TI);
249 BasicBlock *GetValueEqualityComparisonCases(
250 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases);
251 bool SimplifyEqualityComparisonWithOnlyPredecessor(Instruction *TI,
252 BasicBlock *Pred,
253 IRBuilder<> &Builder);
254 bool PerformValueComparisonIntoPredecessorFolding(Instruction *TI, Value *&CV,
255 Instruction *PTI,
256 IRBuilder<> &Builder);
257 bool FoldValueComparisonIntoPredecessors(Instruction *TI,
258 IRBuilder<> &Builder);
259
260 bool simplifyResume(ResumeInst *RI, IRBuilder<> &Builder);
261 bool simplifySingleResume(ResumeInst *RI);
262 bool simplifyCommonResume(ResumeInst *RI);
263 bool simplifyCleanupReturn(CleanupReturnInst *RI);
264 bool simplifyUnreachable(UnreachableInst *UI);
265 bool simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder);
266 bool simplifyIndirectBr(IndirectBrInst *IBI);
267 bool simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder);
268 bool simplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder);
269 bool simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder);
270
271 bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
272 IRBuilder<> &Builder);
273
274 bool hoistCommonCodeFromSuccessors(BasicBlock *BB, bool EqTermsOnly);
275 bool hoistSuccIdenticalTerminatorToSwitchOrIf(
276 Instruction *TI, Instruction *I1,
277 SmallVectorImpl<Instruction *> &OtherSuccTIs);
278 bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB);
279 bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
280 BasicBlock *TrueBB, BasicBlock *FalseBB,
281 uint32_t TrueWeight, uint32_t FalseWeight);
282 bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder,
283 const DataLayout &DL);
284 bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select);
285 bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI);
286 bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder);
287
288public:
289 SimplifyCFGOpt(const TargetTransformInfo &TTI, DomTreeUpdater *DTU,
290 const DataLayout &DL, ArrayRef<WeakVH> LoopHeaders,
291 const SimplifyCFGOptions &Opts)
292 : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) {
293 assert((!DTU || !DTU->hasPostDomTree()) &&
294 "SimplifyCFG is not yet capable of maintaining validity of a "
295 "PostDomTree, so don't ask for it.");
296 }
297
298 bool simplifyOnce(BasicBlock *BB);
299 bool run(BasicBlock *BB);
300
301 // Helper to set Resimplify and return change indication.
302 bool requestResimplify() {
303 Resimplify = true;
304 return true;
305 }
306};
307
308} // end anonymous namespace
309
310/// Return true if all the PHI nodes in the basic block \p BB
311/// receive compatible (identical) incoming values when coming from
312/// all of the predecessor blocks that are specified in \p IncomingBlocks.
313///
314/// Note that if the values aren't exactly identical, but \p EquivalenceSet
315/// is provided, and *both* of the values are present in the set,
316/// then they are considered equal.
318 BasicBlock *BB, ArrayRef<BasicBlock *> IncomingBlocks,
319 SmallPtrSetImpl<Value *> *EquivalenceSet = nullptr) {
320 assert(IncomingBlocks.size() == 2 &&
321 "Only for a pair of incoming blocks at the time!");
322
323 // FIXME: it is okay if one of the incoming values is an `undef` value,
324 // iff the other incoming value is guaranteed to be a non-poison value.
325 // FIXME: it is okay if one of the incoming values is a `poison` value.
326 return all_of(BB->phis(), [IncomingBlocks, EquivalenceSet](PHINode &PN) {
327 Value *IV0 = PN.getIncomingValueForBlock(IncomingBlocks[0]);
328 Value *IV1 = PN.getIncomingValueForBlock(IncomingBlocks[1]);
329 if (IV0 == IV1)
330 return true;
331 if (EquivalenceSet && EquivalenceSet->contains(IV0) &&
332 EquivalenceSet->contains(IV1))
333 return true;
334 return false;
335 });
336}
337
338/// Return true if it is safe to merge these two
339/// terminator instructions together.
340static bool
342 SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) {
343 if (SI1 == SI2)
344 return false; // Can't merge with self!
345
346 // It is not safe to merge these two switch instructions if they have a common
347 // successor, and if that successor has a PHI node, and if *that* PHI node has
348 // conflicting incoming values from the two switch blocks.
349 BasicBlock *SI1BB = SI1->getParent();
350 BasicBlock *SI2BB = SI2->getParent();
351
352 SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
353 bool Fail = false;
354 for (BasicBlock *Succ : successors(SI2BB)) {
355 if (!SI1Succs.count(Succ))
356 continue;
357 if (IncomingValuesAreCompatible(Succ, {SI1BB, SI2BB}))
358 continue;
359 Fail = true;
360 if (FailBlocks)
361 FailBlocks->insert(Succ);
362 else
363 break;
364 }
365
366 return !Fail;
367}
368
369/// Update PHI nodes in Succ to indicate that there will now be entries in it
370/// from the 'NewPred' block. The values that will be flowing into the PHI nodes
371/// will be the same as those coming in from ExistPred, an existing predecessor
372/// of Succ.
373static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
374 BasicBlock *ExistPred,
375 MemorySSAUpdater *MSSAU = nullptr) {
376 for (PHINode &PN : Succ->phis())
377 PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred);
378 if (MSSAU)
379 if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ))
380 MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred);
381}
382
383/// Compute an abstract "cost" of speculating the given instruction,
384/// which is assumed to be safe to speculate. TCC_Free means cheap,
385/// TCC_Basic means less cheap, and TCC_Expensive means prohibitively
386/// expensive.
388 const TargetTransformInfo &TTI) {
389 assert((!isa<Instruction>(I) ||
390 isSafeToSpeculativelyExecute(cast<Instruction>(I))) &&
391 "Instruction is not safe to speculatively execute!");
393}
394
395/// If we have a merge point of an "if condition" as accepted above,
396/// return true if the specified value dominates the block. We
397/// don't handle the true generality of domination here, just a special case
398/// which works well enough for us.
399///
400/// If AggressiveInsts is non-null, and if V does not dominate BB, we check to
401/// see if V (which must be an instruction) and its recursive operands
402/// that do not dominate BB have a combined cost lower than Budget and
403/// are non-trapping. If both are true, the instruction is inserted into the
404/// set and true is returned.
405///
406/// The cost for most non-trapping instructions is defined as 1 except for
407/// Select whose cost is 2.
408///
409/// After this function returns, Cost is increased by the cost of
410/// V plus its non-dominating operands. If that cost is greater than
411/// Budget, false is returned and Cost is undefined.
413 SmallPtrSetImpl<Instruction *> &AggressiveInsts,
415 InstructionCost Budget,
417 unsigned Depth = 0) {
418 // It is possible to hit a zero-cost cycle (phi/gep instructions for example),
419 // so limit the recursion depth.
420 // TODO: While this recursion limit does prevent pathological behavior, it
421 // would be better to track visited instructions to avoid cycles.
423 return false;
424
425 Instruction *I = dyn_cast<Instruction>(V);
426 if (!I) {
427 // Non-instructions dominate all instructions and can be executed
428 // unconditionally.
429 return true;
430 }
431 BasicBlock *PBB = I->getParent();
432
433 // We don't want to allow weird loops that might have the "if condition" in
434 // the bottom of this block.
435 if (PBB == BB)
436 return false;
437
438 // If this instruction is defined in a block that contains an unconditional
439 // branch to BB, then it must be in the 'conditional' part of the "if
440 // statement". If not, it definitely dominates the region.
441 BranchInst *BI = dyn_cast<BranchInst>(PBB->getTerminator());
442 if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB)
443 return true;
444
445 // If we have seen this instruction before, don't count it again.
446 if (AggressiveInsts.count(I))
447 return true;
448
449 // Okay, it looks like the instruction IS in the "condition". Check to
450 // see if it's a cheap instruction to unconditionally compute, and if it
451 // only uses stuff defined outside of the condition. If so, hoist it out.
453 return false;
454
456
457 // Allow exactly one instruction to be speculated regardless of its cost
458 // (as long as it is safe to do so).
459 // This is intended to flatten the CFG even if the instruction is a division
460 // or other expensive operation. The speculation of an expensive instruction
461 // is expected to be undone in CodeGenPrepare if the speculation has not
462 // enabled further IR optimizations.
463 if (Cost > Budget &&
464 (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0 ||
465 !Cost.isValid()))
466 return false;
467
468 // Okay, we can only really hoist these out if their operands do
469 // not take us over the cost threshold.
470 for (Use &Op : I->operands())
471 if (!dominatesMergePoint(Op, BB, AggressiveInsts, Cost, Budget, TTI,
472 Depth + 1))
473 return false;
474 // Okay, it's safe to do this! Remember this instruction.
475 AggressiveInsts.insert(I);
476 return true;
477}
478
479/// Extract ConstantInt from value, looking through IntToPtr
480/// and PointerNullValue. Return NULL if value is not a constant int.
482 // Normal constant int.
483 ConstantInt *CI = dyn_cast<ConstantInt>(V);
484 if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy() ||
485 DL.isNonIntegralPointerType(V->getType()))
486 return CI;
487
488 // This is some kind of pointer constant. Turn it into a pointer-sized
489 // ConstantInt if possible.
490 IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType()));
491
492 // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
493 if (isa<ConstantPointerNull>(V))
494 return ConstantInt::get(PtrTy, 0);
495
496 // IntToPtr const int.
497 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
498 if (CE->getOpcode() == Instruction::IntToPtr)
499 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) {
500 // The constant is very likely to have the right type already.
501 if (CI->getType() == PtrTy)
502 return CI;
503 else
504 return cast<ConstantInt>(
505 ConstantFoldIntegerCast(CI, PtrTy, /*isSigned=*/false, DL));
506 }
507 return nullptr;
508}
509
510namespace {
511
512/// Given a chain of or (||) or and (&&) comparison of a value against a
513/// constant, this will try to recover the information required for a switch
514/// structure.
515/// It will depth-first traverse the chain of comparison, seeking for patterns
516/// like %a == 12 or %a < 4 and combine them to produce a set of integer
517/// representing the different cases for the switch.
518/// Note that if the chain is composed of '||' it will build the set of elements
519/// that matches the comparisons (i.e. any of this value validate the chain)
520/// while for a chain of '&&' it will build the set elements that make the test
521/// fail.
522struct ConstantComparesGatherer {
523 const DataLayout &DL;
524
525 /// Value found for the switch comparison
526 Value *CompValue = nullptr;
527
528 /// Extra clause to be checked before the switch
529 Value *Extra = nullptr;
530
531 /// Set of integers to match in switch
533
534 /// Number of comparisons matched in the and/or chain
535 unsigned UsedICmps = 0;
536
537 /// Construct and compute the result for the comparison instruction Cond
538 ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) {
539 gather(Cond);
540 }
541
542 ConstantComparesGatherer(const ConstantComparesGatherer &) = delete;
543 ConstantComparesGatherer &
544 operator=(const ConstantComparesGatherer &) = delete;
545
546private:
547 /// Try to set the current value used for the comparison, it succeeds only if
548 /// it wasn't set before or if the new value is the same as the old one
549 bool setValueOnce(Value *NewVal) {
550 if (CompValue && CompValue != NewVal)
551 return false;
552 CompValue = NewVal;
553 return (CompValue != nullptr);
554 }
555
556 /// Try to match Instruction "I" as a comparison against a constant and
557 /// populates the array Vals with the set of values that match (or do not
558 /// match depending on isEQ).
559 /// Return false on failure. On success, the Value the comparison matched
560 /// against is placed in CompValue.
561 /// If CompValue is already set, the function is expected to fail if a match
562 /// is found but the value compared to is different.
563 bool matchInstruction(Instruction *I, bool isEQ) {
564 // If this is an icmp against a constant, handle this as one of the cases.
565 ICmpInst *ICI;
566 ConstantInt *C;
567 if (!((ICI = dyn_cast<ICmpInst>(I)) &&
568 (C = GetConstantInt(I->getOperand(1), DL)))) {
569 return false;
570 }
571
572 Value *RHSVal;
573 const APInt *RHSC;
574
575 // Pattern match a special case
576 // (x & ~2^z) == y --> x == y || x == y|2^z
577 // This undoes a transformation done by instcombine to fuse 2 compares.
578 if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
579 // It's a little bit hard to see why the following transformations are
580 // correct. Here is a CVC3 program to verify them for 64-bit values:
581
582 /*
583 ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63);
584 x : BITVECTOR(64);
585 y : BITVECTOR(64);
586 z : BITVECTOR(64);
587 mask : BITVECTOR(64) = BVSHL(ONE, z);
588 QUERY( (y & ~mask = y) =>
589 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
590 );
591 QUERY( (y | mask = y) =>
592 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
593 );
594 */
595
596 // Please note that each pattern must be a dual implication (<--> or
597 // iff). One directional implication can create spurious matches. If the
598 // implication is only one-way, an unsatisfiable condition on the left
599 // side can imply a satisfiable condition on the right side. Dual
600 // implication ensures that satisfiable conditions are transformed to
601 // other satisfiable conditions and unsatisfiable conditions are
602 // transformed to other unsatisfiable conditions.
603
604 // Here is a concrete example of a unsatisfiable condition on the left
605 // implying a satisfiable condition on the right:
606 //
607 // mask = (1 << z)
608 // (x & ~mask) == y --> (x == y || x == (y | mask))
609 //
610 // Substituting y = 3, z = 0 yields:
611 // (x & -2) == 3 --> (x == 3 || x == 2)
612
613 // Pattern match a special case:
614 /*
615 QUERY( (y & ~mask = y) =>
616 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
617 );
618 */
619 if (match(ICI->getOperand(0),
620 m_And(m_Value(RHSVal), m_APInt(RHSC)))) {
621 APInt Mask = ~*RHSC;
622 if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) {
623 // If we already have a value for the switch, it has to match!
624 if (!setValueOnce(RHSVal))
625 return false;
626
627 Vals.push_back(C);
628 Vals.push_back(
629 ConstantInt::get(C->getContext(),
630 C->getValue() | Mask));
631 UsedICmps++;
632 return true;
633 }
634 }
635
636 // Pattern match a special case:
637 /*
638 QUERY( (y | mask = y) =>
639 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
640 );
641 */
642 if (match(ICI->getOperand(0),
643 m_Or(m_Value(RHSVal), m_APInt(RHSC)))) {
644 APInt Mask = *RHSC;
645 if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) {
646 // If we already have a value for the switch, it has to match!
647 if (!setValueOnce(RHSVal))
648 return false;
649
650 Vals.push_back(C);
651 Vals.push_back(ConstantInt::get(C->getContext(),
652 C->getValue() & ~Mask));
653 UsedICmps++;
654 return true;
655 }
656 }
657
658 // If we already have a value for the switch, it has to match!
659 if (!setValueOnce(ICI->getOperand(0)))
660 return false;
661
662 UsedICmps++;
663 Vals.push_back(C);
664 return ICI->getOperand(0);
665 }
666
667 // If we have "x ult 3", for example, then we can add 0,1,2 to the set.
668 ConstantRange Span =
670
671 // Shift the range if the compare is fed by an add. This is the range
672 // compare idiom as emitted by instcombine.
673 Value *CandidateVal = I->getOperand(0);
674 if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) {
675 Span = Span.subtract(*RHSC);
676 CandidateVal = RHSVal;
677 }
678
679 // If this is an and/!= check, then we are looking to build the set of
680 // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into
681 // x != 0 && x != 1.
682 if (!isEQ)
683 Span = Span.inverse();
684
685 // If there are a ton of values, we don't want to make a ginormous switch.
686 if (Span.isSizeLargerThan(8) || Span.isEmptySet()) {
687 return false;
688 }
689
690 // If we already have a value for the switch, it has to match!
691 if (!setValueOnce(CandidateVal))
692 return false;
693
694 // Add all values from the range to the set
695 for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp)
696 Vals.push_back(ConstantInt::get(I->getContext(), Tmp));
697
698 UsedICmps++;
699 return true;
700 }
701
702 /// Given a potentially 'or'd or 'and'd together collection of icmp
703 /// eq/ne/lt/gt instructions that compare a value against a constant, extract
704 /// the value being compared, and stick the list constants into the Vals
705 /// vector.
706 /// One "Extra" case is allowed to differ from the other.
707 void gather(Value *V) {
708 bool isEQ = match(V, m_LogicalOr(m_Value(), m_Value()));
709
710 // Keep a stack (SmallVector for efficiency) for depth-first traversal
713
714 // Initialize
715 Visited.insert(V);
716 DFT.push_back(V);
717
718 while (!DFT.empty()) {
719 V = DFT.pop_back_val();
720
721 if (Instruction *I = dyn_cast<Instruction>(V)) {
722 // If it is a || (or && depending on isEQ), process the operands.
723 Value *Op0, *Op1;
724 if (isEQ ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1)))
725 : match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
726 if (Visited.insert(Op1).second)
727 DFT.push_back(Op1);
728 if (Visited.insert(Op0).second)
729 DFT.push_back(Op0);
730
731 continue;
732 }
733
734 // Try to match the current instruction
735 if (matchInstruction(I, isEQ))
736 // Match succeed, continue the loop
737 continue;
738 }
739
740 // One element of the sequence of || (or &&) could not be match as a
741 // comparison against the same value as the others.
742 // We allow only one "Extra" case to be checked before the switch
743 if (!Extra) {
744 Extra = V;
745 continue;
746 }
747 // Failed to parse a proper sequence, abort now
748 CompValue = nullptr;
749 break;
750 }
751 }
752};
753
754} // end anonymous namespace
755
757 MemorySSAUpdater *MSSAU = nullptr) {
758 Instruction *Cond = nullptr;
759 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
760 Cond = dyn_cast<Instruction>(SI->getCondition());
761 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
762 if (BI->isConditional())
763 Cond = dyn_cast<Instruction>(BI->getCondition());
764 } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) {
765 Cond = dyn_cast<Instruction>(IBI->getAddress());
766 }
767
768 TI->eraseFromParent();
769 if (Cond)
771}
772
773/// Return true if the specified terminator checks
774/// to see if a value is equal to constant integer value.
775Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) {
776 Value *CV = nullptr;
777 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
778 // Do not permit merging of large switch instructions into their
779 // predecessors unless there is only one predecessor.
780 if (!SI->getParent()->hasNPredecessorsOrMore(128 / SI->getNumSuccessors()))
781 CV = SI->getCondition();
782 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI))
783 if (BI->isConditional() && BI->getCondition()->hasOneUse())
784 if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) {
785 if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL))
786 CV = ICI->getOperand(0);
787 }
788
789 // Unwrap any lossless ptrtoint cast.
790 if (CV) {
791 if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) {
792 Value *Ptr = PTII->getPointerOperand();
793 if (PTII->getType() == DL.getIntPtrType(Ptr->getType()))
794 CV = Ptr;
795 }
796 }
797 return CV;
798}
799
800/// Given a value comparison instruction,
801/// decode all of the 'cases' that it represents and return the 'default' block.
802BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases(
803 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases) {
804 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
805 Cases.reserve(SI->getNumCases());
806 for (auto Case : SI->cases())
807 Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(),
808 Case.getCaseSuccessor()));
809 return SI->getDefaultDest();
810 }
811
812 BranchInst *BI = cast<BranchInst>(TI);
813 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
814 BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE);
815 Cases.push_back(ValueEqualityComparisonCase(
816 GetConstantInt(ICI->getOperand(1), DL), Succ));
817 return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ);
818}
819
820/// Given a vector of bb/value pairs, remove any entries
821/// in the list that match the specified block.
822static void
824 std::vector<ValueEqualityComparisonCase> &Cases) {
825 llvm::erase(Cases, BB);
826}
827
828/// Return true if there are any keys in C1 that exist in C2 as well.
829static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
830 std::vector<ValueEqualityComparisonCase> &C2) {
831 std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2;
832
833 // Make V1 be smaller than V2.
834 if (V1->size() > V2->size())
835 std::swap(V1, V2);
836
837 if (V1->empty())
838 return false;
839 if (V1->size() == 1) {
840 // Just scan V2.
841 ConstantInt *TheVal = (*V1)[0].Value;
842 for (const ValueEqualityComparisonCase &VECC : *V2)
843 if (TheVal == VECC.Value)
844 return true;
845 }
846
847 // Otherwise, just sort both lists and compare element by element.
848 array_pod_sort(V1->begin(), V1->end());
849 array_pod_sort(V2->begin(), V2->end());
850 unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size();
851 while (i1 != e1 && i2 != e2) {
852 if ((*V1)[i1].Value == (*V2)[i2].Value)
853 return true;
854 if ((*V1)[i1].Value < (*V2)[i2].Value)
855 ++i1;
856 else
857 ++i2;
858 }
859 return false;
860}
861
862// Set branch weights on SwitchInst. This sets the metadata if there is at
863// least one non-zero weight.
865 bool IsExpected) {
866 // Check that there is at least one non-zero weight. Otherwise, pass
867 // nullptr to setMetadata which will erase the existing metadata.
868 MDNode *N = nullptr;
869 if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; }))
870 N = MDBuilder(SI->getParent()->getContext())
871 .createBranchWeights(Weights, IsExpected);
872 SI->setMetadata(LLVMContext::MD_prof, N);
873}
874
875// Similar to the above, but for branch and select instructions that take
876// exactly 2 weights.
877static void setBranchWeights(Instruction *I, uint32_t TrueWeight,
878 uint32_t FalseWeight, bool IsExpected) {
879 assert(isa<BranchInst>(I) || isa<SelectInst>(I));
880 // Check that there is at least one non-zero weight. Otherwise, pass
881 // nullptr to setMetadata which will erase the existing metadata.
882 MDNode *N = nullptr;
883 if (TrueWeight || FalseWeight)
884 N = MDBuilder(I->getParent()->getContext())
885 .createBranchWeights(TrueWeight, FalseWeight, IsExpected);
886 I->setMetadata(LLVMContext::MD_prof, N);
887}
888
889/// If TI is known to be a terminator instruction and its block is known to
890/// only have a single predecessor block, check to see if that predecessor is
891/// also a value comparison with the same value, and if that comparison
892/// determines the outcome of this comparison. If so, simplify TI. This does a
893/// very limited form of jump threading.
894bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
895 Instruction *TI, BasicBlock *Pred, IRBuilder<> &Builder) {
896 Value *PredVal = isValueEqualityComparison(Pred->getTerminator());
897 if (!PredVal)
898 return false; // Not a value comparison in predecessor.
899
900 Value *ThisVal = isValueEqualityComparison(TI);
901 assert(ThisVal && "This isn't a value comparison!!");
902 if (ThisVal != PredVal)
903 return false; // Different predicates.
904
905 // TODO: Preserve branch weight metadata, similarly to how
906 // FoldValueComparisonIntoPredecessors preserves it.
907
908 // Find out information about when control will move from Pred to TI's block.
909 std::vector<ValueEqualityComparisonCase> PredCases;
910 BasicBlock *PredDef =
911 GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases);
912 EliminateBlockCases(PredDef, PredCases); // Remove default from cases.
913
914 // Find information about how control leaves this block.
915 std::vector<ValueEqualityComparisonCase> ThisCases;
916 BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases);
917 EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases.
918
919 // If TI's block is the default block from Pred's comparison, potentially
920 // simplify TI based on this knowledge.
921 if (PredDef == TI->getParent()) {
922 // If we are here, we know that the value is none of those cases listed in
923 // PredCases. If there are any cases in ThisCases that are in PredCases, we
924 // can simplify TI.
925 if (!ValuesOverlap(PredCases, ThisCases))
926 return false;
927
928 if (isa<BranchInst>(TI)) {
929 // Okay, one of the successors of this condbr is dead. Convert it to a
930 // uncond br.
931 assert(ThisCases.size() == 1 && "Branch can only have one case!");
932 // Insert the new branch.
933 Instruction *NI = Builder.CreateBr(ThisDef);
934 (void)NI;
935
936 // Remove PHI node entries for the dead edge.
937 ThisCases[0].Dest->removePredecessor(PredDef);
938
939 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
940 << "Through successor TI: " << *TI << "Leaving: " << *NI
941 << "\n");
942
944
945 if (DTU)
946 DTU->applyUpdates(
947 {{DominatorTree::Delete, PredDef, ThisCases[0].Dest}});
948
949 return true;
950 }
951
952 SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI);
953 // Okay, TI has cases that are statically dead, prune them away.
955 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
956 DeadCases.insert(PredCases[i].Value);
957
958 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
959 << "Through successor TI: " << *TI);
960
961 SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
962 for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
963 --i;
964 auto *Successor = i->getCaseSuccessor();
965 if (DTU)
966 ++NumPerSuccessorCases[Successor];
967 if (DeadCases.count(i->getCaseValue())) {
968 Successor->removePredecessor(PredDef);
969 SI.removeCase(i);
970 if (DTU)
971 --NumPerSuccessorCases[Successor];
972 }
973 }
974
975 if (DTU) {
976 std::vector<DominatorTree::UpdateType> Updates;
977 for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
978 if (I.second == 0)
979 Updates.push_back({DominatorTree::Delete, PredDef, I.first});
980 DTU->applyUpdates(Updates);
981 }
982
983 LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n");
984 return true;
985 }
986
987 // Otherwise, TI's block must correspond to some matched value. Find out
988 // which value (or set of values) this is.
989 ConstantInt *TIV = nullptr;
990 BasicBlock *TIBB = TI->getParent();
991 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
992 if (PredCases[i].Dest == TIBB) {
993 if (TIV)
994 return false; // Cannot handle multiple values coming to this block.
995 TIV = PredCases[i].Value;
996 }
997 assert(TIV && "No edge from pred to succ?");
998
999 // Okay, we found the one constant that our value can be if we get into TI's
1000 // BB. Find out which successor will unconditionally be branched to.
1001 BasicBlock *TheRealDest = nullptr;
1002 for (unsigned i = 0, e = ThisCases.size(); i != e; ++i)
1003 if (ThisCases[i].Value == TIV) {
1004 TheRealDest = ThisCases[i].Dest;
1005 break;
1006 }
1007
1008 // If not handled by any explicit cases, it is handled by the default case.
1009 if (!TheRealDest)
1010 TheRealDest = ThisDef;
1011
1012 SmallPtrSet<BasicBlock *, 2> RemovedSuccs;
1013
1014 // Remove PHI node entries for dead edges.
1015 BasicBlock *CheckEdge = TheRealDest;
1016 for (BasicBlock *Succ : successors(TIBB))
1017 if (Succ != CheckEdge) {
1018 if (Succ != TheRealDest)
1019 RemovedSuccs.insert(Succ);
1020 Succ->removePredecessor(TIBB);
1021 } else
1022 CheckEdge = nullptr;
1023
1024 // Insert the new branch.
1025 Instruction *NI = Builder.CreateBr(TheRealDest);
1026 (void)NI;
1027
1028 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
1029 << "Through successor TI: " << *TI << "Leaving: " << *NI
1030 << "\n");
1031
1033 if (DTU) {
1035 Updates.reserve(RemovedSuccs.size());
1036 for (auto *RemovedSucc : RemovedSuccs)
1037 Updates.push_back({DominatorTree::Delete, TIBB, RemovedSucc});
1038 DTU->applyUpdates(Updates);
1039 }
1040 return true;
1041}
1042
1043namespace {
1044
1045/// This class implements a stable ordering of constant
1046/// integers that does not depend on their address. This is important for
1047/// applications that sort ConstantInt's to ensure uniqueness.
1048struct ConstantIntOrdering {
1049 bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const {
1050 return LHS->getValue().ult(RHS->getValue());
1051 }
1052};
1053
1054} // end anonymous namespace
1055
1057 ConstantInt *const *P2) {
1058 const ConstantInt *LHS = *P1;
1059 const ConstantInt *RHS = *P2;
1060 if (LHS == RHS)
1061 return 0;
1062 return LHS->getValue().ult(RHS->getValue()) ? 1 : -1;
1063}
1064
1065/// Get Weights of a given terminator, the default weight is at the front
1066/// of the vector. If TI is a conditional eq, we need to swap the branch-weight
1067/// metadata.
1069 SmallVectorImpl<uint64_t> &Weights) {
1070 MDNode *MD = TI->getMetadata(LLVMContext::MD_prof);
1071 assert(MD && "Invalid branch-weight metadata");
1072 extractFromBranchWeightMD64(MD, Weights);
1073
1074 // If TI is a conditional eq, the default case is the false case,
1075 // and the corresponding branch-weight data is at index 2. We swap the
1076 // default weight to be the first entry.
1077 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
1078 assert(Weights.size() == 2);
1079 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
1080 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
1081 std::swap(Weights.front(), Weights.back());
1082 }
1083}
1084
1085/// Keep halving the weights until all can fit in uint32_t.
1087 uint64_t Max = *llvm::max_element(Weights);
1088 if (Max > UINT_MAX) {
1089 unsigned Offset = 32 - llvm::countl_zero(Max);
1090 for (uint64_t &I : Weights)
1091 I >>= Offset;
1092 }
1093}
1094
1096 BasicBlock *BB, BasicBlock *PredBlock, ValueToValueMapTy &VMap) {
1097 Instruction *PTI = PredBlock->getTerminator();
1098
1099 // If we have bonus instructions, clone them into the predecessor block.
1100 // Note that there may be multiple predecessor blocks, so we cannot move
1101 // bonus instructions to a predecessor block.
1102 for (Instruction &BonusInst : *BB) {
1103 if (BonusInst.isTerminator())
1104 continue;
1105
1106 Instruction *NewBonusInst = BonusInst.clone();
1107
1108 if (!isa<DbgInfoIntrinsic>(BonusInst) &&
1109 PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) {
1110 // Unless the instruction has the same !dbg location as the original
1111 // branch, drop it. When we fold the bonus instructions we want to make
1112 // sure we reset their debug locations in order to avoid stepping on
1113 // dead code caused by folding dead branches.
1114 NewBonusInst->setDebugLoc(DebugLoc());
1115 }
1116
1117 RemapInstruction(NewBonusInst, VMap,
1119
1120 // If we speculated an instruction, we need to drop any metadata that may
1121 // result in undefined behavior, as the metadata might have been valid
1122 // only given the branch precondition.
1123 // Similarly strip attributes on call parameters that may cause UB in
1124 // location the call is moved to.
1125 NewBonusInst->dropUBImplyingAttrsAndMetadata();
1126
1127 NewBonusInst->insertInto(PredBlock, PTI->getIterator());
1128 auto Range = NewBonusInst->cloneDebugInfoFrom(&BonusInst);
1129 RemapDbgRecordRange(NewBonusInst->getModule(), Range, VMap,
1131
1132 if (isa<DbgInfoIntrinsic>(BonusInst))
1133 continue;
1134
1135 NewBonusInst->takeName(&BonusInst);
1136 BonusInst.setName(NewBonusInst->getName() + ".old");
1137 VMap[&BonusInst] = NewBonusInst;
1138
1139 // Update (liveout) uses of bonus instructions,
1140 // now that the bonus instruction has been cloned into predecessor.
1141 // Note that we expect to be in a block-closed SSA form for this to work!
1142 for (Use &U : make_early_inc_range(BonusInst.uses())) {
1143 auto *UI = cast<Instruction>(U.getUser());
1144 auto *PN = dyn_cast<PHINode>(UI);
1145 if (!PN) {
1146 assert(UI->getParent() == BB && BonusInst.comesBefore(UI) &&
1147 "If the user is not a PHI node, then it should be in the same "
1148 "block as, and come after, the original bonus instruction.");
1149 continue; // Keep using the original bonus instruction.
1150 }
1151 // Is this the block-closed SSA form PHI node?
1152 if (PN->getIncomingBlock(U) == BB)
1153 continue; // Great, keep using the original bonus instruction.
1154 // The only other alternative is an "use" when coming from
1155 // the predecessor block - here we should refer to the cloned bonus instr.
1156 assert(PN->getIncomingBlock(U) == PredBlock &&
1157 "Not in block-closed SSA form?");
1158 U.set(NewBonusInst);
1159 }
1160 }
1161}
1162
1163bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding(
1164 Instruction *TI, Value *&CV, Instruction *PTI, IRBuilder<> &Builder) {
1165 BasicBlock *BB = TI->getParent();
1166 BasicBlock *Pred = PTI->getParent();
1167
1169
1170 // Figure out which 'cases' to copy from SI to PSI.
1171 std::vector<ValueEqualityComparisonCase> BBCases;
1172 BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases);
1173
1174 std::vector<ValueEqualityComparisonCase> PredCases;
1175 BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases);
1176
1177 // Based on whether the default edge from PTI goes to BB or not, fill in
1178 // PredCases and PredDefault with the new switch cases we would like to
1179 // build.
1181
1182 // Update the branch weight metadata along the way
1184 bool PredHasWeights = hasBranchWeightMD(*PTI);
1185 bool SuccHasWeights = hasBranchWeightMD(*TI);
1186
1187 if (PredHasWeights) {
1188 GetBranchWeights(PTI, Weights);
1189 // branch-weight metadata is inconsistent here.
1190 if (Weights.size() != 1 + PredCases.size())
1191 PredHasWeights = SuccHasWeights = false;
1192 } else if (SuccHasWeights)
1193 // If there are no predecessor weights but there are successor weights,
1194 // populate Weights with 1, which will later be scaled to the sum of
1195 // successor's weights
1196 Weights.assign(1 + PredCases.size(), 1);
1197
1198 SmallVector<uint64_t, 8> SuccWeights;
1199 if (SuccHasWeights) {
1200 GetBranchWeights(TI, SuccWeights);
1201 // branch-weight metadata is inconsistent here.
1202 if (SuccWeights.size() != 1 + BBCases.size())
1203 PredHasWeights = SuccHasWeights = false;
1204 } else if (PredHasWeights)
1205 SuccWeights.assign(1 + BBCases.size(), 1);
1206
1207 if (PredDefault == BB) {
1208 // If this is the default destination from PTI, only the edges in TI
1209 // that don't occur in PTI, or that branch to BB will be activated.
1210 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1211 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1212 if (PredCases[i].Dest != BB)
1213 PTIHandled.insert(PredCases[i].Value);
1214 else {
1215 // The default destination is BB, we don't need explicit targets.
1216 std::swap(PredCases[i], PredCases.back());
1217
1218 if (PredHasWeights || SuccHasWeights) {
1219 // Increase weight for the default case.
1220 Weights[0] += Weights[i + 1];
1221 std::swap(Weights[i + 1], Weights.back());
1222 Weights.pop_back();
1223 }
1224
1225 PredCases.pop_back();
1226 --i;
1227 --e;
1228 }
1229
1230 // Reconstruct the new switch statement we will be building.
1231 if (PredDefault != BBDefault) {
1232 PredDefault->removePredecessor(Pred);
1233 if (DTU && PredDefault != BB)
1234 Updates.push_back({DominatorTree::Delete, Pred, PredDefault});
1235 PredDefault = BBDefault;
1236 ++NewSuccessors[BBDefault];
1237 }
1238
1239 unsigned CasesFromPred = Weights.size();
1240 uint64_t ValidTotalSuccWeight = 0;
1241 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1242 if (!PTIHandled.count(BBCases[i].Value) && BBCases[i].Dest != BBDefault) {
1243 PredCases.push_back(BBCases[i]);
1244 ++NewSuccessors[BBCases[i].Dest];
1245 if (SuccHasWeights || PredHasWeights) {
1246 // The default weight is at index 0, so weight for the ith case
1247 // should be at index i+1. Scale the cases from successor by
1248 // PredDefaultWeight (Weights[0]).
1249 Weights.push_back(Weights[0] * SuccWeights[i + 1]);
1250 ValidTotalSuccWeight += SuccWeights[i + 1];
1251 }
1252 }
1253
1254 if (SuccHasWeights || PredHasWeights) {
1255 ValidTotalSuccWeight += SuccWeights[0];
1256 // Scale the cases from predecessor by ValidTotalSuccWeight.
1257 for (unsigned i = 1; i < CasesFromPred; ++i)
1258 Weights[i] *= ValidTotalSuccWeight;
1259 // Scale the default weight by SuccDefaultWeight (SuccWeights[0]).
1260 Weights[0] *= SuccWeights[0];
1261 }
1262 } else {
1263 // If this is not the default destination from PSI, only the edges
1264 // in SI that occur in PSI with a destination of BB will be
1265 // activated.
1266 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1267 std::map<ConstantInt *, uint64_t> WeightsForHandled;
1268 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1269 if (PredCases[i].Dest == BB) {
1270 PTIHandled.insert(PredCases[i].Value);
1271
1272 if (PredHasWeights || SuccHasWeights) {
1273 WeightsForHandled[PredCases[i].Value] = Weights[i + 1];
1274 std::swap(Weights[i + 1], Weights.back());
1275 Weights.pop_back();
1276 }
1277
1278 std::swap(PredCases[i], PredCases.back());
1279 PredCases.pop_back();
1280 --i;
1281 --e;
1282 }
1283
1284 // Okay, now we know which constants were sent to BB from the
1285 // predecessor. Figure out where they will all go now.
1286 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1287 if (PTIHandled.count(BBCases[i].Value)) {
1288 // If this is one we are capable of getting...
1289 if (PredHasWeights || SuccHasWeights)
1290 Weights.push_back(WeightsForHandled[BBCases[i].Value]);
1291 PredCases.push_back(BBCases[i]);
1292 ++NewSuccessors[BBCases[i].Dest];
1293 PTIHandled.erase(BBCases[i].Value); // This constant is taken care of
1294 }
1295
1296 // If there are any constants vectored to BB that TI doesn't handle,
1297 // they must go to the default destination of TI.
1298 for (ConstantInt *I : PTIHandled) {
1299 if (PredHasWeights || SuccHasWeights)
1300 Weights.push_back(WeightsForHandled[I]);
1301 PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault));
1302 ++NewSuccessors[BBDefault];
1303 }
1304 }
1305
1306 // Okay, at this point, we know which new successor Pred will get. Make
1307 // sure we update the number of entries in the PHI nodes for these
1308 // successors.
1309 SmallPtrSet<BasicBlock *, 2> SuccsOfPred;
1310 if (DTU) {
1311 SuccsOfPred = {succ_begin(Pred), succ_end(Pred)};
1312 Updates.reserve(Updates.size() + NewSuccessors.size());
1313 }
1314 for (const std::pair<BasicBlock *, int /*Num*/> &NewSuccessor :
1315 NewSuccessors) {
1316 for (auto I : seq(NewSuccessor.second)) {
1317 (void)I;
1318 AddPredecessorToBlock(NewSuccessor.first, Pred, BB);
1319 }
1320 if (DTU && !SuccsOfPred.contains(NewSuccessor.first))
1321 Updates.push_back({DominatorTree::Insert, Pred, NewSuccessor.first});
1322 }
1323
1324 Builder.SetInsertPoint(PTI);
1325 // Convert pointer to int before we switch.
1326 if (CV->getType()->isPointerTy()) {
1327 CV =
1328 Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()), "magicptr");
1329 }
1330
1331 // Now that the successors are updated, create the new Switch instruction.
1332 SwitchInst *NewSI = Builder.CreateSwitch(CV, PredDefault, PredCases.size());
1333 NewSI->setDebugLoc(PTI->getDebugLoc());
1334 for (ValueEqualityComparisonCase &V : PredCases)
1335 NewSI->addCase(V.Value, V.Dest);
1336
1337 if (PredHasWeights || SuccHasWeights) {
1338 // Halve the weights if any of them cannot fit in an uint32_t
1339 FitWeights(Weights);
1340
1341 SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
1342
1343 setBranchWeights(NewSI, MDWeights, /*IsExpected=*/false);
1344 }
1345
1347
1348 // Okay, last check. If BB is still a successor of PSI, then we must
1349 // have an infinite loop case. If so, add an infinitely looping block
1350 // to handle the case to preserve the behavior of the code.
1351 BasicBlock *InfLoopBlock = nullptr;
1352 for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i)
1353 if (NewSI->getSuccessor(i) == BB) {
1354 if (!InfLoopBlock) {
1355 // Insert it at the end of the function, because it's either code,
1356 // or it won't matter if it's hot. :)
1357 InfLoopBlock =
1358 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
1359 BranchInst::Create(InfLoopBlock, InfLoopBlock);
1360 if (DTU)
1361 Updates.push_back(
1362 {DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
1363 }
1364 NewSI->setSuccessor(i, InfLoopBlock);
1365 }
1366
1367 if (DTU) {
1368 if (InfLoopBlock)
1369 Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock});
1370
1371 Updates.push_back({DominatorTree::Delete, Pred, BB});
1372
1373 DTU->applyUpdates(Updates);
1374 }
1375
1376 ++NumFoldValueComparisonIntoPredecessors;
1377 return true;
1378}
1379
1380/// The specified terminator is a value equality comparison instruction
1381/// (either a switch or a branch on "X == c").
1382/// See if any of the predecessors of the terminator block are value comparisons
1383/// on the same value. If so, and if safe to do so, fold them together.
1384bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(Instruction *TI,
1385 IRBuilder<> &Builder) {
1386 BasicBlock *BB = TI->getParent();
1387 Value *CV = isValueEqualityComparison(TI); // CondVal
1388 assert(CV && "Not a comparison?");
1389
1390 bool Changed = false;
1391
1393 while (!Preds.empty()) {
1394 BasicBlock *Pred = Preds.pop_back_val();
1395 Instruction *PTI = Pred->getTerminator();
1396
1397 // Don't try to fold into itself.
1398 if (Pred == BB)
1399 continue;
1400
1401 // See if the predecessor is a comparison with the same value.
1402 Value *PCV = isValueEqualityComparison(PTI); // PredCondVal
1403 if (PCV != CV)
1404 continue;
1405
1407 if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) {
1408 for (auto *Succ : FailBlocks) {
1409 if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split", DTU))
1410 return false;
1411 }
1412 }
1413
1414 PerformValueComparisonIntoPredecessorFolding(TI, CV, PTI, Builder);
1415 Changed = true;
1416 }
1417 return Changed;
1418}
1419
1420// If we would need to insert a select that uses the value of this invoke
1421// (comments in hoistSuccIdenticalTerminatorToSwitchOrIf explain why we would
1422// need to do this), we can't hoist the invoke, as there is nowhere to put the
1423// select in this case.
1425 Instruction *I1, Instruction *I2) {
1426 for (BasicBlock *Succ : successors(BB1)) {
1427 for (const PHINode &PN : Succ->phis()) {
1428 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1429 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1430 if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) {
1431 return false;
1432 }
1433 }
1434 }
1435 return true;
1436}
1437
1438// Get interesting characteristics of instructions that
1439// `hoistCommonCodeFromSuccessors` didn't hoist. They restrict what kind of
1440// instructions can be reordered across.
1446
1448 unsigned Flags = 0;
1449 if (I->mayReadFromMemory())
1450 Flags |= SkipReadMem;
1451 // We can't arbitrarily move around allocas, e.g. moving allocas (especially
1452 // inalloca) across stacksave/stackrestore boundaries.
1453 if (I->mayHaveSideEffects() || isa<AllocaInst>(I))
1454 Flags |= SkipSideEffect;
1456 Flags |= SkipImplicitControlFlow;
1457 return Flags;
1458}
1459
1460// Returns true if it is safe to reorder an instruction across preceding
1461// instructions in a basic block.
1462static bool isSafeToHoistInstr(Instruction *I, unsigned Flags) {
1463 // Don't reorder a store over a load.
1464 if ((Flags & SkipReadMem) && I->mayWriteToMemory())
1465 return false;
1466
1467 // If we have seen an instruction with side effects, it's unsafe to reorder an
1468 // instruction which reads memory or itself has side effects.
1469 if ((Flags & SkipSideEffect) &&
1470 (I->mayReadFromMemory() || I->mayHaveSideEffects() || isa<AllocaInst>(I)))
1471 return false;
1472
1473 // Reordering across an instruction which does not necessarily transfer
1474 // control to the next instruction is speculation.
1476 return false;
1477
1478 // Hoisting of llvm.deoptimize is only legal together with the next return
1479 // instruction, which this pass is not always able to do.
1480 if (auto *CB = dyn_cast<CallBase>(I))
1481 if (CB->getIntrinsicID() == Intrinsic::experimental_deoptimize)
1482 return false;
1483
1484 // It's also unsafe/illegal to hoist an instruction above its instruction
1485 // operands
1486 BasicBlock *BB = I->getParent();
1487 for (Value *Op : I->operands()) {
1488 if (auto *J = dyn_cast<Instruction>(Op))
1489 if (J->getParent() == BB)
1490 return false;
1491 }
1492
1493 return true;
1494}
1495
1496static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified = false);
1497
1498/// Helper function for hoistCommonCodeFromSuccessors. Return true if identical
1499/// instructions \p I1 and \p I2 can and should be hoisted.
1501 const TargetTransformInfo &TTI) {
1502 // If we're going to hoist a call, make sure that the two instructions
1503 // we're commoning/hoisting are both marked with musttail, or neither of
1504 // them is marked as such. Otherwise, we might end up in a situation where
1505 // we hoist from a block where the terminator is a `ret` to a block where
1506 // the terminator is a `br`, and `musttail` calls expect to be followed by
1507 // a return.
1508 auto *C1 = dyn_cast<CallInst>(I1);
1509 auto *C2 = dyn_cast<CallInst>(I2);
1510 if (C1 && C2)
1511 if (C1->isMustTailCall() != C2->isMustTailCall())
1512 return false;
1513
1515 return false;
1516
1517 // If any of the two call sites has nomerge or convergent attribute, stop
1518 // hoisting.
1519 if (const auto *CB1 = dyn_cast<CallBase>(I1))
1520 if (CB1->cannotMerge() || CB1->isConvergent())
1521 return false;
1522 if (const auto *CB2 = dyn_cast<CallBase>(I2))
1523 if (CB2->cannotMerge() || CB2->isConvergent())
1524 return false;
1525
1526 return true;
1527}
1528
1529/// Hoists DbgVariableRecords from \p I1 and \p OtherInstrs that are identical
1530/// in lock-step to \p TI. This matches how dbg.* intrinsics are hoisting in
1531/// hoistCommonCodeFromSuccessors. e.g. The input:
1532/// I1 DVRs: { x, z },
1533/// OtherInsts: { I2 DVRs: { x, y, z } }
1534/// would result in hoisting only DbgVariableRecord x.
1536 Instruction *TI, Instruction *I1,
1537 SmallVectorImpl<Instruction *> &OtherInsts) {
1538 if (!I1->hasDbgRecords())
1539 return;
1540 using CurrentAndEndIt =
1541 std::pair<DbgRecord::self_iterator, DbgRecord::self_iterator>;
1542 // Vector of {Current, End} iterators.
1544 Itrs.reserve(OtherInsts.size() + 1);
1545 // Helper lambdas for lock-step checks:
1546 // Return true if this Current == End.
1547 auto atEnd = [](const CurrentAndEndIt &Pair) {
1548 return Pair.first == Pair.second;
1549 };
1550 // Return true if all Current are identical.
1551 auto allIdentical = [](const SmallVector<CurrentAndEndIt> &Itrs) {
1552 return all_of(make_first_range(ArrayRef(Itrs).drop_front()),
1554 return Itrs[0].first->isIdenticalToWhenDefined(*I);
1555 });
1556 };
1557
1558 // Collect the iterators.
1559 Itrs.push_back(
1560 {I1->getDbgRecordRange().begin(), I1->getDbgRecordRange().end()});
1561 for (Instruction *Other : OtherInsts) {
1562 if (!Other->hasDbgRecords())
1563 return;
1564 Itrs.push_back(
1565 {Other->getDbgRecordRange().begin(), Other->getDbgRecordRange().end()});
1566 }
1567
1568 // Iterate in lock-step until any of the DbgRecord lists are exausted. If
1569 // the lock-step DbgRecord are identical, hoist all of them to TI.
1570 // This replicates the dbg.* intrinsic behaviour in
1571 // hoistCommonCodeFromSuccessors.
1572 while (none_of(Itrs, atEnd)) {
1573 bool HoistDVRs = allIdentical(Itrs);
1574 for (CurrentAndEndIt &Pair : Itrs) {
1575 // Increment Current iterator now as we may be about to move the
1576 // DbgRecord.
1577 DbgRecord &DR = *Pair.first++;
1578 if (HoistDVRs) {
1579 DR.removeFromParent();
1580 TI->getParent()->insertDbgRecordBefore(&DR, TI->getIterator());
1581 }
1582 }
1583 }
1584}
1585
1586/// Hoist any common code in the successor blocks up into the block. This
1587/// function guarantees that BB dominates all successors. If EqTermsOnly is
1588/// given, only perform hoisting in case both blocks only contain a terminator.
1589/// In that case, only the original BI will be replaced and selects for PHIs are
1590/// added.
1591bool SimplifyCFGOpt::hoistCommonCodeFromSuccessors(BasicBlock *BB,
1592 bool EqTermsOnly) {
1593 // This does very trivial matching, with limited scanning, to find identical
1594 // instructions in the two blocks. In particular, we don't want to get into
1595 // O(N1*N2*...) situations here where Ni are the sizes of these successors. As
1596 // such, we currently just scan for obviously identical instructions in an
1597 // identical order, possibly separated by the same number of non-identical
1598 // instructions.
1599 unsigned int SuccSize = succ_size(BB);
1600 if (SuccSize < 2)
1601 return false;
1602
1603 // If either of the blocks has it's address taken, then we can't do this fold,
1604 // because the code we'd hoist would no longer run when we jump into the block
1605 // by it's address.
1606 for (auto *Succ : successors(BB))
1607 if (Succ->hasAddressTaken() || !Succ->getSinglePredecessor())
1608 return false;
1609
1610 auto *TI = BB->getTerminator();
1611
1612 // The second of pair is a SkipFlags bitmask.
1613 using SuccIterPair = std::pair<BasicBlock::iterator, unsigned>;
1614 SmallVector<SuccIterPair, 8> SuccIterPairs;
1615 for (auto *Succ : successors(BB)) {
1616 BasicBlock::iterator SuccItr = Succ->begin();
1617 if (isa<PHINode>(*SuccItr))
1618 return false;
1619 SuccIterPairs.push_back(SuccIterPair(SuccItr, 0));
1620 }
1621
1622 // Check if only hoisting terminators is allowed. This does not add new
1623 // instructions to the hoist location.
1624 if (EqTermsOnly) {
1625 // Skip any debug intrinsics, as they are free to hoist.
1626 for (auto &SuccIter : make_first_range(SuccIterPairs)) {
1627 auto *INonDbg = &*skipDebugIntrinsics(SuccIter);
1628 if (!INonDbg->isTerminator())
1629 return false;
1630 }
1631 // Now we know that we only need to hoist debug intrinsics and the
1632 // terminator. Let the loop below handle those 2 cases.
1633 }
1634
1635 // Count how many instructions were not hoisted so far. There's a limit on how
1636 // many instructions we skip, serving as a compilation time control as well as
1637 // preventing excessive increase of life ranges.
1638 unsigned NumSkipped = 0;
1639 // If we find an unreachable instruction at the beginning of a basic block, we
1640 // can still hoist instructions from the rest of the basic blocks.
1641 if (SuccIterPairs.size() > 2) {
1642 erase_if(SuccIterPairs,
1643 [](const auto &Pair) { return isa<UnreachableInst>(Pair.first); });
1644 if (SuccIterPairs.size() < 2)
1645 return false;
1646 }
1647
1648 bool Changed = false;
1649
1650 for (;;) {
1651 auto *SuccIterPairBegin = SuccIterPairs.begin();
1652 auto &BB1ItrPair = *SuccIterPairBegin++;
1653 auto OtherSuccIterPairRange =
1654 iterator_range(SuccIterPairBegin, SuccIterPairs.end());
1655 auto OtherSuccIterRange = make_first_range(OtherSuccIterPairRange);
1656
1657 Instruction *I1 = &*BB1ItrPair.first;
1658
1659 // Skip debug info if it is not identical.
1660 bool AllDbgInstsAreIdentical = all_of(OtherSuccIterRange, [I1](auto &Iter) {
1661 Instruction *I2 = &*Iter;
1662 return I1->isIdenticalToWhenDefined(I2);
1663 });
1664 if (!AllDbgInstsAreIdentical) {
1665 while (isa<DbgInfoIntrinsic>(I1))
1666 I1 = &*++BB1ItrPair.first;
1667 for (auto &SuccIter : OtherSuccIterRange) {
1668 Instruction *I2 = &*SuccIter;
1669 while (isa<DbgInfoIntrinsic>(I2))
1670 I2 = &*++SuccIter;
1671 }
1672 }
1673
1674 bool AllInstsAreIdentical = true;
1675 bool HasTerminator = I1->isTerminator();
1676 for (auto &SuccIter : OtherSuccIterRange) {
1677 Instruction *I2 = &*SuccIter;
1678 HasTerminator |= I2->isTerminator();
1679 if (AllInstsAreIdentical && (!I1->isIdenticalToWhenDefined(I2) ||
1680 MMRAMetadata(*I1) != MMRAMetadata(*I2)))
1681 AllInstsAreIdentical = false;
1682 }
1683
1685 for (auto &SuccIter : OtherSuccIterRange)
1686 OtherInsts.push_back(&*SuccIter);
1687
1688 // If we are hoisting the terminator instruction, don't move one (making a
1689 // broken BB), instead clone it, and remove BI.
1690 if (HasTerminator) {
1691 // Even if BB, which contains only one unreachable instruction, is ignored
1692 // at the beginning of the loop, we can hoist the terminator instruction.
1693 // If any instructions remain in the block, we cannot hoist terminators.
1694 if (NumSkipped || !AllInstsAreIdentical) {
1695 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
1696 return Changed;
1697 }
1698
1699 return hoistSuccIdenticalTerminatorToSwitchOrIf(TI, I1, OtherInsts) ||
1700 Changed;
1701 }
1702
1703 if (AllInstsAreIdentical) {
1704 unsigned SkipFlagsBB1 = BB1ItrPair.second;
1705 AllInstsAreIdentical =
1706 isSafeToHoistInstr(I1, SkipFlagsBB1) &&
1707 all_of(OtherSuccIterPairRange, [=](const auto &Pair) {
1708 Instruction *I2 = &*Pair.first;
1709 unsigned SkipFlagsBB2 = Pair.second;
1710 // Even if the instructions are identical, it may not
1711 // be safe to hoist them if we have skipped over
1712 // instructions with side effects or their operands
1713 // weren't hoisted.
1714 return isSafeToHoistInstr(I2, SkipFlagsBB2) &&
1716 });
1717 }
1718
1719 if (AllInstsAreIdentical) {
1720 BB1ItrPair.first++;
1721 if (isa<DbgInfoIntrinsic>(I1)) {
1722 // The debug location is an integral part of a debug info intrinsic
1723 // and can't be separated from it or replaced. Instead of attempting
1724 // to merge locations, simply hoist both copies of the intrinsic.
1725 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
1726 // We've just hoisted DbgVariableRecords; move I1 after them (before TI)
1727 // and leave any that were not hoisted behind (by calling moveBefore
1728 // rather than moveBeforePreserving).
1729 I1->moveBefore(TI);
1730 for (auto &SuccIter : OtherSuccIterRange) {
1731 auto *I2 = &*SuccIter++;
1732 assert(isa<DbgInfoIntrinsic>(I2));
1733 I2->moveBefore(TI);
1734 }
1735 } else {
1736 // For a normal instruction, we just move one to right before the
1737 // branch, then replace all uses of the other with the first. Finally,
1738 // we remove the now redundant second instruction.
1739 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
1740 // We've just hoisted DbgVariableRecords; move I1 after them (before TI)
1741 // and leave any that were not hoisted behind (by calling moveBefore
1742 // rather than moveBeforePreserving).
1743 I1->moveBefore(TI);
1744 for (auto &SuccIter : OtherSuccIterRange) {
1745 Instruction *I2 = &*SuccIter++;
1746 assert(I2 != I1);
1747 if (!I2->use_empty())
1748 I2->replaceAllUsesWith(I1);
1749 I1->andIRFlags(I2);
1750 combineMetadataForCSE(I1, I2, true);
1751 // I1 and I2 are being combined into a single instruction. Its debug
1752 // location is the merged locations of the original instructions.
1753 I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1754 I2->eraseFromParent();
1755 }
1756 }
1757 if (!Changed)
1758 NumHoistCommonCode += SuccIterPairs.size();
1759 Changed = true;
1760 NumHoistCommonInstrs += SuccIterPairs.size();
1761 } else {
1762 if (NumSkipped >= HoistCommonSkipLimit) {
1763 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
1764 return Changed;
1765 }
1766 // We are about to skip over a pair of non-identical instructions. Record
1767 // if any have characteristics that would prevent reordering instructions
1768 // across them.
1769 for (auto &SuccIterPair : SuccIterPairs) {
1770 Instruction *I = &*SuccIterPair.first++;
1771 SuccIterPair.second |= skippedInstrFlags(I);
1772 }
1773 ++NumSkipped;
1774 }
1775 }
1776}
1777
1778bool SimplifyCFGOpt::hoistSuccIdenticalTerminatorToSwitchOrIf(
1779 Instruction *TI, Instruction *I1,
1780 SmallVectorImpl<Instruction *> &OtherSuccTIs) {
1781
1782 auto *BI = dyn_cast<BranchInst>(TI);
1783
1784 bool Changed = false;
1785 BasicBlock *TIParent = TI->getParent();
1786 BasicBlock *BB1 = I1->getParent();
1787
1788 // Use only for an if statement.
1789 auto *I2 = *OtherSuccTIs.begin();
1790 auto *BB2 = I2->getParent();
1791 if (BI) {
1792 assert(OtherSuccTIs.size() == 1);
1793 assert(BI->getSuccessor(0) == I1->getParent());
1794 assert(BI->getSuccessor(1) == I2->getParent());
1795 }
1796
1797 // In the case of an if statement, we try to hoist an invoke.
1798 // FIXME: Can we define a safety predicate for CallBr?
1799 // FIXME: Test case llvm/test/Transforms/SimplifyCFG/2009-06-15-InvokeCrash.ll
1800 // removed in 4c923b3b3fd0ac1edebf0603265ca3ba51724937 commit?
1801 if (isa<InvokeInst>(I1) && (!BI || !isSafeToHoistInvoke(BB1, BB2, I1, I2)))
1802 return false;
1803
1804 // TODO: callbr hoisting currently disabled pending further study.
1805 if (isa<CallBrInst>(I1))
1806 return false;
1807
1808 for (BasicBlock *Succ : successors(BB1)) {
1809 for (PHINode &PN : Succ->phis()) {
1810 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1811 for (Instruction *OtherSuccTI : OtherSuccTIs) {
1812 Value *BB2V = PN.getIncomingValueForBlock(OtherSuccTI->getParent());
1813 if (BB1V == BB2V)
1814 continue;
1815
1816 // In the case of an if statement, check for
1817 // passingValueIsAlwaysUndefined here because we would rather eliminate
1818 // undefined control flow then converting it to a select.
1819 if (!BI || passingValueIsAlwaysUndefined(BB1V, &PN) ||
1821 return false;
1822 }
1823 }
1824 }
1825
1826 // Hoist DbgVariableRecords attached to the terminator to match dbg.*
1827 // intrinsic hoisting behaviour in hoistCommonCodeFromSuccessors.
1828 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherSuccTIs);
1829 // Clone the terminator and hoist it into the pred, without any debug info.
1830 Instruction *NT = I1->clone();
1831 NT->insertInto(TIParent, TI->getIterator());
1832 if (!NT->getType()->isVoidTy()) {
1833 I1->replaceAllUsesWith(NT);
1834 for (Instruction *OtherSuccTI : OtherSuccTIs)
1835 OtherSuccTI->replaceAllUsesWith(NT);
1836 NT->takeName(I1);
1837 }
1838 Changed = true;
1839 NumHoistCommonInstrs += OtherSuccTIs.size() + 1;
1840
1841 // Ensure terminator gets a debug location, even an unknown one, in case
1842 // it involves inlinable calls.
1844 Locs.push_back(I1->getDebugLoc());
1845 for (auto *OtherSuccTI : OtherSuccTIs)
1846 Locs.push_back(OtherSuccTI->getDebugLoc());
1847 NT->setDebugLoc(DILocation::getMergedLocations(Locs));
1848
1849 // PHIs created below will adopt NT's merged DebugLoc.
1850 IRBuilder<NoFolder> Builder(NT);
1851
1852 // In the case of an if statement, hoisting one of the terminators from our
1853 // successor is a great thing. Unfortunately, the successors of the if/else
1854 // blocks may have PHI nodes in them. If they do, all PHI entries for BB1/BB2
1855 // must agree for all PHI nodes, so we insert select instruction to compute
1856 // the final result.
1857 if (BI) {
1858 std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects;
1859 for (BasicBlock *Succ : successors(BB1)) {
1860 for (PHINode &PN : Succ->phis()) {
1861 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1862 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1863 if (BB1V == BB2V)
1864 continue;
1865
1866 // These values do not agree. Insert a select instruction before NT
1867 // that determines the right value.
1868 SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)];
1869 if (!SI) {
1870 // Propagate fast-math-flags from phi node to its replacement select.
1871 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
1872 if (isa<FPMathOperator>(PN))
1873 Builder.setFastMathFlags(PN.getFastMathFlags());
1874
1875 SI = cast<SelectInst>(Builder.CreateSelect(
1876 BI->getCondition(), BB1V, BB2V,
1877 BB1V->getName() + "." + BB2V->getName(), BI));
1878 }
1879
1880 // Make the PHI node use the select for all incoming values for BB1/BB2
1881 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
1882 if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2)
1883 PN.setIncomingValue(i, SI);
1884 }
1885 }
1886 }
1887
1889
1890 // Update any PHI nodes in our new successors.
1891 for (BasicBlock *Succ : successors(BB1)) {
1892 AddPredecessorToBlock(Succ, TIParent, BB1);
1893 if (DTU)
1894 Updates.push_back({DominatorTree::Insert, TIParent, Succ});
1895 }
1896
1897 if (DTU)
1898 for (BasicBlock *Succ : successors(TI))
1899 Updates.push_back({DominatorTree::Delete, TIParent, Succ});
1900
1902 if (DTU)
1903 DTU->applyUpdates(Updates);
1904 return Changed;
1905}
1906
1907// Check lifetime markers.
1908static bool isLifeTimeMarker(const Instruction *I) {
1909 if (auto II = dyn_cast<IntrinsicInst>(I)) {
1910 switch (II->getIntrinsicID()) {
1911 default:
1912 break;
1913 case Intrinsic::lifetime_start:
1914 case Intrinsic::lifetime_end:
1915 return true;
1916 }
1917 }
1918 return false;
1919}
1920
1921// TODO: Refine this. This should avoid cases like turning constant memcpy sizes
1922// into variables.
1924 int OpIdx) {
1925 return !isa<IntrinsicInst>(I);
1926}
1927
1928// All instructions in Insts belong to different blocks that all unconditionally
1929// branch to a common successor. Analyze each instruction and return true if it
1930// would be possible to sink them into their successor, creating one common
1931// instruction instead. For every value that would be required to be provided by
1932// PHI node (because an operand varies in each input block), add to PHIOperands.
1935 DenseMap<const Use *, SmallVector<Value *, 4>> &PHIOperands) {
1936 // Prune out obviously bad instructions to move. Each instruction must have
1937 // exactly zero or one use, and we check later that use is by a single, common
1938 // PHI instruction in the successor.
1939 bool HasUse = !Insts.front()->user_empty();
1940 for (auto *I : Insts) {
1941 // These instructions may change or break semantics if moved.
1942 if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) ||
1943 I->getType()->isTokenTy())
1944 return false;
1945
1946 // Do not try to sink an instruction in an infinite loop - it can cause
1947 // this algorithm to infinite loop.
1948 if (I->getParent()->getSingleSuccessor() == I->getParent())
1949 return false;
1950
1951 // Conservatively return false if I is an inline-asm instruction. Sinking
1952 // and merging inline-asm instructions can potentially create arguments
1953 // that cannot satisfy the inline-asm constraints.
1954 // If the instruction has nomerge or convergent attribute, return false.
1955 if (const auto *C = dyn_cast<CallBase>(I))
1956 if (C->isInlineAsm() || C->cannotMerge() || C->isConvergent())
1957 return false;
1958
1959 // Each instruction must have zero or one use.
1960 if (HasUse && !I->hasOneUse())
1961 return false;
1962 if (!HasUse && !I->user_empty())
1963 return false;
1964 }
1965
1966 const Instruction *I0 = Insts.front();
1967 const auto I0MMRA = MMRAMetadata(*I0);
1968 for (auto *I : Insts) {
1969 if (!I->isSameOperationAs(I0))
1970 return false;
1971
1972 // swifterror pointers can only be used by a load or store; sinking a load
1973 // or store would require introducing a select for the pointer operand,
1974 // which isn't allowed for swifterror pointers.
1975 if (isa<StoreInst>(I) && I->getOperand(1)->isSwiftError())
1976 return false;
1977 if (isa<LoadInst>(I) && I->getOperand(0)->isSwiftError())
1978 return false;
1979
1980 // Treat MMRAs conservatively. This pass can be quite aggressive and
1981 // could drop a lot of MMRAs otherwise.
1982 if (MMRAMetadata(*I) != I0MMRA)
1983 return false;
1984 }
1985
1986 // Uses must be consistent: If I0 is used in a phi node in the sink target,
1987 // then the other phi operands must match the instructions from Insts. This
1988 // also has to hold true for any phi nodes that would be created as a result
1989 // of sinking. Both of these cases are represented by PhiOperands.
1990 if (HasUse) {
1991 const Use &U = *I0->use_begin();
1992 auto It = PHIOperands.find(&U);
1993 if (It == PHIOperands.end())
1994 // There may be uses in other blocks when sinking into a loop header.
1995 return false;
1996 if (!equal(Insts, It->second))
1997 return false;
1998 }
1999
2000 // Because SROA can't handle speculating stores of selects, try not to sink
2001 // loads, stores or lifetime markers of allocas when we'd have to create a
2002 // PHI for the address operand. Also, because it is likely that loads or
2003 // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink
2004 // them.
2005 // This can cause code churn which can have unintended consequences down
2006 // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244.
2007 // FIXME: This is a workaround for a deficiency in SROA - see
2008 // https://llvm.org/bugs/show_bug.cgi?id=30188
2009 if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) {
2010 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
2011 }))
2012 return false;
2013 if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) {
2014 return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
2015 }))
2016 return false;
2017 if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) {
2018 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
2019 }))
2020 return false;
2021
2022 // For calls to be sinkable, they must all be indirect, or have same callee.
2023 // I.e. if we have two direct calls to different callees, we don't want to
2024 // turn that into an indirect call. Likewise, if we have an indirect call,
2025 // and a direct call, we don't actually want to have a single indirect call.
2026 if (isa<CallBase>(I0)) {
2027 auto IsIndirectCall = [](const Instruction *I) {
2028 return cast<CallBase>(I)->isIndirectCall();
2029 };
2030 bool HaveIndirectCalls = any_of(Insts, IsIndirectCall);
2031 bool AllCallsAreIndirect = all_of(Insts, IsIndirectCall);
2032 if (HaveIndirectCalls) {
2033 if (!AllCallsAreIndirect)
2034 return false;
2035 } else {
2036 // All callees must be identical.
2037 Value *Callee = nullptr;
2038 for (const Instruction *I : Insts) {
2039 Value *CurrCallee = cast<CallBase>(I)->getCalledOperand();
2040 if (!Callee)
2041 Callee = CurrCallee;
2042 else if (Callee != CurrCallee)
2043 return false;
2044 }
2045 }
2046 }
2047
2048 for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) {
2049 Value *Op = I0->getOperand(OI);
2050 if (Op->getType()->isTokenTy())
2051 // Don't touch any operand of token type.
2052 return false;
2053
2054 auto SameAsI0 = [&I0, OI](const Instruction *I) {
2055 assert(I->getNumOperands() == I0->getNumOperands());
2056 return I->getOperand(OI) == I0->getOperand(OI);
2057 };
2058 if (!all_of(Insts, SameAsI0)) {
2059 if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) ||
2061 // We can't create a PHI from this GEP.
2062 return false;
2063 auto &Ops = PHIOperands[&I0->getOperandUse(OI)];
2064 for (auto *I : Insts)
2065 Ops.push_back(I->getOperand(OI));
2066 }
2067 }
2068 return true;
2069}
2070
2071// Assuming canSinkInstructions(Blocks) has returned true, sink the last
2072// instruction of every block in Blocks to their common successor, commoning
2073// into one instruction.
2075 auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0);
2076
2077 // canSinkInstructions returning true guarantees that every block has at
2078 // least one non-terminator instruction.
2080 for (auto *BB : Blocks) {
2081 Instruction *I = BB->getTerminator();
2082 do {
2083 I = I->getPrevNode();
2084 } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front());
2085 if (!isa<DbgInfoIntrinsic>(I))
2086 Insts.push_back(I);
2087 }
2088
2089 // We don't need to do any more checking here; canSinkInstructions should
2090 // have done it all for us.
2091 SmallVector<Value*, 4> NewOperands;
2092 Instruction *I0 = Insts.front();
2093 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) {
2094 // This check is different to that in canSinkInstructions. There, we
2095 // cared about the global view once simplifycfg (and instcombine) have
2096 // completed - it takes into account PHIs that become trivially
2097 // simplifiable. However here we need a more local view; if an operand
2098 // differs we create a PHI and rely on instcombine to clean up the very
2099 // small mess we may make.
2100 bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) {
2101 return I->getOperand(O) != I0->getOperand(O);
2102 });
2103 if (!NeedPHI) {
2104 NewOperands.push_back(I0->getOperand(O));
2105 continue;
2106 }
2107
2108 // Create a new PHI in the successor block and populate it.
2109 auto *Op = I0->getOperand(O);
2110 assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!");
2111 auto *PN =
2112 PHINode::Create(Op->getType(), Insts.size(), Op->getName() + ".sink");
2113 PN->insertBefore(BBEnd->begin());
2114 for (auto *I : Insts)
2115 PN->addIncoming(I->getOperand(O), I->getParent());
2116 NewOperands.push_back(PN);
2117 }
2118
2119 // Arbitrarily use I0 as the new "common" instruction; remap its operands
2120 // and move it to the start of the successor block.
2121 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O)
2122 I0->getOperandUse(O).set(NewOperands[O]);
2123
2124 I0->moveBefore(*BBEnd, BBEnd->getFirstInsertionPt());
2125
2126 // Update metadata and IR flags, and merge debug locations.
2127 for (auto *I : Insts)
2128 if (I != I0) {
2129 // The debug location for the "common" instruction is the merged locations
2130 // of all the commoned instructions. We start with the original location
2131 // of the "common" instruction and iteratively merge each location in the
2132 // loop below.
2133 // This is an N-way merge, which will be inefficient if I0 is a CallInst.
2134 // However, as N-way merge for CallInst is rare, so we use simplified API
2135 // instead of using complex API for N-way merge.
2136 I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc());
2137 combineMetadataForCSE(I0, I, true);
2138 I0->andIRFlags(I);
2139 }
2140
2141 if (!I0->user_empty()) {
2142 // canSinkLastInstruction checked that all instructions were used by
2143 // one and only one PHI node. Find that now, RAUW it to our common
2144 // instruction and nuke it.
2145 auto *PN = cast<PHINode>(*I0->user_begin());
2146 PN->replaceAllUsesWith(I0);
2147 PN->eraseFromParent();
2148 }
2149
2150 // Finally nuke all instructions apart from the common instruction.
2151 for (auto *I : Insts) {
2152 if (I == I0)
2153 continue;
2154 // The remaining uses are debug users, replace those with the common inst.
2155 // In most (all?) cases this just introduces a use-before-def.
2156 assert(I->user_empty() && "Inst unexpectedly still has non-dbg users");
2157 I->replaceAllUsesWith(I0);
2158 I->eraseFromParent();
2159 }
2160}
2161
2162namespace {
2163
2164 // LockstepReverseIterator - Iterates through instructions
2165 // in a set of blocks in reverse order from the first non-terminator.
2166 // For example (assume all blocks have size n):
2167 // LockstepReverseIterator I([B1, B2, B3]);
2168 // *I-- = [B1[n], B2[n], B3[n]];
2169 // *I-- = [B1[n-1], B2[n-1], B3[n-1]];
2170 // *I-- = [B1[n-2], B2[n-2], B3[n-2]];
2171 // ...
2172 class LockstepReverseIterator {
2175 bool Fail;
2176
2177 public:
2178 LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) : Blocks(Blocks) {
2179 reset();
2180 }
2181
2182 void reset() {
2183 Fail = false;
2184 Insts.clear();
2185 for (auto *BB : Blocks) {
2186 Instruction *Inst = BB->getTerminator();
2187 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
2188 Inst = Inst->getPrevNode();
2189 if (!Inst) {
2190 // Block wasn't big enough.
2191 Fail = true;
2192 return;
2193 }
2194 Insts.push_back(Inst);
2195 }
2196 }
2197
2198 bool isValid() const {
2199 return !Fail;
2200 }
2201
2202 void operator--() {
2203 if (Fail)
2204 return;
2205 for (auto *&Inst : Insts) {
2206 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
2207 Inst = Inst->getPrevNode();
2208 // Already at beginning of block.
2209 if (!Inst) {
2210 Fail = true;
2211 return;
2212 }
2213 }
2214 }
2215
2216 void operator++() {
2217 if (Fail)
2218 return;
2219 for (auto *&Inst : Insts) {
2220 for (Inst = Inst->getNextNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
2221 Inst = Inst->getNextNode();
2222 // Already at end of block.
2223 if (!Inst) {
2224 Fail = true;
2225 return;
2226 }
2227 }
2228 }
2229
2231 return Insts;
2232 }
2233 };
2234
2235} // end anonymous namespace
2236
2237/// Check whether BB's predecessors end with unconditional branches. If it is
2238/// true, sink any common code from the predecessors to BB.
2240 DomTreeUpdater *DTU) {
2241 // We support two situations:
2242 // (1) all incoming arcs are unconditional
2243 // (2) there are non-unconditional incoming arcs
2244 //
2245 // (2) is very common in switch defaults and
2246 // else-if patterns;
2247 //
2248 // if (a) f(1);
2249 // else if (b) f(2);
2250 //
2251 // produces:
2252 //
2253 // [if]
2254 // / \
2255 // [f(1)] [if]
2256 // | | \
2257 // | | |
2258 // | [f(2)]|
2259 // \ | /
2260 // [ end ]
2261 //
2262 // [end] has two unconditional predecessor arcs and one conditional. The
2263 // conditional refers to the implicit empty 'else' arc. This conditional
2264 // arc can also be caused by an empty default block in a switch.
2265 //
2266 // In this case, we attempt to sink code from all *unconditional* arcs.
2267 // If we can sink instructions from these arcs (determined during the scan
2268 // phase below) we insert a common successor for all unconditional arcs and
2269 // connect that to [end], to enable sinking:
2270 //
2271 // [if]
2272 // / \
2273 // [x(1)] [if]
2274 // | | \
2275 // | | \
2276 // | [x(2)] |
2277 // \ / |
2278 // [sink.split] |
2279 // \ /
2280 // [ end ]
2281 //
2282 SmallVector<BasicBlock*,4> UnconditionalPreds;
2283 bool HaveNonUnconditionalPredecessors = false;
2284 for (auto *PredBB : predecessors(BB)) {
2285 auto *PredBr = dyn_cast<BranchInst>(PredBB->getTerminator());
2286 if (PredBr && PredBr->isUnconditional())
2287 UnconditionalPreds.push_back(PredBB);
2288 else
2289 HaveNonUnconditionalPredecessors = true;
2290 }
2291 if (UnconditionalPreds.size() < 2)
2292 return false;
2293
2294 // We take a two-step approach to tail sinking. First we scan from the end of
2295 // each block upwards in lockstep. If the n'th instruction from the end of each
2296 // block can be sunk, those instructions are added to ValuesToSink and we
2297 // carry on. If we can sink an instruction but need to PHI-merge some operands
2298 // (because they're not identical in each instruction) we add these to
2299 // PHIOperands.
2300 // We prepopulate PHIOperands with the phis that already exist in BB.
2302 for (PHINode &PN : BB->phis()) {
2304 for (const Use &U : PN.incoming_values())
2305 IncomingVals.insert({PN.getIncomingBlock(U), &U});
2306 auto &Ops = PHIOperands[IncomingVals[UnconditionalPreds[0]]];
2307 for (BasicBlock *Pred : UnconditionalPreds)
2308 Ops.push_back(*IncomingVals[Pred]);
2309 }
2310
2311 int ScanIdx = 0;
2312 SmallPtrSet<Value*,4> InstructionsToSink;
2313 LockstepReverseIterator LRI(UnconditionalPreds);
2314 while (LRI.isValid() &&
2315 canSinkInstructions(*LRI, PHIOperands)) {
2316 LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0]
2317 << "\n");
2318 InstructionsToSink.insert((*LRI).begin(), (*LRI).end());
2319 ++ScanIdx;
2320 --LRI;
2321 }
2322
2323 // If no instructions can be sunk, early-return.
2324 if (ScanIdx == 0)
2325 return false;
2326
2327 bool followedByDeoptOrUnreachable = IsBlockFollowedByDeoptOrUnreachable(BB);
2328
2329 if (!followedByDeoptOrUnreachable) {
2330 // Okay, we *could* sink last ScanIdx instructions. But how many can we
2331 // actually sink before encountering instruction that is unprofitable to
2332 // sink?
2333 auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) {
2334 unsigned NumPHIInsts = 0;
2335 for (Use &U : (*LRI)[0]->operands()) {
2336 auto It = PHIOperands.find(&U);
2337 if (It != PHIOperands.end() && !all_of(It->second, [&](Value *V) {
2338 return InstructionsToSink.contains(V);
2339 })) {
2340 ++NumPHIInsts;
2341 // FIXME: this check is overly optimistic. We may end up not sinking
2342 // said instruction, due to the very same profitability check.
2343 // See @creating_too_many_phis in sink-common-code.ll.
2344 }
2345 }
2346 LLVM_DEBUG(dbgs() << "SINK: #phi insts: " << NumPHIInsts << "\n");
2347 return NumPHIInsts <= 1;
2348 };
2349
2350 // We've determined that we are going to sink last ScanIdx instructions,
2351 // and recorded them in InstructionsToSink. Now, some instructions may be
2352 // unprofitable to sink. But that determination depends on the instructions
2353 // that we are going to sink.
2354
2355 // First, forward scan: find the first instruction unprofitable to sink,
2356 // recording all the ones that are profitable to sink.
2357 // FIXME: would it be better, after we detect that not all are profitable.
2358 // to either record the profitable ones, or erase the unprofitable ones?
2359 // Maybe we need to choose (at runtime) the one that will touch least
2360 // instrs?
2361 LRI.reset();
2362 int Idx = 0;
2363 SmallPtrSet<Value *, 4> InstructionsProfitableToSink;
2364 while (Idx < ScanIdx) {
2365 if (!ProfitableToSinkInstruction(LRI)) {
2366 // Too many PHIs would be created.
2367 LLVM_DEBUG(
2368 dbgs() << "SINK: stopping here, too many PHIs would be created!\n");
2369 break;
2370 }
2371 InstructionsProfitableToSink.insert((*LRI).begin(), (*LRI).end());
2372 --LRI;
2373 ++Idx;
2374 }
2375
2376 // If no instructions can be sunk, early-return.
2377 if (Idx == 0)
2378 return false;
2379
2380 // Did we determine that (only) some instructions are unprofitable to sink?
2381 if (Idx < ScanIdx) {
2382 // Okay, some instructions are unprofitable.
2383 ScanIdx = Idx;
2384 InstructionsToSink = InstructionsProfitableToSink;
2385
2386 // But, that may make other instructions unprofitable, too.
2387 // So, do a backward scan, do any earlier instructions become
2388 // unprofitable?
2389 assert(
2390 !ProfitableToSinkInstruction(LRI) &&
2391 "We already know that the last instruction is unprofitable to sink");
2392 ++LRI;
2393 --Idx;
2394 while (Idx >= 0) {
2395 // If we detect that an instruction becomes unprofitable to sink,
2396 // all earlier instructions won't be sunk either,
2397 // so preemptively keep InstructionsProfitableToSink in sync.
2398 // FIXME: is this the most performant approach?
2399 for (auto *I : *LRI)
2400 InstructionsProfitableToSink.erase(I);
2401 if (!ProfitableToSinkInstruction(LRI)) {
2402 // Everything starting with this instruction won't be sunk.
2403 ScanIdx = Idx;
2404 InstructionsToSink = InstructionsProfitableToSink;
2405 }
2406 ++LRI;
2407 --Idx;
2408 }
2409 }
2410
2411 // If no instructions can be sunk, early-return.
2412 if (ScanIdx == 0)
2413 return false;
2414 }
2415
2416 bool Changed = false;
2417
2418 if (HaveNonUnconditionalPredecessors) {
2419 if (!followedByDeoptOrUnreachable) {
2420 // It is always legal to sink common instructions from unconditional
2421 // predecessors. However, if not all predecessors are unconditional,
2422 // this transformation might be pessimizing. So as a rule of thumb,
2423 // don't do it unless we'd sink at least one non-speculatable instruction.
2424 // See https://bugs.llvm.org/show_bug.cgi?id=30244
2425 LRI.reset();
2426 int Idx = 0;
2427 bool Profitable = false;
2428 while (Idx < ScanIdx) {
2429 if (!isSafeToSpeculativelyExecute((*LRI)[0])) {
2430 Profitable = true;
2431 break;
2432 }
2433 --LRI;
2434 ++Idx;
2435 }
2436 if (!Profitable)
2437 return false;
2438 }
2439
2440 LLVM_DEBUG(dbgs() << "SINK: Splitting edge\n");
2441 // We have a conditional edge and we're going to sink some instructions.
2442 // Insert a new block postdominating all blocks we're going to sink from.
2443 if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split", DTU))
2444 // Edges couldn't be split.
2445 return false;
2446 Changed = true;
2447 }
2448
2449 // Now that we've analyzed all potential sinking candidates, perform the
2450 // actual sink. We iteratively sink the last non-terminator of the source
2451 // blocks into their common successor unless doing so would require too
2452 // many PHI instructions to be generated (currently only one PHI is allowed
2453 // per sunk instruction).
2454 //
2455 // We can use InstructionsToSink to discount values needing PHI-merging that will
2456 // actually be sunk in a later iteration. This allows us to be more
2457 // aggressive in what we sink. This does allow a false positive where we
2458 // sink presuming a later value will also be sunk, but stop half way through
2459 // and never actually sink it which means we produce more PHIs than intended.
2460 // This is unlikely in practice though.
2461 int SinkIdx = 0;
2462 for (; SinkIdx != ScanIdx; ++SinkIdx) {
2463 LLVM_DEBUG(dbgs() << "SINK: Sink: "
2464 << *UnconditionalPreds[0]->getTerminator()->getPrevNode()
2465 << "\n");
2466
2467 // Because we've sunk every instruction in turn, the current instruction to
2468 // sink is always at index 0.
2469 LRI.reset();
2470
2471 sinkLastInstruction(UnconditionalPreds);
2472 NumSinkCommonInstrs++;
2473 Changed = true;
2474 }
2475 if (SinkIdx != 0)
2476 ++NumSinkCommonCode;
2477 return Changed;
2478}
2479
2480namespace {
2481
2482struct CompatibleSets {
2483 using SetTy = SmallVector<InvokeInst *, 2>;
2484
2486
2487 static bool shouldBelongToSameSet(ArrayRef<InvokeInst *> Invokes);
2488
2489 SetTy &getCompatibleSet(InvokeInst *II);
2490
2491 void insert(InvokeInst *II);
2492};
2493
2494CompatibleSets::SetTy &CompatibleSets::getCompatibleSet(InvokeInst *II) {
2495 // Perform a linear scan over all the existing sets, see if the new `invoke`
2496 // is compatible with any particular set. Since we know that all the `invokes`
2497 // within a set are compatible, only check the first `invoke` in each set.
2498 // WARNING: at worst, this has quadratic complexity.
2499 for (CompatibleSets::SetTy &Set : Sets) {
2500 if (CompatibleSets::shouldBelongToSameSet({Set.front(), II}))
2501 return Set;
2502 }
2503
2504 // Otherwise, we either had no sets yet, or this invoke forms a new set.
2505 return Sets.emplace_back();
2506}
2507
2508void CompatibleSets::insert(InvokeInst *II) {
2509 getCompatibleSet(II).emplace_back(II);
2510}
2511
2512bool CompatibleSets::shouldBelongToSameSet(ArrayRef<InvokeInst *> Invokes) {
2513 assert(Invokes.size() == 2 && "Always called with exactly two candidates.");
2514
2515 // Can we theoretically merge these `invoke`s?
2516 auto IsIllegalToMerge = [](InvokeInst *II) {
2517 return II->cannotMerge() || II->isInlineAsm();
2518 };
2519 if (any_of(Invokes, IsIllegalToMerge))
2520 return false;
2521
2522 // Either both `invoke`s must be direct,
2523 // or both `invoke`s must be indirect.
2524 auto IsIndirectCall = [](InvokeInst *II) { return II->isIndirectCall(); };
2525 bool HaveIndirectCalls = any_of(Invokes, IsIndirectCall);
2526 bool AllCallsAreIndirect = all_of(Invokes, IsIndirectCall);
2527 if (HaveIndirectCalls) {
2528 if (!AllCallsAreIndirect)
2529 return false;
2530 } else {
2531 // All callees must be identical.
2532 Value *Callee = nullptr;
2533 for (InvokeInst *II : Invokes) {
2534 Value *CurrCallee = II->getCalledOperand();
2535 assert(CurrCallee && "There is always a called operand.");
2536 if (!Callee)
2537 Callee = CurrCallee;
2538 else if (Callee != CurrCallee)
2539 return false;
2540 }
2541 }
2542
2543 // Either both `invoke`s must not have a normal destination,
2544 // or both `invoke`s must have a normal destination,
2545 auto HasNormalDest = [](InvokeInst *II) {
2546 return !isa<UnreachableInst>(II->getNormalDest()->getFirstNonPHIOrDbg());
2547 };
2548 if (any_of(Invokes, HasNormalDest)) {
2549 // Do not merge `invoke` that does not have a normal destination with one
2550 // that does have a normal destination, even though doing so would be legal.
2551 if (!all_of(Invokes, HasNormalDest))
2552 return false;
2553
2554 // All normal destinations must be identical.
2555 BasicBlock *NormalBB = nullptr;
2556 for (InvokeInst *II : Invokes) {
2557 BasicBlock *CurrNormalBB = II->getNormalDest();
2558 assert(CurrNormalBB && "There is always a 'continue to' basic block.");
2559 if (!NormalBB)
2560 NormalBB = CurrNormalBB;
2561 else if (NormalBB != CurrNormalBB)
2562 return false;
2563 }
2564
2565 // In the normal destination, the incoming values for these two `invoke`s
2566 // must be compatible.
2567 SmallPtrSet<Value *, 16> EquivalenceSet(Invokes.begin(), Invokes.end());
2569 NormalBB, {Invokes[0]->getParent(), Invokes[1]->getParent()},
2570 &EquivalenceSet))
2571 return false;
2572 }
2573
2574#ifndef NDEBUG
2575 // All unwind destinations must be identical.
2576 // We know that because we have started from said unwind destination.
2577 BasicBlock *UnwindBB = nullptr;
2578 for (InvokeInst *II : Invokes) {
2579 BasicBlock *CurrUnwindBB = II->getUnwindDest();
2580 assert(CurrUnwindBB && "There is always an 'unwind to' basic block.");
2581 if (!UnwindBB)
2582 UnwindBB = CurrUnwindBB;
2583 else
2584 assert(UnwindBB == CurrUnwindBB && "Unexpected unwind destination.");
2585 }
2586#endif
2587
2588 // In the unwind destination, the incoming values for these two `invoke`s
2589 // must be compatible.
2591 Invokes.front()->getUnwindDest(),
2592 {Invokes[0]->getParent(), Invokes[1]->getParent()}))
2593 return false;
2594
2595 // Ignoring arguments, these `invoke`s must be identical,
2596 // including operand bundles.
2597 const InvokeInst *II0 = Invokes.front();
2598 for (auto *II : Invokes.drop_front())
2599 if (!II->isSameOperationAs(II0))
2600 return false;
2601
2602 // Can we theoretically form the data operands for the merged `invoke`?
2603 auto IsIllegalToMergeArguments = [](auto Ops) {
2604 Use &U0 = std::get<0>(Ops);
2605 Use &U1 = std::get<1>(Ops);
2606 if (U0 == U1)
2607 return false;
2608 return U0->getType()->isTokenTy() ||
2609 !canReplaceOperandWithVariable(cast<Instruction>(U0.getUser()),
2610 U0.getOperandNo());
2611 };
2612 assert(Invokes.size() == 2 && "Always called with exactly two candidates.");
2613 if (any_of(zip(Invokes[0]->data_ops(), Invokes[1]->data_ops()),
2614 IsIllegalToMergeArguments))
2615 return false;
2616
2617 return true;
2618}
2619
2620} // namespace
2621
2622// Merge all invokes in the provided set, all of which are compatible
2623// as per the `CompatibleSets::shouldBelongToSameSet()`.
2625 DomTreeUpdater *DTU) {
2626 assert(Invokes.size() >= 2 && "Must have at least two invokes to merge.");
2627
2629 if (DTU)
2630 Updates.reserve(2 + 3 * Invokes.size());
2631
2632 bool HasNormalDest =
2633 !isa<UnreachableInst>(Invokes[0]->getNormalDest()->getFirstNonPHIOrDbg());
2634
2635 // Clone one of the invokes into a new basic block.
2636 // Since they are all compatible, it doesn't matter which invoke is cloned.
2637 InvokeInst *MergedInvoke = [&Invokes, HasNormalDest]() {
2638 InvokeInst *II0 = Invokes.front();
2639 BasicBlock *II0BB = II0->getParent();
2640 BasicBlock *InsertBeforeBlock =
2641 II0->getParent()->getIterator()->getNextNode();
2642 Function *Func = II0BB->getParent();
2643 LLVMContext &Ctx = II0->getContext();
2644
2645 BasicBlock *MergedInvokeBB = BasicBlock::Create(
2646 Ctx, II0BB->getName() + ".invoke", Func, InsertBeforeBlock);
2647
2648 auto *MergedInvoke = cast<InvokeInst>(II0->clone());
2649 // NOTE: all invokes have the same attributes, so no handling needed.
2650 MergedInvoke->insertInto(MergedInvokeBB, MergedInvokeBB->end());
2651
2652 if (!HasNormalDest) {
2653 // This set does not have a normal destination,
2654 // so just form a new block with unreachable terminator.
2655 BasicBlock *MergedNormalDest = BasicBlock::Create(
2656 Ctx, II0BB->getName() + ".cont", Func, InsertBeforeBlock);
2657 new UnreachableInst(Ctx, MergedNormalDest);
2658 MergedInvoke->setNormalDest(MergedNormalDest);
2659 }
2660
2661 // The unwind destination, however, remainds identical for all invokes here.
2662
2663 return MergedInvoke;
2664 }();
2665
2666 if (DTU) {
2667 // Predecessor blocks that contained these invokes will now branch to
2668 // the new block that contains the merged invoke, ...
2669 for (InvokeInst *II : Invokes)
2670 Updates.push_back(
2671 {DominatorTree::Insert, II->getParent(), MergedInvoke->getParent()});
2672
2673 // ... which has the new `unreachable` block as normal destination,
2674 // or unwinds to the (same for all `invoke`s in this set) `landingpad`,
2675 for (BasicBlock *SuccBBOfMergedInvoke : successors(MergedInvoke))
2676 Updates.push_back({DominatorTree::Insert, MergedInvoke->getParent(),
2677 SuccBBOfMergedInvoke});
2678
2679 // Since predecessor blocks now unconditionally branch to a new block,
2680 // they no longer branch to their original successors.
2681 for (InvokeInst *II : Invokes)
2682 for (BasicBlock *SuccOfPredBB : successors(II->getParent()))
2683 Updates.push_back(
2684 {DominatorTree::Delete, II->getParent(), SuccOfPredBB});
2685 }
2686
2687 bool IsIndirectCall = Invokes[0]->isIndirectCall();
2688
2689 // Form the merged operands for the merged invoke.
2690 for (Use &U : MergedInvoke->operands()) {
2691 // Only PHI together the indirect callees and data operands.
2692 if (MergedInvoke->isCallee(&U)) {
2693 if (!IsIndirectCall)
2694 continue;
2695 } else if (!MergedInvoke->isDataOperand(&U))
2696 continue;
2697
2698 // Don't create trivial PHI's with all-identical incoming values.
2699 bool NeedPHI = any_of(Invokes, [&U](InvokeInst *II) {
2700 return II->getOperand(U.getOperandNo()) != U.get();
2701 });
2702 if (!NeedPHI)
2703 continue;
2704
2705 // Form a PHI out of all the data ops under this index.
2707 U->getType(), /*NumReservedValues=*/Invokes.size(), "", MergedInvoke->getIterator());
2708 for (InvokeInst *II : Invokes)
2709 PN->addIncoming(II->getOperand(U.getOperandNo()), II->getParent());
2710
2711 U.set(PN);
2712 }
2713
2714 // We've ensured that each PHI node has compatible (identical) incoming values
2715 // when coming from each of the `invoke`s in the current merge set,
2716 // so update the PHI nodes accordingly.
2717 for (BasicBlock *Succ : successors(MergedInvoke))
2718 AddPredecessorToBlock(Succ, /*NewPred=*/MergedInvoke->getParent(),
2719 /*ExistPred=*/Invokes.front()->getParent());
2720
2721 // And finally, replace the original `invoke`s with an unconditional branch
2722 // to the block with the merged `invoke`. Also, give that merged `invoke`
2723 // the merged debugloc of all the original `invoke`s.
2724 DILocation *MergedDebugLoc = nullptr;
2725 for (InvokeInst *II : Invokes) {
2726 // Compute the debug location common to all the original `invoke`s.
2727 if (!MergedDebugLoc)
2728 MergedDebugLoc = II->getDebugLoc();
2729 else
2730 MergedDebugLoc =
2731 DILocation::getMergedLocation(MergedDebugLoc, II->getDebugLoc());
2732
2733 // And replace the old `invoke` with an unconditionally branch
2734 // to the block with the merged `invoke`.
2735 for (BasicBlock *OrigSuccBB : successors(II->getParent()))
2736 OrigSuccBB->removePredecessor(II->getParent());
2737 BranchInst::Create(MergedInvoke->getParent(), II->getParent());
2738 II->replaceAllUsesWith(MergedInvoke);
2739 II->eraseFromParent();
2740 ++NumInvokesMerged;
2741 }
2742 MergedInvoke->setDebugLoc(MergedDebugLoc);
2743 ++NumInvokeSetsFormed;
2744
2745 if (DTU)
2746 DTU->applyUpdates(Updates);
2747}
2748
2749/// If this block is a `landingpad` exception handling block, categorize all
2750/// the predecessor `invoke`s into sets, with all `invoke`s in each set
2751/// being "mergeable" together, and then merge invokes in each set together.
2752///
2753/// This is a weird mix of hoisting and sinking. Visually, it goes from:
2754/// [...] [...]
2755/// | |
2756/// [invoke0] [invoke1]
2757/// / \ / \
2758/// [cont0] [landingpad] [cont1]
2759/// to:
2760/// [...] [...]
2761/// \ /
2762/// [invoke]
2763/// / \
2764/// [cont] [landingpad]
2765///
2766/// But of course we can only do that if the invokes share the `landingpad`,
2767/// edges invoke0->cont0 and invoke1->cont1 are "compatible",
2768/// and the invoked functions are "compatible".
2771 return false;
2772
2773 bool Changed = false;
2774
2775 // FIXME: generalize to all exception handling blocks?
2776 if (!BB->isLandingPad())
2777 return Changed;
2778
2779 CompatibleSets Grouper;
2780
2781 // Record all the predecessors of this `landingpad`. As per verifier,
2782 // the only allowed predecessor is the unwind edge of an `invoke`.
2783 // We want to group "compatible" `invokes` into the same set to be merged.
2784 for (BasicBlock *PredBB : predecessors(BB))
2785 Grouper.insert(cast<InvokeInst>(PredBB->getTerminator()));
2786
2787 // And now, merge `invoke`s that were grouped togeter.
2788 for (ArrayRef<InvokeInst *> Invokes : Grouper.Sets) {
2789 if (Invokes.size() < 2)
2790 continue;
2791 Changed = true;
2792 MergeCompatibleInvokesImpl(Invokes, DTU);
2793 }
2794
2795 return Changed;
2796}
2797
2798namespace {
2799/// Track ephemeral values, which should be ignored for cost-modelling
2800/// purposes. Requires walking instructions in reverse order.
2801class EphemeralValueTracker {
2803
2804 bool isEphemeral(const Instruction *I) {
2805 if (isa<AssumeInst>(I))
2806 return true;
2807 return !I->mayHaveSideEffects() && !I->isTerminator() &&
2808 all_of(I->users(), [&](const User *U) {
2809 return EphValues.count(cast<Instruction>(U));
2810 });
2811 }
2812
2813public:
2814 bool track(const Instruction *I) {
2815 if (isEphemeral(I)) {
2816 EphValues.insert(I);
2817 return true;
2818 }
2819 return false;
2820 }
2821
2822 bool contains(const Instruction *I) const { return EphValues.contains(I); }
2823};
2824} // namespace
2825
2826/// Determine if we can hoist sink a sole store instruction out of a
2827/// conditional block.
2828///
2829/// We are looking for code like the following:
2830/// BrBB:
2831/// store i32 %add, i32* %arrayidx2
2832/// ... // No other stores or function calls (we could be calling a memory
2833/// ... // function).
2834/// %cmp = icmp ult %x, %y
2835/// br i1 %cmp, label %EndBB, label %ThenBB
2836/// ThenBB:
2837/// store i32 %add5, i32* %arrayidx2
2838/// br label EndBB
2839/// EndBB:
2840/// ...
2841/// We are going to transform this into:
2842/// BrBB:
2843/// store i32 %add, i32* %arrayidx2
2844/// ... //
2845/// %cmp = icmp ult %x, %y
2846/// %add.add5 = select i1 %cmp, i32 %add, %add5
2847/// store i32 %add.add5, i32* %arrayidx2
2848/// ...
2849///
2850/// \return The pointer to the value of the previous store if the store can be
2851/// hoisted into the predecessor block. 0 otherwise.
2853 BasicBlock *StoreBB, BasicBlock *EndBB) {
2854 StoreInst *StoreToHoist = dyn_cast<StoreInst>(I);
2855 if (!StoreToHoist)
2856 return nullptr;
2857
2858 // Volatile or atomic.
2859 if (!StoreToHoist->isSimple())
2860 return nullptr;
2861
2862 Value *StorePtr = StoreToHoist->getPointerOperand();
2863 Type *StoreTy = StoreToHoist->getValueOperand()->getType();
2864
2865 // Look for a store to the same pointer in BrBB.
2866 unsigned MaxNumInstToLookAt = 9;
2867 // Skip pseudo probe intrinsic calls which are not really killing any memory
2868 // accesses.
2869 for (Instruction &CurI : reverse(BrBB->instructionsWithoutDebug(true))) {
2870 if (!MaxNumInstToLookAt)
2871 break;
2872 --MaxNumInstToLookAt;
2873
2874 // Could be calling an instruction that affects memory like free().
2875 if (CurI.mayWriteToMemory() && !isa<StoreInst>(CurI))
2876 return nullptr;
2877
2878 if (auto *SI = dyn_cast<StoreInst>(&CurI)) {
2879 // Found the previous store to same location and type. Make sure it is
2880 // simple, to avoid introducing a spurious non-atomic write after an
2881 // atomic write.
2882 if (SI->getPointerOperand() == StorePtr &&
2883 SI->getValueOperand()->getType() == StoreTy && SI->isSimple() &&
2884 SI->getAlign() >= StoreToHoist->getAlign())
2885 // Found the previous store, return its value operand.
2886 return SI->getValueOperand();
2887 return nullptr; // Unknown store.
2888 }
2889
2890 if (auto *LI = dyn_cast<LoadInst>(&CurI)) {
2891 if (LI->getPointerOperand() == StorePtr && LI->getType() == StoreTy &&
2892 LI->isSimple() && LI->getAlign() >= StoreToHoist->getAlign()) {
2893 // Local objects (created by an `alloca` instruction) are always
2894 // writable, so once we are past a read from a location it is valid to
2895 // also write to that same location.
2896 // If the address of the local object never escapes the function, that
2897 // means it's never concurrently read or written, hence moving the store
2898 // from under the condition will not introduce a data race.
2899 auto *AI = dyn_cast<AllocaInst>(getUnderlyingObject(StorePtr));
2900 if (AI && !PointerMayBeCaptured(AI, false, true))
2901 // Found a previous load, return it.
2902 return LI;
2903 }
2904 // The load didn't work out, but we may still find a store.
2905 }
2906 }
2907
2908 return nullptr;
2909}
2910
2911/// Estimate the cost of the insertion(s) and check that the PHI nodes can be
2912/// converted to selects.
2914 BasicBlock *EndBB,
2915 unsigned &SpeculatedInstructions,
2917 const TargetTransformInfo &TTI) {
2919 BB->getParent()->hasMinSize()
2922
2923 bool HaveRewritablePHIs = false;
2924 for (PHINode &PN : EndBB->phis()) {
2925 Value *OrigV = PN.getIncomingValueForBlock(BB);
2926 Value *ThenV = PN.getIncomingValueForBlock(ThenBB);
2927
2928 // FIXME: Try to remove some of the duplication with
2929 // hoistCommonCodeFromSuccessors. Skip PHIs which are trivial.
2930 if (ThenV == OrigV)
2931 continue;
2932
2933 Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr,
2935
2936 // Don't convert to selects if we could remove undefined behavior instead.
2937 if (passingValueIsAlwaysUndefined(OrigV, &PN) ||
2939 return false;
2940
2941 HaveRewritablePHIs = true;
2942 ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV);
2943 ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV);
2944 if (!OrigCE && !ThenCE)
2945 continue; // Known cheap (FIXME: Maybe not true for aggregates).
2946
2947 InstructionCost OrigCost = OrigCE ? computeSpeculationCost(OrigCE, TTI) : 0;
2948 InstructionCost ThenCost = ThenCE ? computeSpeculationCost(ThenCE, TTI) : 0;
2949 InstructionCost MaxCost =
2951 if (OrigCost + ThenCost > MaxCost)
2952 return false;
2953
2954 // Account for the cost of an unfolded ConstantExpr which could end up
2955 // getting expanded into Instructions.
2956 // FIXME: This doesn't account for how many operations are combined in the
2957 // constant expression.
2958 ++SpeculatedInstructions;
2959 if (SpeculatedInstructions > 1)
2960 return false;
2961 }
2962
2963 return HaveRewritablePHIs;
2964}
2965
2966/// Speculate a conditional basic block flattening the CFG.
2967///
2968/// Note that this is a very risky transform currently. Speculating
2969/// instructions like this is most often not desirable. Instead, there is an MI
2970/// pass which can do it with full awareness of the resource constraints.
2971/// However, some cases are "obvious" and we should do directly. An example of
2972/// this is speculating a single, reasonably cheap instruction.
2973///
2974/// There is only one distinct advantage to flattening the CFG at the IR level:
2975/// it makes very common but simplistic optimizations such as are common in
2976/// instcombine and the DAG combiner more powerful by removing CFG edges and
2977/// modeling their effects with easier to reason about SSA value graphs.
2978///
2979///
2980/// An illustration of this transform is turning this IR:
2981/// \code
2982/// BB:
2983/// %cmp = icmp ult %x, %y
2984/// br i1 %cmp, label %EndBB, label %ThenBB
2985/// ThenBB:
2986/// %sub = sub %x, %y
2987/// br label BB2
2988/// EndBB:
2989/// %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ]
2990/// ...
2991/// \endcode
2992///
2993/// Into this IR:
2994/// \code
2995/// BB:
2996/// %cmp = icmp ult %x, %y
2997/// %sub = sub %x, %y
2998/// %cond = select i1 %cmp, 0, %sub
2999/// ...
3000/// \endcode
3001///
3002/// \returns true if the conditional block is removed.
3003bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI,
3004 BasicBlock *ThenBB) {
3005 if (!Options.SpeculateBlocks)
3006 return false;
3007
3008 // Be conservative for now. FP select instruction can often be expensive.
3009 Value *BrCond = BI->getCondition();
3010 if (isa<FCmpInst>(BrCond))
3011 return false;
3012
3013 BasicBlock *BB = BI->getParent();
3014 BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0);
3015 InstructionCost Budget =
3017
3018 // If ThenBB is actually on the false edge of the conditional branch, remember
3019 // to swap the select operands later.
3020 bool Invert = false;
3021 if (ThenBB != BI->getSuccessor(0)) {
3022 assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?");
3023 Invert = true;
3024 }
3025 assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block");
3026
3027 // If the branch is non-unpredictable, and is predicted to *not* branch to
3028 // the `then` block, then avoid speculating it.
3029 if (!BI->getMetadata(LLVMContext::MD_unpredictable)) {
3030 uint64_t TWeight, FWeight;
3031 if (extractBranchWeights(*BI, TWeight, FWeight) &&
3032 (TWeight + FWeight) != 0) {
3033 uint64_t EndWeight = Invert ? TWeight : FWeight;
3034 BranchProbability BIEndProb =
3035 BranchProbability::getBranchProbability(EndWeight, TWeight + FWeight);
3037 if (BIEndProb >= Likely)
3038 return false;
3039 }
3040 }
3041
3042 // Keep a count of how many times instructions are used within ThenBB when
3043 // they are candidates for sinking into ThenBB. Specifically:
3044 // - They are defined in BB, and
3045 // - They have no side effects, and
3046 // - All of their uses are in ThenBB.
3047 SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts;
3048
3049 SmallVector<Instruction *, 4> SpeculatedDbgIntrinsics;
3050
3051 unsigned SpeculatedInstructions = 0;
3052 Value *SpeculatedStoreValue = nullptr;
3053 StoreInst *SpeculatedStore = nullptr;
3054 EphemeralValueTracker EphTracker;
3055 for (Instruction &I : reverse(drop_end(*ThenBB))) {
3056 // Skip debug info.
3057 if (isa<DbgInfoIntrinsic>(I)) {
3058 SpeculatedDbgIntrinsics.push_back(&I);
3059 continue;
3060 }
3061
3062 // Skip pseudo probes. The consequence is we lose track of the branch
3063 // probability for ThenBB, which is fine since the optimization here takes
3064 // place regardless of the branch probability.
3065 if (isa<PseudoProbeInst>(I)) {
3066 // The probe should be deleted so that it will not be over-counted when
3067 // the samples collected on the non-conditional path are counted towards
3068 // the conditional path. We leave it for the counts inference algorithm to
3069 // figure out a proper count for an unknown probe.
3070 SpeculatedDbgIntrinsics.push_back(&I);
3071 continue;
3072 }
3073
3074 // Ignore ephemeral values, they will be dropped by the transform.
3075 if (EphTracker.track(&I))
3076 continue;
3077
3078 // Only speculatively execute a single instruction (not counting the
3079 // terminator) for now.
3080 ++SpeculatedInstructions;
3081 if (SpeculatedInstructions > 1)
3082 return false;
3083
3084 // Don't hoist the instruction if it's unsafe or expensive.
3086 !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore(
3087 &I, BB, ThenBB, EndBB))))
3088 return false;
3089 if (!SpeculatedStoreValue &&
3092 return false;
3093
3094 // Store the store speculation candidate.
3095 if (SpeculatedStoreValue)
3096 SpeculatedStore = cast<StoreInst>(&I);
3097
3098 // Do not hoist the instruction if any of its operands are defined but not
3099 // used in BB. The transformation will prevent the operand from
3100 // being sunk into the use block.
3101 for (Use &Op : I.operands()) {
3102 Instruction *OpI = dyn_cast<Instruction>(Op);
3103 if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects())
3104 continue; // Not a candidate for sinking.
3105
3106 ++SinkCandidateUseCounts[OpI];
3107 }
3108 }
3109
3110 // Consider any sink candidates which are only used in ThenBB as costs for
3111 // speculation. Note, while we iterate over a DenseMap here, we are summing
3112 // and so iteration order isn't significant.
3113 for (const auto &[Inst, Count] : SinkCandidateUseCounts)
3114 if (Inst->hasNUses(Count)) {
3115 ++SpeculatedInstructions;
3116 if (SpeculatedInstructions > 1)
3117 return false;
3118 }
3119
3120 // Check that we can insert the selects and that it's not too expensive to do
3121 // so.
3122 bool Convert = SpeculatedStore != nullptr;
3124 Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB,
3125 SpeculatedInstructions,
3126 Cost, TTI);
3127 if (!Convert || Cost > Budget)
3128 return false;
3129
3130 // If we get here, we can hoist the instruction and if-convert.
3131 LLVM_DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";);
3132
3133 // Insert a select of the value of the speculated store.
3134 if (SpeculatedStoreValue) {
3135 IRBuilder<NoFolder> Builder(BI);
3136 Value *OrigV = SpeculatedStore->getValueOperand();
3137 Value *TrueV = SpeculatedStore->getValueOperand();
3138 Value *FalseV = SpeculatedStoreValue;
3139 if (Invert)
3140 std::swap(TrueV, FalseV);
3141 Value *S = Builder.CreateSelect(
3142 BrCond, TrueV, FalseV, "spec.store.select", BI);
3143 SpeculatedStore->setOperand(0, S);
3144 SpeculatedStore->applyMergedLocation(BI->getDebugLoc(),
3145 SpeculatedStore->getDebugLoc());
3146 // The value stored is still conditional, but the store itself is now
3147 // unconditonally executed, so we must be sure that any linked dbg.assign
3148 // intrinsics are tracking the new stored value (the result of the
3149 // select). If we don't, and the store were to be removed by another pass
3150 // (e.g. DSE), then we'd eventually end up emitting a location describing
3151 // the conditional value, unconditionally.
3152 //
3153 // === Before this transformation ===
3154 // pred:
3155 // store %one, %x.dest, !DIAssignID !1
3156 // dbg.assign %one, "x", ..., !1, ...
3157 // br %cond if.then
3158 //
3159 // if.then:
3160 // store %two, %x.dest, !DIAssignID !2
3161 // dbg.assign %two, "x", ..., !2, ...
3162 //
3163 // === After this transformation ===
3164 // pred:
3165 // store %one, %x.dest, !DIAssignID !1
3166 // dbg.assign %one, "x", ..., !1
3167 /// ...
3168 // %merge = select %cond, %two, %one
3169 // store %merge, %x.dest, !DIAssignID !2
3170 // dbg.assign %merge, "x", ..., !2
3171 auto replaceVariable = [OrigV, S](auto *DbgAssign) {
3172 if (llvm::is_contained(DbgAssign->location_ops(), OrigV))
3173 DbgAssign->replaceVariableLocationOp(OrigV, S);
3174 };
3175 for_each(at::getAssignmentMarkers(SpeculatedStore), replaceVariable);
3176 for_each(at::getDVRAssignmentMarkers(SpeculatedStore), replaceVariable);
3177 }
3178
3179 // Metadata can be dependent on the condition we are hoisting above.
3180 // Strip all UB-implying metadata on the instruction. Drop the debug loc
3181 // to avoid making it appear as if the condition is a constant, which would
3182 // be misleading while debugging.
3183 // Similarly strip attributes that maybe dependent on condition we are
3184 // hoisting above.
3185 for (auto &I : make_early_inc_range(*ThenBB)) {
3186 if (!SpeculatedStoreValue || &I != SpeculatedStore) {
3187 // Don't update the DILocation of dbg.assign intrinsics.
3188 if (!isa<DbgAssignIntrinsic>(&I))
3189 I.setDebugLoc(DebugLoc());
3190 }
3191 I.dropUBImplyingAttrsAndMetadata();
3192
3193 // Drop ephemeral values.
3194 if (EphTracker.contains(&I)) {
3195 I.replaceAllUsesWith(PoisonValue::get(I.getType()));
3196 I.eraseFromParent();
3197 }
3198 }
3199
3200 // Hoist the instructions.
3201 // In "RemoveDIs" non-instr debug-info mode, drop DbgVariableRecords attached
3202 // to these instructions, in the same way that dbg.value intrinsics are
3203 // dropped at the end of this block.
3204 for (auto &It : make_range(ThenBB->begin(), ThenBB->end()))
3205 for (DbgRecord &DR : make_early_inc_range(It.getDbgRecordRange()))
3206 // Drop all records except assign-kind DbgVariableRecords (dbg.assign
3207 // equivalent).
3208 if (DbgVariableRecord *DVR = dyn_cast<DbgVariableRecord>(&DR);
3209 !DVR || !DVR->isDbgAssign())
3210 It.dropOneDbgRecord(&DR);
3211 BB->splice(BI->getIterator(), ThenBB, ThenBB->begin(),
3212 std::prev(ThenBB->end()));
3213
3214 // Insert selects and rewrite the PHI operands.
3215 IRBuilder<NoFolder> Builder(BI);
3216 for (PHINode &PN : EndBB->phis()) {
3217 unsigned OrigI = PN.getBasicBlockIndex(BB);
3218 unsigned ThenI = PN.getBasicBlockIndex(ThenBB);
3219 Value *OrigV = PN.getIncomingValue(OrigI);
3220 Value *ThenV = PN.getIncomingValue(ThenI);
3221
3222 // Skip PHIs which are trivial.
3223 if (OrigV == ThenV)
3224 continue;
3225
3226 // Create a select whose true value is the speculatively executed value and
3227 // false value is the pre-existing value. Swap them if the branch
3228 // destinations were inverted.
3229 Value *TrueV = ThenV, *FalseV = OrigV;
3230 if (Invert)
3231 std::swap(TrueV, FalseV);
3232 Value *V = Builder.CreateSelect(BrCond, TrueV, FalseV, "spec.select", BI);
3233 PN.setIncomingValue(OrigI, V);
3234 PN.setIncomingValue(ThenI, V);
3235 }
3236
3237 // Remove speculated dbg intrinsics.
3238 // FIXME: Is it possible to do this in a more elegant way? Moving/merging the
3239 // dbg value for the different flows and inserting it after the select.
3240 for (Instruction *I : SpeculatedDbgIntrinsics) {
3241 // We still want to know that an assignment took place so don't remove
3242 // dbg.assign intrinsics.
3243 if (!isa<DbgAssignIntrinsic>(I))
3244 I->eraseFromParent();
3245 }
3246
3247 ++NumSpeculations;
3248 return true;
3249}
3250
3251/// Return true if we can thread a branch across this block.
3253 int Size = 0;
3254 EphemeralValueTracker EphTracker;
3255
3256 // Walk the loop in reverse so that we can identify ephemeral values properly
3257 // (values only feeding assumes).
3258 for (Instruction &I : reverse(BB->instructionsWithoutDebug(false))) {
3259 // Can't fold blocks that contain noduplicate or convergent calls.
3260 if (CallInst *CI = dyn_cast<CallInst>(&I))
3261 if (CI->cannotDuplicate() || CI->isConvergent())
3262 return false;
3263
3264 // Ignore ephemeral values which are deleted during codegen.
3265 // We will delete Phis while threading, so Phis should not be accounted in
3266 // block's size.
3267 if (!EphTracker.track(&I) && !isa<PHINode>(I)) {
3268 if (Size++ > MaxSmallBlockSize)
3269 return false; // Don't clone large BB's.
3270 }
3271
3272 // We can only support instructions that do not define values that are
3273 // live outside of the current basic block.
3274 for (User *U : I.users()) {
3275 Instruction *UI = cast<Instruction>(U);
3276 if (UI->getParent() != BB || isa<PHINode>(UI))
3277 return false;
3278 }
3279
3280 // Looks ok, continue checking.
3281 }
3282
3283 return true;
3284}
3285
3287 BasicBlock *To) {
3288 // Don't look past the block defining the value, we might get the value from
3289 // a previous loop iteration.
3290 auto *I = dyn_cast<Instruction>(V);
3291 if (I && I->getParent() == To)
3292 return nullptr;
3293
3294 // We know the value if the From block branches on it.
3295 auto *BI = dyn_cast<BranchInst>(From->getTerminator());
3296 if (BI && BI->isConditional() && BI->getCondition() == V &&
3297 BI->getSuccessor(0) != BI->getSuccessor(1))
3298 return BI->getSuccessor(0) == To ? ConstantInt::getTrue(BI->getContext())
3300
3301 return nullptr;
3302}
3303
3304/// If we have a conditional branch on something for which we know the constant
3305/// value in predecessors (e.g. a phi node in the current block), thread edges
3306/// from the predecessor to their ultimate destination.
3307static std::optional<bool>
3309 const DataLayout &DL,
3310 AssumptionCache *AC) {
3312 BasicBlock *BB = BI->getParent();
3313 Value *Cond = BI->getCondition();
3314 PHINode *PN = dyn_cast<PHINode>(Cond);
3315 if (PN && PN->getParent() == BB) {
3316 // Degenerate case of a single entry PHI.
3317 if (PN->getNumIncomingValues() == 1) {
3319 return true;
3320 }
3321
3322 for (Use &U : PN->incoming_values())
3323 if (auto *CB = dyn_cast<ConstantInt>(U))
3324 KnownValues[CB].insert(PN->getIncomingBlock(U));
3325 } else {
3326 for (BasicBlock *Pred : predecessors(BB)) {
3327 if (ConstantInt *CB = getKnownValueOnEdge(Cond, Pred, BB))
3328 KnownValues[CB].insert(Pred);
3329 }
3330 }
3331
3332 if (KnownValues.empty())
3333 return false;
3334
3335 // Now we know that this block has multiple preds and two succs.
3336 // Check that the block is small enough and values defined in the block are
3337 // not used outside of it.
3339 return false;
3340
3341 for (const auto &Pair : KnownValues) {
3342 // Okay, we now know that all edges from PredBB should be revectored to
3343 // branch to RealDest.
3344 ConstantInt *CB = Pair.first;
3345 ArrayRef<BasicBlock *> PredBBs = Pair.second.getArrayRef();
3346 BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue());
3347
3348 if (RealDest == BB)
3349 continue; // Skip self loops.
3350
3351 // Skip if the predecessor's terminator is an indirect branch.
3352 if (any_of(PredBBs, [](BasicBlock *PredBB) {
3353 return isa<IndirectBrInst>(PredBB->getTerminator());
3354 }))
3355 continue;
3356
3357 LLVM_DEBUG({
3358 dbgs() << "Condition " << *Cond << " in " << BB->getName()
3359 << " has value " << *Pair.first << " in predecessors:\n";
3360 for (const BasicBlock *PredBB : Pair.second)
3361 dbgs() << " " << PredBB->getName() << "\n";
3362 dbgs() << "Threading to destination " << RealDest->getName() << ".\n";
3363 });
3364
3365 // Split the predecessors we are threading into a new edge block. We'll
3366 // clone the instructions into this block, and then redirect it to RealDest.
3367 BasicBlock *EdgeBB = SplitBlockPredecessors(BB, PredBBs, ".critedge", DTU);
3368
3369 // TODO: These just exist to reduce test diff, we can drop them if we like.
3370 EdgeBB->setName(RealDest->getName() + ".critedge");
3371 EdgeBB->moveBefore(RealDest);
3372
3373 // Update PHI nodes.
3374 AddPredecessorToBlock(RealDest, EdgeBB, BB);
3375
3376 // BB may have instructions that are being threaded over. Clone these
3377 // instructions into EdgeBB. We know that there will be no uses of the
3378 // cloned instructions outside of EdgeBB.
3379 BasicBlock::iterator InsertPt = EdgeBB->getFirstInsertionPt();
3380 DenseMap<Value *, Value *> TranslateMap; // Track translated values.
3381 TranslateMap[Cond] = CB;
3382
3383 // RemoveDIs: track instructions that we optimise away while folding, so
3384 // that we can copy DbgVariableRecords from them later.
3385 BasicBlock::iterator SrcDbgCursor = BB->begin();
3386 for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
3387 if (PHINode *PN = dyn_cast<PHINode>(BBI)) {
3388 TranslateMap[PN] = PN->getIncomingValueForBlock(EdgeBB);
3389 continue;
3390 }
3391 // Clone the instruction.
3392 Instruction *N = BBI->clone();
3393 // Insert the new instruction into its new home.
3394 N->insertInto(EdgeBB, InsertPt);
3395
3396 if (BBI->hasName())
3397 N->setName(BBI->getName() + ".c");
3398
3399 // Update operands due to translation.
3400 for (Use &Op : N->operands()) {
3401 DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(Op);
3402 if (PI != TranslateMap.end())
3403 Op = PI->second;
3404 }
3405
3406 // Check for trivial simplification.
3407 if (Value *V = simplifyInstruction(N, {DL, nullptr, nullptr, AC})) {
3408 if (!BBI->use_empty())
3409 TranslateMap[&*BBI] = V;
3410 if (!N->mayHaveSideEffects()) {
3411 N->eraseFromParent(); // Instruction folded away, don't need actual
3412 // inst
3413 N = nullptr;
3414 }
3415 } else {
3416 if (!BBI->use_empty())
3417 TranslateMap[&*BBI] = N;
3418 }
3419 if (N) {
3420 // Copy all debug-info attached to instructions from the last we
3421 // successfully clone, up to this instruction (they might have been
3422 // folded away).
3423 for (; SrcDbgCursor != BBI; ++SrcDbgCursor)
3424 N->cloneDebugInfoFrom(&*SrcDbgCursor);
3425 SrcDbgCursor = std::next(BBI);
3426 // Clone debug-info on this instruction too.
3427 N->cloneDebugInfoFrom(&*BBI);
3428
3429 // Register the new instruction with the assumption cache if necessary.
3430 if (auto *Assume = dyn_cast<AssumeInst>(N))
3431 if (AC)
3432 AC->registerAssumption(Assume);
3433 }
3434 }
3435
3436 for (; &*SrcDbgCursor != BI; ++SrcDbgCursor)
3437 InsertPt->cloneDebugInfoFrom(&*SrcDbgCursor);
3438 InsertPt->cloneDebugInfoFrom(BI);
3439
3440 BB->removePredecessor(EdgeBB);
3441 BranchInst *EdgeBI = cast<BranchInst>(EdgeBB->getTerminator());
3442 EdgeBI->setSuccessor(0, RealDest);
3443 EdgeBI->setDebugLoc(BI->getDebugLoc());
3444
3445 if (DTU) {
3447 Updates.push_back({DominatorTree::Delete, EdgeBB, BB});
3448 Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest});
3449 DTU->applyUpdates(Updates);
3450 }
3451
3452 // For simplicity, we created a separate basic block for the edge. Merge
3453 // it back into the predecessor if possible. This not only avoids
3454 // unnecessary SimplifyCFG iterations, but also makes sure that we don't
3455 // bypass the check for trivial cycles above.
3456 MergeBlockIntoPredecessor(EdgeBB, DTU);
3457
3458 // Signal repeat, simplifying any other constants.
3459 return std::nullopt;
3460 }
3461
3462 return false;
3463}
3464
3466 DomTreeUpdater *DTU,
3467 const DataLayout &DL,
3468 AssumptionCache *AC) {
3469 std::optional<bool> Result;
3470 bool EverChanged = false;
3471 do {
3472 // Note that None means "we changed things, but recurse further."
3473 Result = FoldCondBranchOnValueKnownInPredecessorImpl(BI, DTU, DL, AC);
3474 EverChanged |= Result == std::nullopt || *Result;
3475 } while (Result == std::nullopt);
3476 return EverChanged;
3477}
3478
3479/// Given a BB that starts with the specified two-entry PHI node,
3480/// see if we can eliminate it.
3482 DomTreeUpdater *DTU, const DataLayout &DL) {
3483 // Ok, this is a two entry PHI node. Check to see if this is a simple "if
3484 // statement", which has a very simple dominance structure. Basically, we
3485 // are trying to find the condition that is being branched on, which
3486 // subsequently causes this merge to happen. We really want control
3487 // dependence information for this check, but simplifycfg can't keep it up
3488 // to date, and this catches most of the cases we care about anyway.
3489 BasicBlock *BB = PN->getParent();
3490
3491 BasicBlock *IfTrue, *IfFalse;
3492 BranchInst *DomBI = GetIfCondition(BB, IfTrue, IfFalse);
3493 if (!DomBI)
3494 return false;
3495 Value *IfCond = DomBI->getCondition();
3496 // Don't bother if the branch will be constant folded trivially.
3497 if (isa<ConstantInt>(IfCond))
3498 return false;
3499
3500 BasicBlock *DomBlock = DomBI->getParent();
3503 PN->blocks(), std::back_inserter(IfBlocks), [](BasicBlock *IfBlock) {
3504 return cast<BranchInst>(IfBlock->getTerminator())->isUnconditional();
3505 });
3506 assert((IfBlocks.size() == 1 || IfBlocks.size() == 2) &&
3507 "Will have either one or two blocks to speculate.");
3508
3509 // If the branch is non-unpredictable, see if we either predictably jump to
3510 // the merge bb (if we have only a single 'then' block), or if we predictably
3511 // jump to one specific 'then' block (if we have two of them).
3512 // It isn't beneficial to speculatively execute the code
3513 // from the block that we know is predictably not entered.
3514 if (!DomBI->getMetadata(LLVMContext::MD_unpredictable)) {
3515 uint64_t TWeight, FWeight;
3516 if (extractBranchWeights(*DomBI, TWeight, FWeight) &&
3517 (TWeight + FWeight) != 0) {
3518 BranchProbability BITrueProb =
3519 BranchProbability::getBranchProbability(TWeight, TWeight + FWeight);
3521 BranchProbability BIFalseProb = BITrueProb.getCompl();
3522 if (IfBlocks.size() == 1) {
3523 BranchProbability BIBBProb =
3524 DomBI->getSuccessor(0) == BB ? BITrueProb : BIFalseProb;
3525 if (BIBBProb >= Likely)
3526 return false;
3527 } else {
3528 if (BITrueProb >= Likely || BIFalseProb >= Likely)
3529 return false;
3530 }
3531 }
3532 }
3533
3534 // Don't try to fold an unreachable block. For example, the phi node itself
3535 // can't be the candidate if-condition for a select that we want to form.
3536 if (auto *IfCondPhiInst = dyn_cast<PHINode>(IfCond))
3537 if (IfCondPhiInst->getParent() == BB)
3538 return false;
3539
3540 // Okay, we found that we can merge this two-entry phi node into a select.
3541 // Doing so would require us to fold *all* two entry phi nodes in this block.
3542 // At some point this becomes non-profitable (particularly if the target
3543 // doesn't support cmov's). Only do this transformation if there are two or
3544 // fewer PHI nodes in this block.
3545 unsigned NumPhis = 0;
3546 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I)
3547 if (NumPhis > 2)
3548 return false;
3549
3550 // Loop over the PHI's seeing if we can promote them all to select
3551 // instructions. While we are at it, keep track of the instructions
3552 // that need to be moved to the dominating block.
3553 SmallPtrSet<Instruction *, 4> AggressiveInsts;
3555 InstructionCost Budget =
3557
3558 bool Changed = false;
3559 for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
3560 PHINode *PN = cast<PHINode>(II++);
3561 if (Value *V = simplifyInstruction(PN, {DL, PN})) {
3562 PN->replaceAllUsesWith(V);
3563 PN->eraseFromParent();
3564 Changed = true;
3565 continue;
3566 }
3567
3568 if (!dominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts,
3569 Cost, Budget, TTI) ||
3570 !dominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts,
3571 Cost, Budget, TTI))
3572 return Changed;
3573 }
3574
3575 // If we folded the first phi, PN dangles at this point. Refresh it. If
3576 // we ran out of PHIs then we simplified them all.
3577 PN = dyn_cast<PHINode>(BB->begin());
3578 if (!PN)
3579 return true;
3580
3581 // Return true if at least one of these is a 'not', and another is either
3582 // a 'not' too, or a constant.
3583 auto CanHoistNotFromBothValues = [](Value *V0, Value *V1) {
3584 if (!match(V0, m_Not(m_Value())))
3585 std::swap(V0, V1);
3586 auto Invertible = m_CombineOr(m_Not(m_Value()), m_AnyIntegralConstant());
3587 return match(V0, m_Not(m_Value())) && match(V1, Invertible);
3588 };
3589
3590 // Don't fold i1 branches on PHIs which contain binary operators or
3591 // (possibly inverted) select form of or/ands, unless one of
3592 // the incoming values is an 'not' and another one is freely invertible.
3593 // These can often be turned into switches and other things.
3594 auto IsBinOpOrAnd = [](Value *V) {
3595 return match(
3596 V, m_CombineOr(
3597 m_BinOp(),
3600 };
3601 if (PN->getType()->isIntegerTy(1) &&
3602 (IsBinOpOrAnd(PN->getIncomingValue(0)) ||
3603 IsBinOpOrAnd(PN->getIncomingValue(1)) || IsBinOpOrAnd(IfCond)) &&
3604 !CanHoistNotFromBothValues(PN->getIncomingValue(0),
3605 PN->getIncomingValue(1)))
3606 return Changed;
3607
3608 // If all PHI nodes are promotable, check to make sure that all instructions
3609 // in the predecessor blocks can be promoted as well. If not, we won't be able
3610 // to get rid of the control flow, so it's not worth promoting to select
3611 // instructions.
3612 for (BasicBlock *IfBlock : IfBlocks)
3613 for (BasicBlock::iterator I = IfBlock->begin(); !I->isTerminator(); ++I)
3614 if (!AggressiveInsts.count(&*I) && !I->isDebugOrPseudoInst()) {
3615 // This is not an aggressive instruction that we can promote.
3616 // Because of this, we won't be able to get rid of the control flow, so
3617 // the xform is not worth it.
3618 return Changed;
3619 }
3620
3621 // If either of the blocks has it's address taken, we can't do this fold.
3622 if (any_of(IfBlocks,
3623 [](BasicBlock *IfBlock) { return IfBlock->hasAddressTaken(); }))
3624 return Changed;
3625
3626 LLVM_DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfCond
3627 << " T: " << IfTrue->getName()
3628 << " F: " << IfFalse->getName() << "\n");
3629
3630 // If we can still promote the PHI nodes after this gauntlet of tests,
3631 // do all of the PHI's now.
3632
3633 // Move all 'aggressive' instructions, which are defined in the
3634 // conditional parts of the if's up to the dominating block.
3635 for (BasicBlock *IfBlock : IfBlocks)
3636 hoistAllInstructionsInto(DomBlock, DomBI, IfBlock);
3637
3638 IRBuilder<NoFolder> Builder(DomBI);
3639 // Propagate fast-math-flags from phi nodes to replacement selects.
3640 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
3641 while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
3642 if (isa<FPMathOperator>(PN))
3643 Builder.setFastMathFlags(PN->getFastMathFlags());
3644
3645 // Change the PHI node into a select instruction.
3646 Value *TrueVal = PN->getIncomingValueForBlock(IfTrue);
3647 Value *FalseVal = PN->getIncomingValueForBlock(IfFalse);
3648
3649 Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", DomBI);
3650 PN->replaceAllUsesWith(Sel);
3651 Sel->takeName(PN);
3652 PN->eraseFromParent();
3653 }
3654
3655 // At this point, all IfBlocks are empty, so our if statement
3656 // has been flattened. Change DomBlock to jump directly to our new block to
3657 // avoid other simplifycfg's kicking in on the diamond.
3658 Builder.CreateBr(BB);
3659
3661 if (DTU) {
3662 Updates.push_back({DominatorTree::Insert, DomBlock, BB});
3663 for (auto *Successor : successors(DomBlock))
3664 Updates.push_back({DominatorTree::Delete, DomBlock, Successor});
3665 }
3666
3667 DomBI->eraseFromParent();
3668 if (DTU)
3669 DTU->applyUpdates(Updates);
3670
3671 return true;
3672}
3673
3675 Instruction::BinaryOps Opc, Value *LHS,
3676 Value *RHS, const Twine &Name = "") {
3677 // Try to relax logical op to binary op.
3678 if (impliesPoison(RHS, LHS))
3679 return Builder.CreateBinOp(Opc, LHS, RHS, Name);
3680 if (Opc == Instruction::And)
3681 return Builder.CreateLogicalAnd(LHS, RHS, Name);
3682 if (Opc == Instruction::Or)
3683 return Builder.CreateLogicalOr(LHS, RHS, Name);
3684 llvm_unreachable("Invalid logical opcode");
3685}
3686
3687/// Return true if either PBI or BI has branch weight available, and store
3688/// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does
3689/// not have branch weight, use 1:1 as its weight.
3691 uint64_t &PredTrueWeight,
3692 uint64_t &PredFalseWeight,
3693 uint64_t &SuccTrueWeight,
3694 uint64_t &SuccFalseWeight) {
3695 bool PredHasWeights =
3696 extractBranchWeights(*PBI, PredTrueWeight, PredFalseWeight);
3697 bool SuccHasWeights =
3698 extractBranchWeights(*BI, SuccTrueWeight, SuccFalseWeight);
3699 if (PredHasWeights || SuccHasWeights) {
3700 if (!PredHasWeights)
3701 PredTrueWeight = PredFalseWeight = 1;
3702 if (!SuccHasWeights)
3703 SuccTrueWeight = SuccFalseWeight = 1;
3704 return true;
3705 } else {
3706 return false;
3707 }
3708}
3709
3710/// Determine if the two branches share a common destination and deduce a glue
3711/// that joins the branches' conditions to arrive at the common destination if
3712/// that would be profitable.
3713static std::optional<std::tuple<BasicBlock *, Instruction::BinaryOps, bool>>
3715 const TargetTransformInfo *TTI) {
3716 assert(BI && PBI && BI->isConditional() && PBI->isConditional() &&
3717 "Both blocks must end with a conditional branches.");
3719 "PredBB must be a predecessor of BB.");
3720
3721 // We have the potential to fold the conditions together, but if the
3722 // predecessor branch is predictable, we may not want to merge them.
3723 uint64_t PTWeight, PFWeight;
3724 BranchProbability PBITrueProb, Likely;
3725 if (TTI && !PBI->getMetadata(LLVMContext::MD_unpredictable) &&
3726 extractBranchWeights(*PBI, PTWeight, PFWeight) &&
3727 (PTWeight + PFWeight) != 0) {
3728 PBITrueProb =
3729 BranchProbability::getBranchProbability(PTWeight, PTWeight + PFWeight);
3731 }
3732
3733 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
3734 // Speculate the 2nd condition unless the 1st is probably true.
3735 if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
3736 return {{BI->getSuccessor(0), Instruction::Or, false}};
3737 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
3738 // Speculate the 2nd condition unless the 1st is probably false.
3739 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
3740 return {{BI->getSuccessor(1), Instruction::And, false}};
3741 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
3742 // Speculate the 2nd condition unless the 1st is probably true.
3743 if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
3744 return {{BI->getSuccessor(1), Instruction::And, true}};
3745 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
3746 // Speculate the 2nd condition unless the 1st is probably false.
3747 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
3748 return {{BI->getSuccessor(0), Instruction::Or, true}};
3749 }
3750 return std::nullopt;
3751}
3752
3754 DomTreeUpdater *DTU,
3755 MemorySSAUpdater *MSSAU,
3756 const TargetTransformInfo *TTI) {
3757 BasicBlock *BB = BI->getParent();
3758 BasicBlock *PredBlock = PBI->getParent();
3759
3760 // Determine if the two branches share a common destination.
3761 BasicBlock *CommonSucc;
3763 bool InvertPredCond;
3764 std::tie(CommonSucc, Opc, InvertPredCond) =
3766
3767 LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB);
3768
3769 IRBuilder<> Builder(PBI);
3770 // The builder is used to create instructions to eliminate the branch in BB.
3771 // If BB's terminator has !annotation metadata, add it to the new
3772 // instructions.
3774 {LLVMContext::MD_annotation});
3775
3776 // If we need to invert the condition in the pred block to match, do so now.
3777 if (InvertPredCond) {
3778 InvertBranch(PBI, Builder);
3779 }
3780
3781 BasicBlock *UniqueSucc =
3782 PBI->getSuccessor(0) == BB ? BI->getSuccessor(0) : BI->getSuccessor(1);
3783
3784 // Before cloning instructions, notify the successor basic block that it
3785 // is about to have a new predecessor. This will update PHI nodes,
3786 // which will allow us to update live-out uses of bonus instructions.
3787 AddPredecessorToBlock(UniqueSucc, PredBlock, BB, MSSAU);
3788
3789 // Try to update branch weights.
3790 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
3791 if (extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
3792 SuccTrueWeight, SuccFalseWeight)) {
3793 SmallVector<uint64_t, 8> NewWeights;
3794
3795 if (PBI->getSuccessor(0) == BB) {
3796 // PBI: br i1 %x, BB, FalseDest
3797 // BI: br i1 %y, UniqueSucc, FalseDest
3798 // TrueWeight is TrueWeight for PBI * TrueWeight for BI.
3799 NewWeights.push_back(PredTrueWeight * SuccTrueWeight);
3800 // FalseWeight is FalseWeight for PBI * TotalWeight for BI +
3801 // TrueWeight for PBI * FalseWeight for BI.
3802 // We assume that total weights of a BranchInst can fit into 32 bits.
3803 // Therefore, we will not have overflow using 64-bit arithmetic.
3804 NewWeights.push_back(PredFalseWeight *
3805 (SuccFalseWeight + SuccTrueWeight) +
3806 PredTrueWeight * SuccFalseWeight);
3807 } else {
3808 // PBI: br i1 %x, TrueDest, BB
3809 // BI: br i1 %y, TrueDest, UniqueSucc
3810 // TrueWeight is TrueWeight for PBI * TotalWeight for BI +
3811 // FalseWeight for PBI * TrueWeight for BI.
3812 NewWeights.push_back(PredTrueWeight * (SuccFalseWeight + SuccTrueWeight) +
3813 PredFalseWeight * SuccTrueWeight);
3814 // FalseWeight is FalseWeight for PBI * FalseWeight for BI.
3815 NewWeights.push_back(PredFalseWeight * SuccFalseWeight);
3816 }
3817
3818 // Halve the weights if any of them cannot fit in an uint32_t
3819 FitWeights(NewWeights);
3820
3821 SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), NewWeights.end());
3822 setBranchWeights(PBI, MDWeights[0], MDWeights[1], /*IsExpected=*/false);
3823
3824 // TODO: If BB is reachable from all paths through PredBlock, then we
3825 // could replace PBI's branch probabilities with BI's.
3826 } else
3827 PBI->setMetadata(LLVMContext::MD_prof, nullptr);
3828
3829 // Now, update the CFG.
3830 PBI->setSuccessor(PBI->getSuccessor(0) != BB, UniqueSucc);
3831
3832 if (DTU)
3833 DTU->applyUpdates({{DominatorTree::Insert, PredBlock, UniqueSucc},
3834 {DominatorTree::Delete, PredBlock, BB}});
3835
3836 // If BI was a loop latch, it may have had associated loop metadata.
3837 // We need to copy it to the new latch, that is, PBI.
3838 if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop))
3839 PBI->setMetadata(LLVMContext::MD_loop, LoopMD);
3840
3841 ValueToValueMapTy VMap; // maps original values to cloned values
3843
3844 Module *M = BB->getModule();
3845
3846 if (PredBlock->IsNewDbgInfoFormat) {
3847 PredBlock->getTerminator()->cloneDebugInfoFrom(BB->getTerminator());
3848 for (DbgVariableRecord &DVR :
3850 RemapDbgRecord(M, &DVR, VMap,
3852 }
3853 }
3854
3855 // Now that the Cond was cloned into the predecessor basic block,
3856 // or/and the two conditions together.
3857 Value *BICond = VMap[BI->getCondition()];
3858 PBI->setCondition(
3859 createLogicalOp(Builder, Opc, PBI->getCondition(), BICond, "or.cond"));
3860
3861 ++NumFoldBranchToCommonDest;
3862 return true;
3863}
3864
3865/// Return if an instruction's type or any of its operands' types are a vector
3866/// type.
3867static bool isVectorOp(Instruction &I) {
3868 return I.getType()->isVectorTy() || any_of(I.operands(), [](Use &U) {
3869 return U->getType()->isVectorTy();
3870 });
3871}
3872
3873/// If this basic block is simple enough, and if a predecessor branches to us
3874/// and one of our successors, fold the block into the predecessor and use
3875/// logical operations to pick the right destination.
3877 MemorySSAUpdater *MSSAU,
3878 const TargetTransformInfo *TTI,
3879 unsigned BonusInstThreshold) {
3880 // If this block ends with an unconditional branch,
3881 // let SpeculativelyExecuteBB() deal with it.
3882 if (!BI->isConditional())
3883 return false;
3884
3885 BasicBlock *BB = BI->getParent();
3889
3890 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
3891
3892 if (!Cond ||
3893 (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond) &&
3894 !isa<SelectInst>(Cond)) ||
3895 Cond->getParent() != BB || !Cond->hasOneUse())
3896 return false;
3897
3898 // Finally, don't infinitely unroll conditional loops.
3899 if (is_contained(successors(BB), BB))
3900 return false;
3901
3902 // With which predecessors will we want to deal with?
3904 for (BasicBlock *PredBlock : predecessors(BB)) {
3905 BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
3906
3907 // Check that we have two conditional branches. If there is a PHI node in
3908 // the common successor, verify that the same value flows in from both
3909 // blocks.
3910 if (!PBI || PBI->isUnconditional() || !SafeToMergeTerminators(BI, PBI))
3911 continue;
3912
3913 // Determine if the two branches share a common destination.
3914 BasicBlock *CommonSucc;
3916 bool InvertPredCond;
3917 if (auto Recipe = shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI))
3918 std::tie(CommonSucc, Opc, InvertPredCond) = *Recipe;
3919 else
3920 continue;
3921
3922 // Check the cost of inserting the necessary logic before performing the
3923 // transformation.
3924 if (TTI) {
3925 Type *Ty = BI->getCondition()->getType();
3927 if (InvertPredCond && (!PBI->getCondition()->hasOneUse() ||
3928 !isa<CmpInst>(PBI->getCondition())))
3929 Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind);
3930
3932 continue;
3933 }
3934
3935 // Ok, we do want to deal with this predecessor. Record it.
3936 Preds.emplace_back(PredBlock);
3937 }
3938
3939 // If there aren't any predecessors into which we can fold,
3940 // don't bother checking the cost.
3941 if (Preds.empty())
3942 return false;
3943
3944 // Only allow this transformation if computing the condition doesn't involve
3945 // too many instructions and these involved instructions can be executed
3946 // unconditionally. We denote all involved instructions except the condition
3947 // as "bonus instructions", and only allow this transformation when the
3948 // number of the bonus instructions we'll need to create when cloning into
3949 // each predecessor does not exceed a certain threshold.
3950 unsigned NumBonusInsts = 0;
3951 bool SawVectorOp = false;
3952 const unsigned PredCount = Preds.size();
3953 for (Instruction &I : *BB) {
3954 // Don't check the branch condition comparison itself.
3955 if (&I == Cond)
3956 continue;
3957 // Ignore dbg intrinsics, and the terminator.
3958 if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I))
3959 continue;
3960 // I must be safe to execute unconditionally.
3962 return false;
3963 SawVectorOp |= isVectorOp(I);
3964
3965 // Account for the cost of duplicating this instruction into each
3966 // predecessor. Ignore free instructions.
3967 if (!TTI || TTI->getInstructionCost(&I, CostKind) !=
3969 NumBonusInsts += PredCount;
3970
3971 // Early exits once we reach the limit.
3972 if (NumBonusInsts >
3973 BonusInstThreshold * BranchFoldToCommonDestVectorMultiplier)
3974 return false;
3975 }
3976
3977 auto IsBCSSAUse = [BB, &I](Use &U) {
3978 auto *UI = cast<Instruction>(U.getUser());
3979 if (auto *PN = dyn_cast<PHINode>(UI))
3980 return PN->getIncomingBlock(U) == BB;
3981 return UI->getParent() == BB && I.comesBefore(UI);
3982 };
3983
3984 // Does this instruction require rewriting of uses?
3985 if (!all_of(I.uses(), IsBCSSAUse))
3986 return false;
3987 }
3988 if (NumBonusInsts >
3989 BonusInstThreshold *
3990 (SawVectorOp ? BranchFoldToCommonDestVectorMultiplier : 1))
3991 return false;
3992
3993 // Ok, we have the budget. Perform the transformation.
3994 for (BasicBlock *PredBlock : Preds) {
3995 auto *PBI = cast<BranchInst>(PredBlock->getTerminator());
3996 return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI);
3997 }
3998 return false;
3999}
4000
4001// If there is only one store in BB1 and BB2, return it, otherwise return
4002// nullptr.
4004 StoreInst *S = nullptr;
4005 for (auto *BB : {BB1, BB2}) {
4006 if (!BB)
4007 continue;
4008 for (auto &I : *BB)
4009 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4010 if (S)
4011 // Multiple stores seen.
4012 return nullptr;
4013 else
4014 S = SI;
4015 }
4016 }
4017 return S;
4018}
4019
4021 Value *AlternativeV = nullptr) {
4022 // PHI is going to be a PHI node that allows the value V that is defined in
4023 // BB to be referenced in BB's only successor.
4024 //
4025 // If AlternativeV is nullptr, the only value we care about in PHI is V. It
4026 // doesn't matter to us what the other operand is (it'll never get used). We
4027 // could just create a new PHI with an undef incoming value, but that could
4028 // increase register pressure if EarlyCSE/InstCombine can't fold it with some
4029 // other PHI. So here we directly look for some PHI in BB's successor with V
4030 // as an incoming operand. If we find one, we use it, else we create a new
4031 // one.
4032 //
4033 // If AlternativeV is not nullptr, we care about both incoming values in PHI.
4034 // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV]
4035 // where OtherBB is the single other predecessor of BB's only successor.
4036 PHINode *PHI = nullptr;
4037 BasicBlock *Succ = BB->getSingleSuccessor();
4038
4039 for (auto I = Succ->begin(); isa<PHINode>(I); ++I)
4040 if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) {
4041 PHI = cast<PHINode>(I);
4042 if (!AlternativeV)
4043 break;
4044
4045 assert(Succ->hasNPredecessors(2));
4046 auto PredI = pred_begin(Succ);
4047 BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI;
4048 if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV)
4049 break;
4050 PHI = nullptr;
4051 }
4052 if (PHI)
4053 return PHI;
4054
4055 // If V is not an instruction defined in BB, just return it.
4056 if (!AlternativeV &&
4057 (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB))
4058 return V;
4059
4060 PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge");
4061 PHI->insertBefore(Succ->begin());
4062 PHI->addIncoming(V, BB);
4063 for (BasicBlock *PredBB : predecessors(Succ))
4064 if (PredBB != BB)
4065 PHI->addIncoming(
4066 AlternativeV ? AlternativeV : PoisonValue::get(V->getType()), PredBB);
4067 return PHI;
4068}
4069
4071 BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB,
4072 BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond,
4073 DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI) {
4074 // For every pointer, there must be exactly two stores, one coming from
4075 // PTB or PFB, and the other from QTB or QFB. We don't support more than one
4076 // store (to any address) in PTB,PFB or QTB,QFB.
4077 // FIXME: We could relax this restriction with a bit more work and performance
4078 // testing.
4079 StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB);
4080 StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB);
4081 if (!PStore || !QStore)
4082 return false;
4083
4084 // Now check the stores are compatible.
4085 if (!QStore->isUnordered() || !PStore->isUnordered() ||
4086 PStore->getValueOperand()->getType() !=
4087 QStore->getValueOperand()->getType())
4088 return false;
4089
4090 // Check that sinking the store won't cause program behavior changes. Sinking
4091 // the store out of the Q blocks won't change any behavior as we're sinking
4092 // from a block to its unconditional successor. But we're moving a store from
4093 // the P blocks down through the middle block (QBI) and past both QFB and QTB.
4094 // So we need to check that there are no aliasing loads or stores in
4095 // QBI, QTB and QFB. We also need to check there are no conflicting memory
4096 // operations between PStore and the end of its parent block.
4097 //
4098 // The ideal way to do this is to query AliasAnalysis, but we don't
4099 // preserve AA currently so that is dangerous. Be super safe and just
4100 // check there are no other memory operations at all.
4101 for (auto &I : *QFB->getSinglePredecessor())
4102 if (I.mayReadOrWriteMemory())
4103 return false;
4104 for (auto &I : *QFB)
4105 if (&I != QStore && I.mayReadOrWriteMemory())
4106 return false;
4107 if (QTB)
4108 for (auto &I : *QTB)
4109 if (&I != QStore && I.mayReadOrWriteMemory())
4110 return false;
4111 for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end();
4112 I != E; ++I)
4113 if (&*I != PStore && I->mayReadOrWriteMemory())
4114 return false;
4115
4116 // If we're not in aggressive mode, we only optimize if we have some
4117 // confidence that by optimizing we'll allow P and/or Q to be if-converted.
4118 auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) {
4119 if (!BB)
4120 return true;
4121 // Heuristic: if the block can be if-converted/phi-folded and the
4122 // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
4123 // thread this store.
4125 InstructionCost Budget =
4127 for (auto &I : BB->instructionsWithoutDebug(false)) {
4128 // Consider terminator instruction to be free.
4129 if (I.isTerminator())
4130 continue;
4131 // If this is one the stores that we want to speculate out of this BB,
4132 // then don't count it's cost, consider it to be free.
4133 if (auto *S = dyn_cast<StoreInst>(&I))
4134 if (llvm::find(FreeStores, S))
4135 continue;
4136 // Else, we have a white-list of instructions that we are ak speculating.
4137 if (!isa<BinaryOperator>(I) && !isa<GetElementPtrInst>(I))
4138 return false; // Not in white-list - not worthwhile folding.
4139 // And finally, if this is a non-free instruction that we are okay
4140 // speculating, ensure that we consider the speculation budget.
4141 Cost +=
4143 if (Cost > Budget)
4144 return false; // Eagerly refuse to fold as soon as we're out of budget.
4145 }
4146 assert(Cost <= Budget &&
4147 "When we run out of budget we will eagerly return from within the "
4148 "per-instruction loop.");
4149 return true;
4150 };
4151
4152 const std::array<StoreInst *, 2> FreeStores = {PStore, QStore};
4154 (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) ||
4155 !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores)))
4156 return false;
4157
4158 // If PostBB has more than two predecessors, we need to split it so we can
4159 // sink the store.
4160 if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) {
4161 // We know that QFB's only successor is PostBB. And QFB has a single
4162 // predecessor. If QTB exists, then its only successor is also PostBB.
4163 // If QTB does not exist, then QFB's only predecessor has a conditional
4164 // branch to QFB and PostBB.
4165 BasicBlock *TruePred = QTB ? QTB : QFB->getSinglePredecessor();
4166 BasicBlock *NewBB =
4167 SplitBlockPredecessors(PostBB, {QFB, TruePred}, "condstore.split", DTU);
4168 if (!NewBB)
4169 return false;
4170 PostBB = NewBB;
4171 }
4172
4173 // OK, we're going to sink the stores to PostBB. The store has to be
4174 // conditional though, so first create the predicate.
4175 Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator())
4176 ->getCondition();
4177 Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator())
4178 ->getCondition();
4179
4181 PStore->getParent());
4183 QStore->getParent(), PPHI);
4184
4185 BasicBlock::iterator PostBBFirst = PostBB->getFirstInsertionPt();
4186 IRBuilder<> QB(PostBB, PostBBFirst);
4187 QB.SetCurrentDebugLocation(PostBBFirst->getStableDebugLoc());
4188
4189 Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond);
4190 Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond);
4191
4192 if (InvertPCond)
4193 PPred = QB.CreateNot(PPred);
4194 if (InvertQCond)
4195 QPred = QB.CreateNot(QPred);
4196 Value *CombinedPred = QB.CreateOr(PPred, QPred);
4197
4198 BasicBlock::iterator InsertPt = QB.GetInsertPoint();
4199 auto *T = SplitBlockAndInsertIfThen(CombinedPred, InsertPt,
4200 /*Unreachable=*/false,
4201 /*BranchWeights=*/nullptr, DTU);
4202
4203 QB.SetInsertPoint(T);
4204 StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address));
4205 SI->setAAMetadata(PStore->getAAMetadata().merge(QStore->getAAMetadata()));
4206 // Choose the minimum alignment. If we could prove both stores execute, we
4207 // could use biggest one. In this case, though, we only know that one of the
4208 // stores executes. And we don't know it's safe to take the alignment from a
4209 // store that doesn't execute.
4210 SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign()));
4211
4212 QStore->eraseFromParent();
4213 PStore->eraseFromParent();
4214
4215 return true;
4216}
4217
4219 DomTreeUpdater *DTU, const DataLayout &DL,
4220 const TargetTransformInfo &TTI) {
4221 // The intention here is to find diamonds or triangles (see below) where each
4222 // conditional block contains a store to the same address. Both of these
4223 // stores are conditional, so they can't be unconditionally sunk. But it may
4224 // be profitable to speculatively sink the stores into one merged store at the
4225 // end, and predicate the merged store on the union of the two conditions of
4226 // PBI and QBI.
4227 //
4228 // This can reduce the number of stores executed if both of the conditions are
4229 // true, and can allow the blocks to become small enough to be if-converted.
4230 // This optimization will also chain, so that ladders of test-and-set
4231 // sequences can be if-converted away.
4232 //
4233 // We only deal with simple diamonds or triangles:
4234 //
4235 // PBI or PBI or a combination of the two
4236 // / \ | \
4237 // PTB PFB | PFB
4238 // \ / | /
4239 // QBI QBI
4240 // / \ | \
4241 // QTB QFB | QFB
4242 // \ / | /
4243 // PostBB PostBB
4244 //
4245 // We model triangles as a type of diamond with a nullptr "true" block.
4246 // Triangles are canonicalized so that the fallthrough edge is represented by
4247 // a true condition, as in the diagram above.
4248 BasicBlock *PTB = PBI->getSuccessor(0);
4249 BasicBlock *PFB = PBI->getSuccessor(1);
4250 BasicBlock *QTB = QBI->getSuccessor(0);
4251 BasicBlock *QFB = QBI->getSuccessor(1);
4252 BasicBlock *PostBB = QFB->getSingleSuccessor();
4253
4254 // Make sure we have a good guess for PostBB. If QTB's only successor is
4255 // QFB, then QFB is a better PostBB.
4256 if (QTB->getSingleSuccessor() == QFB)
4257 PostBB = QFB;
4258
4259 // If we couldn't find a good PostBB, stop.
4260 if (!PostBB)
4261 return false;
4262
4263 bool InvertPCond = false, InvertQCond = false;
4264 // Canonicalize fallthroughs to the true branches.
4265 if (PFB == QBI->getParent()) {
4266 std::swap(PFB, PTB);
4267 InvertPCond = true;
4268 }
4269 if (QFB == PostBB) {
4270 std::swap(QFB, QTB);
4271 InvertQCond = true;
4272 }
4273
4274 // From this point on we can assume PTB or QTB may be fallthroughs but PFB
4275 // and QFB may not. Model fallthroughs as a nullptr block.
4276 if (PTB == QBI->getParent())
4277 PTB = nullptr;
4278 if (QTB == PostBB)
4279 QTB = nullptr;
4280
4281 // Legality bailouts. We must have at least the non-fallthrough blocks and
4282 // the post-dominating block, and the non-fallthroughs must only have one
4283 // predecessor.
4284 auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) {
4285 return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S;
4286 };
4287 if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) ||
4288 !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB))
4289 return false;
4290 if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) ||
4291 (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB)))
4292 return false;
4293 if (!QBI->getParent()->hasNUses(2))
4294 return false;
4295
4296 // OK, this is a sequence of two diamonds or triangles.
4297 // Check if there are stores in PTB or PFB that are repeated in QTB or QFB.
4298 SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses;
4299 for (auto *BB : {PTB, PFB}) {
4300 if (!BB)
4301 continue;
4302 for (auto &I : *BB)
4303 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
4304 PStoreAddresses.insert(SI->getPointerOperand());
4305 }
4306 for (auto *BB : {QTB, QFB}) {
4307 if (!BB)
4308 continue;
4309 for (auto &I : *BB)
4310 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
4311 QStoreAddresses.insert(SI->getPointerOperand());
4312 }
4313
4314 set_intersect(PStoreAddresses, QStoreAddresses);
4315 // set_intersect mutates PStoreAddresses in place. Rename it here to make it
4316 // clear what it contains.
4317 auto &CommonAddresses = PStoreAddresses;
4318
4319 bool Changed = false;
4320 for (auto *Address : CommonAddresses)
4321 Changed |=
4322 mergeConditionalStoreToAddress(PTB, PFB, QTB, QFB, PostBB, Address,
4323 InvertPCond, InvertQCond, DTU, DL, TTI);
4324 return Changed;
4325}
4326
4327/// If the previous block ended with a widenable branch, determine if reusing
4328/// the target block is profitable and legal. This will have the effect of
4329/// "widening" PBI, but doesn't require us to reason about hosting safety.
4331 DomTreeUpdater *DTU) {
4332 // TODO: This can be generalized in two important ways:
4333 // 1) We can allow phi nodes in IfFalseBB and simply reuse all the input
4334 // values from the PBI edge.
4335 // 2) We can sink side effecting instructions into BI's fallthrough
4336 // successor provided they doesn't contribute to computation of
4337 // BI's condition.
4338 BasicBlock *IfTrueBB = PBI->getSuccessor(0);
4339 BasicBlock *IfFalseBB = PBI->getSuccessor(1);
4340 if (!isWidenableBranch(PBI) || IfTrueBB != BI->getParent() ||
4342 return false;
4343 if (!IfFalseBB->phis().empty())
4344 return false; // TODO
4345 // This helps avoid infinite loop with SimplifyCondBranchToCondBranch which
4346 // may undo the transform done here.
4347 // TODO: There might be a more fine-grained solution to this.
4348 if (!llvm::succ_empty(IfFalseBB))
4349 return false;
4350 // Use lambda to lazily compute expensive condition after cheap ones.
4351 auto NoSideEffects = [](BasicBlock &BB) {
4352 return llvm::none_of(BB, [](const Instruction &I) {
4353 return I.mayWriteToMemory() || I.mayHaveSideEffects();
4354 });
4355 };
4356 if (BI->getSuccessor(1) != IfFalseBB && // no inf looping
4357 BI->getSuccessor(1)->getTerminatingDeoptimizeCall() && // profitability
4358 NoSideEffects(*BI->getParent())) {
4359 auto *OldSuccessor = BI->getSuccessor(1);
4360 OldSuccessor->removePredecessor(BI->getParent());
4361 BI->setSuccessor(1, IfFalseBB);
4362 if (DTU)
4363 DTU->applyUpdates(
4364 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
4365 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
4366 return true;
4367 }
4368 if (BI->getSuccessor(0) != IfFalseBB && // no inf looping
4369 BI->getSuccessor(0)->getTerminatingDeoptimizeCall() && // profitability
4370 NoSideEffects(*BI->getParent())) {
4371 auto *OldSuccessor = BI->getSuccessor(0);
4372 OldSuccessor->removePredecessor(BI->getParent());
4373 BI->setSuccessor(0, IfFalseBB);
4374 if (DTU)
4375 DTU->applyUpdates(
4376 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
4377 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
4378 return true;
4379 }
4380 return false;
4381}
4382
4383/// If we have a conditional branch as a predecessor of another block,
4384/// this function tries to simplify it. We know
4385/// that PBI and BI are both conditional branches, and BI is in one of the
4386/// successor blocks of PBI - PBI branches to BI.
4388 DomTreeUpdater *DTU,
4389 const DataLayout &DL,
4390 const TargetTransformInfo &TTI) {
4391 assert(PBI->isConditional() && BI->isConditional());
4392 BasicBlock *BB = BI->getParent();
4393
4394 // If this block ends with a branch instruction, and if there is a
4395 // predecessor that ends on a branch of the same condition, make
4396 // this conditional branch redundant.
4397 if (PBI->getCondition() == BI->getCondition() &&
4398 PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
4399 // Okay, the outcome of this conditional branch is statically
4400 // knowable. If this block had a single pred, handle specially, otherwise
4401 // FoldCondBranchOnValueKnownInPredecessor() will handle it.
4402 if (BB->getSinglePredecessor()) {
4403 // Turn this into a branch on constant.
4404 bool CondIsTrue = PBI->getSuccessor(0) == BB;
4405 BI->setCondition(
4406 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue));
4407 return true; // Nuke the branch on constant.
4408 }
4409 }
4410
4411 // If the previous block ended with a widenable branch, determine if reusing
4412 // the target block is profitable and legal. This will have the effect of
4413 // "widening" PBI, but doesn't require us to reason about hosting safety.
4414 if (tryWidenCondBranchToCondBranch(PBI, BI, DTU))
4415 return true;
4416
4417 // If both branches are conditional and both contain stores to the same
4418 // address, remove the stores from the conditionals and create a conditional
4419 // merged store at the end.
4420 if (MergeCondStores && mergeConditionalStores(PBI, BI, DTU, DL, TTI))
4421 return true;
4422
4423 // If this is a conditional branch in an empty block, and if any
4424 // predecessors are a conditional branch to one of our destinations,
4425 // fold the conditions into logical ops and one cond br.
4426
4427 // Ignore dbg intrinsics.
4428 if (&*BB->instructionsWithoutDebug(false).begin() != BI)
4429 return false;
4430
4431 int PBIOp, BIOp;
4432 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
4433 PBIOp = 0;
4434 BIOp = 0;
4435 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
4436 PBIOp = 0;
4437 BIOp = 1;
4438 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
4439 PBIOp = 1;
4440 BIOp = 0;
4441 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
4442 PBIOp = 1;
4443 BIOp = 1;
4444 } else {
4445 return false;
4446 }
4447
4448 // Check to make sure that the other destination of this branch
4449 // isn't BB itself. If so, this is an infinite loop that will
4450 // keep getting unwound.
4451 if (PBI->getSuccessor(PBIOp) == BB)
4452 return false;
4453
4454 // If predecessor's branch probability to BB is too low don't merge branches.
4455 SmallVector<uint32_t, 2> PredWeights;
4456 if (!PBI->getMetadata(LLVMContext::MD_unpredictable) &&
4457 extractBranchWeights(*PBI, PredWeights) &&
4458 (static_cast<uint64_t>(PredWeights[0]) + PredWeights[1]) != 0) {
4459
4461 PredWeights[PBIOp],
4462 static_cast<uint64_t>(PredWeights[0]) + PredWeights[1]);
4463
4465 if (CommonDestProb >= Likely)
4466 return false;
4467 }
4468
4469 // Do not perform this transformation if it would require
4470 // insertion of a large number of select instructions. For targets
4471 // without predication/cmovs, this is a big pessimization.
4472
4473 BasicBlock *CommonDest = PBI->getSuccessor(PBIOp);
4474 BasicBlock *RemovedDest = PBI->getSuccessor(PBIOp ^ 1);
4475 unsigned NumPhis = 0;
4476 for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II);
4477 ++II, ++NumPhis) {
4478 if (NumPhis > 2) // Disable this xform.
4479 return false;
4480 }
4481
4482 // Finally, if everything is ok, fold the branches to logical ops.
4483 BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1);
4484
4485 LLVM_DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()
4486 << "AND: " << *BI->getParent());
4487
4489
4490 // If OtherDest *is* BB, then BB is a basic block with a single conditional
4491 // branch in it, where one edge (OtherDest) goes back to itself but the other
4492 // exits. We don't *know* that the program avoids the infinite loop
4493 // (even though that seems likely). If we do this xform naively, we'll end up
4494 // recursively unpeeling the loop. Since we know that (after the xform is
4495 // done) that the block *is* infinite if reached, we just make it an obviously
4496 // infinite loop with no cond branch.
4497 if (OtherDest == BB) {
4498 // Insert it at the end of the function, because it's either code,
4499 // or it won't matter if it's hot. :)
4500 BasicBlock *InfLoopBlock =
4501 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
4502 BranchInst::Create(InfLoopBlock, InfLoopBlock);
4503 if (DTU)
4504 Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
4505 OtherDest = InfLoopBlock;
4506 }
4507
4508 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent());
4509
4510 // BI may have other predecessors. Because of this, we leave
4511 // it alone, but modify PBI.
4512
4513 // Make sure we get to CommonDest on True&True directions.
4514 Value *PBICond = PBI->getCondition();
4515 IRBuilder<NoFolder> Builder(PBI);
4516 if (PBIOp)
4517 PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not");
4518
4519 Value *BICond = BI->getCondition();
4520 if (BIOp)
4521 BICond = Builder.CreateNot(BICond, BICond->getName() + ".not");
4522
4523 // Merge the conditions.
4524 Value *Cond =
4525 createLogicalOp(Builder, Instruction::Or, PBICond, BICond, "brmerge");
4526
4527 // Modify PBI to branch on the new condition to the new dests.
4528 PBI->setCondition(Cond);
4529 PBI->setSuccessor(0, CommonDest);
4530 PBI->setSuccessor(1, OtherDest);
4531
4532 if (DTU) {
4533 Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest});
4534 Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest});
4535
4536 DTU->applyUpdates(Updates);
4537 }
4538
4539 // Update branch weight for PBI.
4540 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
4541 uint64_t PredCommon, PredOther, SuccCommon, SuccOther;
4542 bool HasWeights =
4543 extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
4544 SuccTrueWeight, SuccFalseWeight);
4545 if (HasWeights) {
4546 PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
4547 PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
4548 SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
4549 SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
4550 // The weight to CommonDest should be PredCommon * SuccTotal +
4551 // PredOther * SuccCommon.
4552 // The weight to OtherDest should be PredOther * SuccOther.
4553 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) +
4554 PredOther * SuccCommon,
4555 PredOther * SuccOther};
4556 // Halve the weights if any of them cannot fit in an uint32_t
4557 FitWeights(NewWeights);
4558
4559 setBranchWeights(PBI, NewWeights[0], NewWeights[1], /*IsExpected=*/false);
4560 }
4561
4562 // OtherDest may have phi nodes. If so, add an entry from PBI's
4563 // block that are identical to the entries for BI's block.
4564 AddPredecessorToBlock(OtherDest, PBI->getParent(), BB);
4565
4566 // We know that the CommonDest already had an edge from PBI to
4567 // it. If it has PHIs though, the PHIs may have different
4568 // entries for BB and PBI's BB. If so, insert a select to make
4569 // them agree.
4570 for (PHINode &PN : CommonDest->phis()) {
4571 Value *BIV = PN.getIncomingValueForBlock(BB);
4572 unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent());
4573 Value *PBIV = PN.getIncomingValue(PBBIdx);
4574 if (BIV != PBIV) {
4575 // Insert a select in PBI to pick the right value.
4576 SelectInst *NV = cast<SelectInst>(
4577 Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux"));
4578 PN.setIncomingValue(PBBIdx, NV);
4579 // Although the select has the same condition as PBI, the original branch
4580 // weights for PBI do not apply to the new select because the select's
4581 // 'logical' edges are incoming edges of the phi that is eliminated, not
4582 // the outgoing edges of PBI.
4583 if (HasWeights) {
4584 uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
4585 uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
4586 uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
4587 uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
4588 // The weight to PredCommonDest should be PredCommon * SuccTotal.
4589 // The weight to PredOtherDest should be PredOther * SuccCommon.
4590 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther),
4591 PredOther * SuccCommon};
4592
4593 FitWeights(NewWeights);
4594
4595 setBranchWeights(NV, NewWeights[0], NewWeights[1],
4596 /*IsExpected=*/false);
4597 }
4598 }
4599 }
4600
4601 LLVM_DEBUG(dbgs() << "INTO: " << *PBI->getParent());
4602 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent());
4603
4604 // This basic block is probably dead. We know it has at least
4605 // one fewer predecessor.
4606 return true;
4607}
4608
4609// Simplifies a terminator by replacing it with a branch to TrueBB if Cond is
4610// true or to FalseBB if Cond is false.
4611// Takes care of updating the successors and removing the old terminator.
4612// Also makes sure not to introduce new successors by assuming that edges to
4613// non-successor TrueBBs and FalseBBs aren't reachable.
4614bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm,
4615 Value *Cond, BasicBlock *TrueBB,
4616 BasicBlock *FalseBB,
4617 uint32_t TrueWeight,
4618 uint32_t FalseWeight) {
4619 auto *BB = OldTerm->getParent();
4620 // Remove any superfluous successor edges from the CFG.
4621 // First, figure out which successors to preserve.
4622 // If TrueBB and FalseBB are equal, only try to preserve one copy of that
4623 // successor.
4624 BasicBlock *KeepEdge1 = TrueBB;
4625 BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr;
4626
4627 SmallSetVector<BasicBlock *, 2> RemovedSuccessors;
4628
4629 // Then remove the rest.
4630 for (BasicBlock *Succ : successors(OldTerm)) {
4631 // Make sure only to keep exactly one copy of each edge.
4632 if (Succ == KeepEdge1)
4633 KeepEdge1 = nullptr;
4634 else if (Succ == KeepEdge2)
4635 KeepEdge2 = nullptr;
4636 else {
4637 Succ->removePredecessor(BB,
4638 /*KeepOneInputPHIs=*/true);
4639
4640 if (Succ != TrueBB && Succ != FalseBB)
4641 RemovedSuccessors.insert(Succ);
4642 }
4643 }
4644
4645 IRBuilder<> Builder(OldTerm);
4646 Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc());
4647
4648 // Insert an appropriate new terminator.
4649 if (!KeepEdge1 && !KeepEdge2) {
4650 if (TrueBB == FalseBB) {
4651 // We were only looking for one successor, and it was present.
4652 // Create an unconditional branch to it.
4653 Builder.CreateBr(TrueBB);
4654 } else {
4655 // We found both of the successors we were looking for.
4656 // Create a conditional branch sharing the condition of the select.
4657 BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
4658 if (TrueWeight != FalseWeight)
4659 setBranchWeights(NewBI, TrueWeight, FalseWeight, /*IsExpected=*/false);
4660 }
4661 } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
4662 // Neither of the selected blocks were successors, so this
4663 // terminator must be unreachable.
4664 new UnreachableInst(OldTerm->getContext(), OldTerm->getIterator());
4665 } else {
4666 // One of the selected values was a successor, but the other wasn't.
4667 // Insert an unconditional branch to the one that was found;
4668 // the edge to the one that wasn't must be unreachable.
4669 if (!KeepEdge1) {
4670 // Only TrueBB was found.
4671 Builder.CreateBr(TrueBB);
4672 } else {
4673 // Only FalseBB was found.
4674 Builder.CreateBr(FalseBB);
4675 }
4676 }
4677
4679
4680 if (DTU) {
4682 Updates.reserve(RemovedSuccessors.size());
4683 for (auto *RemovedSuccessor : RemovedSuccessors)
4684 Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor});
4685 DTU->applyUpdates(Updates);
4686 }
4687
4688 return true;
4689}
4690
4691// Replaces
4692// (switch (select cond, X, Y)) on constant X, Y
4693// with a branch - conditional if X and Y lead to distinct BBs,
4694// unconditional otherwise.
4695bool SimplifyCFGOpt::SimplifySwitchOnSelect(SwitchInst *SI,
4696 SelectInst *Select) {
4697 // Check for constant integer values in the select.
4698 ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue());
4699 ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue());
4700 if (!TrueVal || !FalseVal)
4701 return false;
4702
4703 // Find the relevant condition and destinations.
4704 Value *Condition = Select->getCondition();
4705 BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor();
4706 BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor();
4707
4708 // Get weight for TrueBB and FalseBB.
4709 uint32_t TrueWeight = 0, FalseWeight = 0;
4711 bool HasWeights = hasBranchWeightMD(*SI);
4712 if (HasWeights) {
4713 GetBranchWeights(SI, Weights);
4714 if (Weights.size() == 1 + SI->getNumCases()) {
4715 TrueWeight =
4716 (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()];
4717 FalseWeight =
4718 (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()];
4719 }
4720 }
4721
4722 // Perform the actual simplification.
4723 return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight,
4724 FalseWeight);
4725}
4726
4727// Replaces
4728// (indirectbr (select cond, blockaddress(@fn, BlockA),
4729// blockaddress(@fn, BlockB)))
4730// with
4731// (br cond, BlockA, BlockB).
4732bool SimplifyCFGOpt::SimplifyIndirectBrOnSelect(IndirectBrInst *IBI,
4733 SelectInst *SI) {
4734 // Check that both operands of the select are block addresses.
4735 BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue());
4736 BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue());
4737 if (!TBA || !FBA)
4738 return false;
4739
4740 // Extract the actual blocks.
4741 BasicBlock *TrueBB = TBA->getBasicBlock();
4742 BasicBlock *FalseBB = FBA->getBasicBlock();
4743
4744 // Perform the actual simplification.
4745 return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0,
4746 0);
4747}
4748
4749/// This is called when we find an icmp instruction
4750/// (a seteq/setne with a constant) as the only instruction in a
4751/// block that ends with an uncond branch. We are looking for a very specific
4752/// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In
4753/// this case, we merge the first two "or's of icmp" into a switch, but then the
4754/// default value goes to an uncond block with a seteq in it, we get something
4755/// like:
4756///
4757/// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ]
4758/// DEFAULT:
4759/// %tmp = icmp eq i8 %A, 92
4760/// br label %end
4761/// end:
4762/// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ]
4763///
4764/// We prefer to split the edge to 'end' so that there is a true/false entry to
4765/// the PHI, merging the third icmp into the switch.
4766bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt(
4767 ICmpInst *ICI, IRBuilder<> &Builder) {
4768 BasicBlock *BB = ICI->getParent();
4769
4770 // If the block has any PHIs in it or the icmp has multiple uses, it is too
4771 // complex.
4772 if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse())
4773 return false;
4774
4775 Value *V = ICI->getOperand(0);
4776 ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1));
4777
4778 // The pattern we're looking for is where our only predecessor is a switch on
4779 // 'V' and this block is the default case for the switch. In this case we can
4780 // fold the compared value into the switch to simplify things.
4781 BasicBlock *Pred = BB->getSinglePredecessor();
4782 if (!Pred || !isa<SwitchInst>(Pred->getTerminator()))
4783 return false;
4784
4785 SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator());
4786 if (SI->getCondition() != V)
4787 return false;
4788
4789 // If BB is reachable on a non-default case, then we simply know the value of
4790 // V in this block. Substitute it and constant fold the icmp instruction
4791 // away.
4792 if (SI->getDefaultDest() != BB) {
4793 ConstantInt *VVal = SI->findCaseDest(BB);
4794 assert(VVal && "Should have a unique destination value");
4795 ICI->setOperand(0, VVal);
4796
4797 if (Value *V = simplifyInstruction(ICI, {DL, ICI})) {
4798 ICI->replaceAllUsesWith(V);
4799 ICI->eraseFromParent();
4800 }
4801 // BB is now empty, so it is likely to simplify away.
4802 return requestResimplify();
4803 }
4804
4805 // Ok, the block is reachable from the default dest. If the constant we're
4806 // comparing exists in one of the other edges, then we can constant fold ICI
4807 // and zap it.
4808 if (SI->findCaseValue(Cst) != SI->case_default()) {
4809 Value *V;
4810 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
4812 else
4814
4815 ICI->replaceAllUsesWith(V);
4816 ICI->eraseFromParent();
4817 // BB is now empty, so it is likely to simplify away.
4818 return requestResimplify();
4819 }
4820
4821 // The use of the icmp has to be in the 'end' block, by the only PHI node in
4822 // the block.
4823 BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0);
4824 PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back());
4825 if (PHIUse == nullptr || PHIUse != &SuccBlock->front() ||
4826 isa<PHINode>(++BasicBlock::iterator(PHIUse)))
4827 return false;
4828
4829 // If the icmp is a SETEQ, then the default dest gets false, the new edge gets
4830 // true in the PHI.
4831 Constant *DefaultCst = ConstantInt::getTrue(BB->getContext());
4832 Constant *NewCst = ConstantInt::getFalse(BB->getContext());
4833
4834 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
4835 std::swap(DefaultCst, NewCst);
4836
4837 // Replace ICI (which is used by the PHI for the default value) with true or
4838 // false depending on if it is EQ or NE.
4839 ICI->replaceAllUsesWith(DefaultCst);
4840 ICI->eraseFromParent();
4841
4843
4844 // Okay, the switch goes to this block on a default value. Add an edge from
4845 // the switch to the merge point on the compared value.
4846 BasicBlock *NewBB =
4847 BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB);
4848 {
4850 auto W0 = SIW.getSuccessorWeight(0);
4852 if (W0) {
4853 NewW = ((uint64_t(*W0) + 1) >> 1);
4854 SIW.setSuccessorWeight(0, *NewW);
4855 }
4856 SIW.addCase(Cst, NewBB, NewW);
4857 if (DTU)
4858 Updates.push_back({DominatorTree::Insert, Pred, NewBB});
4859 }
4860
4861 // NewBB branches to the phi block, add the uncond branch and the phi entry.
4862 Builder.SetInsertPoint(NewBB);
4863 Builder.SetCurrentDebugLocation(SI->getDebugLoc());
4864 Builder.CreateBr(SuccBlock);
4865 PHIUse->addIncoming(NewCst, NewBB);
4866 if (DTU) {
4867 Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock});
4868 DTU->applyUpdates(Updates);
4869 }
4870 return true;
4871}
4872
4873/// The specified branch is a conditional branch.
4874/// Check to see if it is branching on an or/and chain of icmp instructions, and
4875/// fold it into a switch instruction if so.
4876bool SimplifyCFGOpt::SimplifyBranchOnICmpChain(BranchInst *BI,
4877 IRBuilder<> &Builder,
4878 const DataLayout &DL) {
4879 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
4880 if (!Cond)
4881 return false;
4882
4883 // Change br (X == 0 | X == 1), T, F into a switch instruction.
4884 // If this is a bunch of seteq's or'd together, or if it's a bunch of
4885 // 'setne's and'ed together, collect them.
4886
4887 // Try to gather values from a chain of and/or to be turned into a switch
4888 ConstantComparesGatherer ConstantCompare(Cond, DL);
4889 // Unpack the result
4890 SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals;
4891 Value *CompVal = ConstantCompare.CompValue;
4892 unsigned UsedICmps = ConstantCompare.UsedICmps;
4893 Value *ExtraCase = ConstantCompare.Extra;
4894
4895 // If we didn't have a multiply compared value, fail.
4896 if (!CompVal)
4897 return false;
4898
4899 // Avoid turning single icmps into a switch.
4900 if (UsedICmps <= 1)
4901 return false;
4902
4903 bool TrueWhenEqual = match(Cond, m_LogicalOr(m_Value(), m_Value()));
4904
4905 // There might be duplicate constants in the list, which the switch
4906 // instruction can't handle, remove them now.
4907 array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate);
4908 Values.erase(std::unique(Values.begin(), Values.end()), Values.end());
4909
4910 // If Extra was used, we require at least two switch values to do the
4911 // transformation. A switch with one value is just a conditional branch.
4912 if (ExtraCase && Values.size() < 2)
4913 return false;
4914
4915 // TODO: Preserve branch weight metadata, similarly to how
4916 // FoldValueComparisonIntoPredecessors preserves it.
4917
4918 // Figure out which block is which destination.
4919 BasicBlock *DefaultBB = BI->getSuccessor(1);
4920 BasicBlock *EdgeBB = BI->getSuccessor(0);
4921 if (!TrueWhenEqual)
4922 std::swap(DefaultBB, EdgeBB);
4923
4924 BasicBlock *BB = BI->getParent();
4925
4926 LLVM_DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()
4927 << " cases into SWITCH. BB is:\n"
4928 << *BB);
4929
4931
4932 // If there are any extra values that couldn't be folded into the switch
4933 // then we evaluate them with an explicit branch first. Split the block
4934 // right before the condbr to handle it.
4935 if (ExtraCase) {
4936 BasicBlock *NewBB = SplitBlock(BB, BI, DTU, /*LI=*/nullptr,
4937 /*MSSAU=*/nullptr, "switch.early.test");
4938
4939 // Remove the uncond branch added to the old block.
4940 Instruction *OldTI = BB->getTerminator();
4941 Builder.SetInsertPoint(OldTI);
4942
4943 // There can be an unintended UB if extra values are Poison. Before the
4944 // transformation, extra values may not be evaluated according to the
4945 // condition, and it will not raise UB. But after transformation, we are
4946 // evaluating extra values before checking the condition, and it will raise
4947 // UB. It can be solved by adding freeze instruction to extra values.
4948 AssumptionCache *AC = Options.AC;
4949
4950 if (!isGuaranteedNotToBeUndefOrPoison(ExtraCase, AC, BI, nullptr))
4951 ExtraCase = Builder.CreateFreeze(ExtraCase);
4952
4953 if (TrueWhenEqual)
4954 Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB);
4955 else
4956 Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB);
4957
4958 OldTI->eraseFromParent();
4959
4960 if (DTU)
4961 Updates.push_back({DominatorTree::Insert, BB, EdgeBB});
4962
4963 // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
4964 // for the edge we just added.
4965 AddPredecessorToBlock(EdgeBB, BB, NewBB);
4966
4967 LLVM_DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCase
4968 << "\nEXTRABB = " << *BB);
4969 BB = NewBB;
4970 }
4971
4972 Builder.SetInsertPoint(BI);
4973 // Convert pointer to int before we switch.
4974 if (CompVal->getType()->isPointerTy()) {
4975 CompVal = Builder.CreatePtrToInt(
4976 CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr");
4977 }
4978
4979 // Create the new switch instruction now.
4980 SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size());
4981
4982 // Add all of the 'cases' to the switch instruction.
4983 for (unsigned i = 0, e = Values.size(); i != e; ++i)
4984 New->addCase(Values[i], EdgeBB);
4985
4986 // We added edges from PI to the EdgeBB. As such, if there were any
4987 // PHI nodes in EdgeBB, they need entries to be added corresponding to
4988 // the number of edges added.
4989 for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) {
4990 PHINode *PN = cast<PHINode>(BBI);
4991 Value *InVal = PN->getIncomingValueForBlock(BB);
4992 for (unsigned i = 0, e = Values.size() - 1; i != e; ++i)
4993 PN->addIncoming(InVal, BB);
4994 }
4995
4996 // Erase the old branch instruction.
4998 if (DTU)
4999 DTU->applyUpdates(Updates);
5000
5001 LLVM_DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n');
5002 return true;
5003}
5004
5005bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
5006 if (isa<PHINode>(RI->getValue()))
5007 return simplifyCommonResume(RI);
5008 else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) &&
5009 RI->getValue() == RI->getParent()->getFirstNonPHI())
5010 // The resume must unwind the exception that caused control to branch here.
5011 return simplifySingleResume(RI);
5012
5013 return false;
5014}
5015
5016// Check if cleanup block is empty
5018 for (Instruction &I : R) {
5019 auto *II = dyn_cast<IntrinsicInst>(&I);
5020 if (!II)
5021 return false;
5022
5023 Intrinsic::ID IntrinsicID = II->getIntrinsicID();
5024 switch (IntrinsicID) {
5025 case Intrinsic::dbg_declare:
5026 case Intrinsic::dbg_value:
5027 case Intrinsic::dbg_label:
5028 case Intrinsic::lifetime_end:
5029 break;
5030 default:
5031 return false;
5032 }
5033 }
5034 return true;
5035}
5036
5037// Simplify resume that is shared by several landing pads (phi of landing pad).
5038bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
5039 BasicBlock *BB = RI->getParent();
5040
5041 // Check that there are no other instructions except for debug and lifetime
5042 // intrinsics between the phi's and resume instruction.
5045 return false;
5046
5047 SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks;
5048 auto *PhiLPInst = cast<PHINode>(RI->getValue());
5049
5050 // Check incoming blocks to see if any of them are trivial.
5051 for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End;
5052 Idx++) {
5053 auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx);
5054 auto *IncomingValue = PhiLPInst->getIncomingValue(Idx);
5055
5056 // If the block has other successors, we can not delete it because
5057 // it has other dependents.
5058 if (IncomingBB->getUniqueSuccessor() != BB)
5059 continue;
5060
5061 auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI());
5062 // Not the landing pad that caused the control to branch here.
5063 if (IncomingValue != LandingPad)
5064 continue;
5065
5067 make_range(LandingPad->getNextNode(), IncomingBB->getTerminator())))
5068 TrivialUnwindBlocks.insert(IncomingBB);
5069 }
5070
5071 // If no trivial unwind blocks, don't do any simplifications.
5072 if (TrivialUnwindBlocks.empty())
5073 return false;
5074
5075 // Turn all invokes that unwind here into calls.
5076 for (auto *TrivialBB : TrivialUnwindBlocks) {
5077 // Blocks that will be simplified should be removed from the phi node.
5078 // Note there could be multiple edges to the resume block, and we need
5079 // to remove them all.
5080 while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1)
5081 BB->removePredecessor(TrivialBB, true);
5082
5083 for (BasicBlock *Pred :
5085 removeUnwindEdge(Pred, DTU);
5086 ++NumInvokes;
5087 }
5088
5089 // In each SimplifyCFG run, only the current processed block can be erased.
5090 // Otherwise, it will break the iteration of SimplifyCFG pass. So instead
5091 // of erasing TrivialBB, we only remove the branch to the common resume
5092 // block so that we can later erase the resume block since it has no
5093 // predecessors.
5094 TrivialBB->getTerminator()->eraseFromParent();
5095 new UnreachableInst(RI->getContext(), TrivialBB);
5096 if (DTU)
5097 DTU->applyUpdates({{DominatorTree::Delete, TrivialBB, BB}});
5098 }
5099
5100 // Delete the resume block if all its predecessors have been removed.
5101 if (pred_empty(BB))
5102 DeleteDeadBlock(BB, DTU);
5103
5104 return !TrivialUnwindBlocks.empty();
5105}
5106
5107// Simplify resume that is only used by a single (non-phi) landing pad.
5108bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) {
5109 BasicBlock *BB = RI->getParent();
5110 auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI());
5111 assert(RI->getValue() == LPInst &&
5112 "Resume must unwind the exception that caused control to here");
5113
5114 // Check that there are no other instructions except for debug intrinsics.
5116 make_range<Instruction *>(LPInst->getNextNode(), RI)))
5117 return false;
5118
5119 // Turn all invokes that unwind here into calls and delete the basic block.
5121 removeUnwindEdge(Pred, DTU);
5122 ++NumInvokes;
5123 }
5124
5125 // The landingpad is now unreachable. Zap it.
5126 DeleteDeadBlock(BB, DTU);
5127 return true;
5128}
5129
5131 // If this is a trivial cleanup pad that executes no instructions, it can be
5132 // eliminated. If the cleanup pad continues to the caller, any predecessor
5133 // that is an EH pad will be updated to continue to the caller and any
5134 // predecessor that terminates with an invoke instruction will have its invoke
5135 // instruction converted to a call instruction. If the cleanup pad being
5136 // simplified does not continue to the caller, each predecessor will be
5137 // updated to continue to the unwind destination of the cleanup pad being
5138 // simplified.
5139 BasicBlock *BB = RI->getParent();
5140 CleanupPadInst *CPInst = RI->getCleanupPad();
5141 if (CPInst->getParent() != BB)
5142 // This isn't an empty cleanup.
5143 return false;
5144
5145 // We cannot kill the pad if it has multiple uses. This typically arises
5146 // from unreachable basic blocks.
5147 if (!CPInst->hasOneUse())
5148 return false;
5149
5150 // Check that there are no other instructions except for benign intrinsics.
5152 make_range<Instruction *>(CPInst->getNextNode(), RI)))
5153 return false;
5154
5155 // If the cleanup return we are simplifying unwinds to the caller, this will
5156 // set UnwindDest to nullptr.
5157 BasicBlock *UnwindDest = RI->getUnwindDest();