LLVM  4.0.0
SimplifyCFG.cpp
Go to the documentation of this file.
1 //===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Peephole optimize the CFG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/SetOperations.h"
19 #include "llvm/ADT/SetVector.h"
20 #include "llvm/ADT/SmallPtrSet.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/Statistic.h"
24 #include "llvm/ADT/STLExtras.h"
30 #include "llvm/IR/BasicBlock.h"
31 #include "llvm/IR/CallSite.h"
32 #include "llvm/IR/CFG.h"
33 #include "llvm/IR/Constant.h"
34 #include "llvm/IR/ConstantRange.h"
35 #include "llvm/IR/Constants.h"
36 #include "llvm/IR/DataLayout.h"
37 #include "llvm/IR/DebugInfo.h"
38 #include "llvm/IR/DerivedTypes.h"
39 #include "llvm/IR/GlobalValue.h"
40 #include "llvm/IR/GlobalVariable.h"
41 #include "llvm/IR/IRBuilder.h"
42 #include "llvm/IR/InstrTypes.h"
43 #include "llvm/IR/Instruction.h"
44 #include "llvm/IR/Instructions.h"
45 #include "llvm/IR/IntrinsicInst.h"
46 #include "llvm/IR/Intrinsics.h"
47 #include "llvm/IR/LLVMContext.h"
48 #include "llvm/IR/MDBuilder.h"
49 #include "llvm/IR/Metadata.h"
50 #include "llvm/IR/Module.h"
51 #include "llvm/IR/NoFolder.h"
52 #include "llvm/IR/Operator.h"
53 #include "llvm/IR/PatternMatch.h"
54 #include "llvm/IR/Type.h"
55 #include "llvm/IR/User.h"
56 #include "llvm/IR/Value.h"
57 #include "llvm/IR/DebugInfo.h"
58 #include "llvm/Support/Casting.h"
60 #include "llvm/Support/Debug.h"
67 #include <algorithm>
68 #include <cassert>
69 #include <climits>
70 #include <cstddef>
71 #include <cstdint>
72 #include <iterator>
73 #include <map>
74 #include <set>
75 #include <utility>
76 #include <vector>
77 
78 using namespace llvm;
79 using namespace PatternMatch;
80 
81 #define DEBUG_TYPE "simplifycfg"
82 
83 // Chosen as 2 so as to be cheap, but still to have enough power to fold
84 // a select, so the "clamp" idiom (of a min followed by a max) will be caught.
85 // To catch this, we need to fold a compare and a select, hence '2' being the
86 // minimum reasonable default.
88  "phi-node-folding-threshold", cl::Hidden, cl::init(2),
89  cl::desc(
90  "Control the amount of phi node folding to perform (default = 2)"));
91 
92 static cl::opt<bool> DupRet(
93  "simplifycfg-dup-ret", cl::Hidden, cl::init(false),
94  cl::desc("Duplicate return instructions into unconditional branches"));
95 
96 static cl::opt<bool>
97  SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
98  cl::desc("Sink common instructions down to the end block"));
99 
101  "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true),
102  cl::desc("Hoist conditional stores if an unconditional store precedes"));
103 
105  "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true),
106  cl::desc("Hoist conditional stores even if an unconditional store does not "
107  "precede - hoist multiple conditional stores into a single "
108  "predicated store"));
109 
111  "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false),
112  cl::desc("When merging conditional stores, do so even if the resultant "
113  "basic blocks are unlikely to be if-converted as a result"));
114 
116  "speculate-one-expensive-inst", cl::Hidden, cl::init(true),
117  cl::desc("Allow exactly one expensive instruction to be speculatively "
118  "executed"));
119 
121  "max-speculation-depth", cl::Hidden, cl::init(10),
122  cl::desc("Limit maximum recursion depth when calculating costs of "
123  "speculatively executed instructions"));
124 
125 STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps");
126 STATISTIC(NumLinearMaps,
127  "Number of switch instructions turned into linear mapping");
128 STATISTIC(NumLookupTables,
129  "Number of switch instructions turned into lookup tables");
130 STATISTIC(
131  NumLookupTablesHoles,
132  "Number of switch instructions turned into lookup tables (holes checked)");
133 STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares");
134 STATISTIC(NumSinkCommons,
135  "Number of common instructions sunk down to the end block");
136 STATISTIC(NumSpeculations, "Number of speculative executed instructions");
137 
138 namespace {
139 
140 // The first field contains the value that the switch produces when a certain
141 // case group is selected, and the second field is a vector containing the
142 // cases composing the case group.
144  SwitchCaseResultVectorTy;
145 // The first field contains the phi node that generates a result of the switch
146 // and the second field contains the value generated for a certain case in the
147 // switch for that PHI.
148 typedef SmallVector<std::pair<PHINode *, Constant *>, 4> SwitchCaseResultsTy;
149 
150 /// ValueEqualityComparisonCase - Represents a case of a switch.
151 struct ValueEqualityComparisonCase {
153  BasicBlock *Dest;
154 
155  ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest)
156  : Value(Value), Dest(Dest) {}
157 
158  bool operator<(ValueEqualityComparisonCase RHS) const {
159  // Comparing pointers is ok as we only rely on the order for uniquing.
160  return Value < RHS.Value;
161  }
162 
163  bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; }
164 };
165 
166 class SimplifyCFGOpt {
167  const TargetTransformInfo &TTI;
168  const DataLayout &DL;
169  unsigned BonusInstThreshold;
170  AssumptionCache *AC;
171  SmallPtrSetImpl<BasicBlock *> *LoopHeaders;
172  Value *isValueEqualityComparison(TerminatorInst *TI);
173  BasicBlock *GetValueEqualityComparisonCases(
174  TerminatorInst *TI, std::vector<ValueEqualityComparisonCase> &Cases);
175  bool SimplifyEqualityComparisonWithOnlyPredecessor(TerminatorInst *TI,
176  BasicBlock *Pred,
177  IRBuilder<> &Builder);
178  bool FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
179  IRBuilder<> &Builder);
180 
181  bool SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder);
182  bool SimplifyResume(ResumeInst *RI, IRBuilder<> &Builder);
183  bool SimplifySingleResume(ResumeInst *RI);
184  bool SimplifyCommonResume(ResumeInst *RI);
185  bool SimplifyCleanupReturn(CleanupReturnInst *RI);
186  bool SimplifyUnreachable(UnreachableInst *UI);
187  bool SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder);
188  bool SimplifyIndirectBr(IndirectBrInst *IBI);
189  bool SimplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder);
190  bool SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder);
191 
192 public:
193  SimplifyCFGOpt(const TargetTransformInfo &TTI, const DataLayout &DL,
194  unsigned BonusInstThreshold, AssumptionCache *AC,
195  SmallPtrSetImpl<BasicBlock *> *LoopHeaders)
196  : TTI(TTI), DL(DL), BonusInstThreshold(BonusInstThreshold), AC(AC),
197  LoopHeaders(LoopHeaders) {}
198 
199  bool run(BasicBlock *BB);
200 };
201 
202 } // end anonymous namespace
203 
204 /// Return true if it is safe to merge these two
205 /// terminator instructions together.
206 static bool
208  SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) {
209  if (SI1 == SI2)
210  return false; // Can't merge with self!
211 
212  // It is not safe to merge these two switch instructions if they have a common
213  // successor, and if that successor has a PHI node, and if *that* PHI node has
214  // conflicting incoming values from the two switch blocks.
215  BasicBlock *SI1BB = SI1->getParent();
216  BasicBlock *SI2BB = SI2->getParent();
217 
218  SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
219  bool Fail = false;
220  for (BasicBlock *Succ : successors(SI2BB))
221  if (SI1Succs.count(Succ))
222  for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) {
223  PHINode *PN = cast<PHINode>(BBI);
224  if (PN->getIncomingValueForBlock(SI1BB) !=
225  PN->getIncomingValueForBlock(SI2BB)) {
226  if (FailBlocks)
227  FailBlocks->insert(Succ);
228  Fail = true;
229  }
230  }
231 
232  return !Fail;
233 }
234 
235 /// Return true if it is safe and profitable to merge these two terminator
236 /// instructions together, where SI1 is an unconditional branch. PhiNodes will
237 /// store all PHI nodes in common successors.
238 static bool
240  Instruction *Cond,
241  SmallVectorImpl<PHINode *> &PhiNodes) {
242  if (SI1 == SI2)
243  return false; // Can't merge with self!
244  assert(SI1->isUnconditional() && SI2->isConditional());
245 
246  // We fold the unconditional branch if we can easily update all PHI nodes in
247  // common successors:
248  // 1> We have a constant incoming value for the conditional branch;
249  // 2> We have "Cond" as the incoming value for the unconditional branch;
250  // 3> SI2->getCondition() and Cond have same operands.
251  CmpInst *Ci2 = dyn_cast<CmpInst>(SI2->getCondition());
252  if (!Ci2)
253  return false;
254  if (!(Cond->getOperand(0) == Ci2->getOperand(0) &&
255  Cond->getOperand(1) == Ci2->getOperand(1)) &&
256  !(Cond->getOperand(0) == Ci2->getOperand(1) &&
257  Cond->getOperand(1) == Ci2->getOperand(0)))
258  return false;
259 
260  BasicBlock *SI1BB = SI1->getParent();
261  BasicBlock *SI2BB = SI2->getParent();
262  SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
263  for (BasicBlock *Succ : successors(SI2BB))
264  if (SI1Succs.count(Succ))
265  for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) {
266  PHINode *PN = cast<PHINode>(BBI);
267  if (PN->getIncomingValueForBlock(SI1BB) != Cond ||
268  !isa<ConstantInt>(PN->getIncomingValueForBlock(SI2BB)))
269  return false;
270  PhiNodes.push_back(PN);
271  }
272  return true;
273 }
274 
275 /// Update PHI nodes in Succ to indicate that there will now be entries in it
276 /// from the 'NewPred' block. The values that will be flowing into the PHI nodes
277 /// will be the same as those coming in from ExistPred, an existing predecessor
278 /// of Succ.
279 static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
280  BasicBlock *ExistPred) {
281  if (!isa<PHINode>(Succ->begin()))
282  return; // Quick exit if nothing to do
283 
284  PHINode *PN;
285  for (BasicBlock::iterator I = Succ->begin(); (PN = dyn_cast<PHINode>(I)); ++I)
286  PN->addIncoming(PN->getIncomingValueForBlock(ExistPred), NewPred);
287 }
288 
289 /// Compute an abstract "cost" of speculating the given instruction,
290 /// which is assumed to be safe to speculate. TCC_Free means cheap,
291 /// TCC_Basic means less cheap, and TCC_Expensive means prohibitively
292 /// expensive.
293 static unsigned ComputeSpeculationCost(const User *I,
294  const TargetTransformInfo &TTI) {
296  "Instruction is not safe to speculatively execute!");
297  return TTI.getUserCost(I);
298 }
299 
300 /// If we have a merge point of an "if condition" as accepted above,
301 /// return true if the specified value dominates the block. We
302 /// don't handle the true generality of domination here, just a special case
303 /// which works well enough for us.
304 ///
305 /// If AggressiveInsts is non-null, and if V does not dominate BB, we check to
306 /// see if V (which must be an instruction) and its recursive operands
307 /// that do not dominate BB have a combined cost lower than CostRemaining and
308 /// are non-trapping. If both are true, the instruction is inserted into the
309 /// set and true is returned.
310 ///
311 /// The cost for most non-trapping instructions is defined as 1 except for
312 /// Select whose cost is 2.
313 ///
314 /// After this function returns, CostRemaining is decreased by the cost of
315 /// V plus its non-dominating operands. If that cost is greater than
316 /// CostRemaining, false is returned and CostRemaining is undefined.
318  SmallPtrSetImpl<Instruction *> *AggressiveInsts,
319  unsigned &CostRemaining,
320  const TargetTransformInfo &TTI,
321  unsigned Depth = 0) {
322  // It is possible to hit a zero-cost cycle (phi/gep instructions for example),
323  // so limit the recursion depth.
324  // TODO: While this recursion limit does prevent pathological behavior, it
325  // would be better to track visited instructions to avoid cycles.
326  if (Depth == MaxSpeculationDepth)
327  return false;
328 
330  if (!I) {
331  // Non-instructions all dominate instructions, but not all constantexprs
332  // can be executed unconditionally.
333  if (ConstantExpr *C = dyn_cast<ConstantExpr>(V))
334  if (C->canTrap())
335  return false;
336  return true;
337  }
338  BasicBlock *PBB = I->getParent();
339 
340  // We don't want to allow weird loops that might have the "if condition" in
341  // the bottom of this block.
342  if (PBB == BB)
343  return false;
344 
345  // If this instruction is defined in a block that contains an unconditional
346  // branch to BB, then it must be in the 'conditional' part of the "if
347  // statement". If not, it definitely dominates the region.
349  if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB)
350  return true;
351 
352  // If we aren't allowing aggressive promotion anymore, then don't consider
353  // instructions in the 'if region'.
354  if (!AggressiveInsts)
355  return false;
356 
357  // If we have seen this instruction before, don't count it again.
358  if (AggressiveInsts->count(I))
359  return true;
360 
361  // Okay, it looks like the instruction IS in the "condition". Check to
362  // see if it's a cheap instruction to unconditionally compute, and if it
363  // only uses stuff defined outside of the condition. If so, hoist it out.
365  return false;
366 
367  unsigned Cost = ComputeSpeculationCost(I, TTI);
368 
369  // Allow exactly one instruction to be speculated regardless of its cost
370  // (as long as it is safe to do so).
371  // This is intended to flatten the CFG even if the instruction is a division
372  // or other expensive operation. The speculation of an expensive instruction
373  // is expected to be undone in CodeGenPrepare if the speculation has not
374  // enabled further IR optimizations.
375  if (Cost > CostRemaining &&
376  (!SpeculateOneExpensiveInst || !AggressiveInsts->empty() || Depth > 0))
377  return false;
378 
379  // Avoid unsigned wrap.
380  CostRemaining = (Cost > CostRemaining) ? 0 : CostRemaining - Cost;
381 
382  // Okay, we can only really hoist these out if their operands do
383  // not take us over the cost threshold.
384  for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i)
385  if (!DominatesMergePoint(*i, BB, AggressiveInsts, CostRemaining, TTI,
386  Depth + 1))
387  return false;
388  // Okay, it's safe to do this! Remember this instruction.
389  AggressiveInsts->insert(I);
390  return true;
391 }
392 
393 /// Extract ConstantInt from value, looking through IntToPtr
394 /// and PointerNullValue. Return NULL if value is not a constant int.
395 static ConstantInt *GetConstantInt(Value *V, const DataLayout &DL) {
396  // Normal constant int.
397  ConstantInt *CI = dyn_cast<ConstantInt>(V);
398  if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy())
399  return CI;
400 
401  // This is some kind of pointer constant. Turn it into a pointer-sized
402  // ConstantInt if possible.
403  IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType()));
404 
405  // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
406  if (isa<ConstantPointerNull>(V))
407  return ConstantInt::get(PtrTy, 0);
408 
409  // IntToPtr const int.
410  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
411  if (CE->getOpcode() == Instruction::IntToPtr)
412  if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) {
413  // The constant is very likely to have the right type already.
414  if (CI->getType() == PtrTy)
415  return CI;
416  else
417  return cast<ConstantInt>(
418  ConstantExpr::getIntegerCast(CI, PtrTy, /*isSigned=*/false));
419  }
420  return nullptr;
421 }
422 
423 namespace {
424 
425 /// Given a chain of or (||) or and (&&) comparison of a value against a
426 /// constant, this will try to recover the information required for a switch
427 /// structure.
428 /// It will depth-first traverse the chain of comparison, seeking for patterns
429 /// like %a == 12 or %a < 4 and combine them to produce a set of integer
430 /// representing the different cases for the switch.
431 /// Note that if the chain is composed of '||' it will build the set of elements
432 /// that matches the comparisons (i.e. any of this value validate the chain)
433 /// while for a chain of '&&' it will build the set elements that make the test
434 /// fail.
435 struct ConstantComparesGatherer {
436  const DataLayout &DL;
437  Value *CompValue; /// Value found for the switch comparison
438  Value *Extra; /// Extra clause to be checked before the switch
439  SmallVector<ConstantInt *, 8> Vals; /// Set of integers to match in switch
440  unsigned UsedICmps; /// Number of comparisons matched in the and/or chain
441 
442  /// Construct and compute the result for the comparison instruction Cond
443  ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL)
444  : DL(DL), CompValue(nullptr), Extra(nullptr), UsedICmps(0) {
445  gather(Cond);
446  }
447 
448  /// Prevent copy
449  ConstantComparesGatherer(const ConstantComparesGatherer &) = delete;
450  ConstantComparesGatherer &
451  operator=(const ConstantComparesGatherer &) = delete;
452 
453 private:
454  /// Try to set the current value used for the comparison, it succeeds only if
455  /// it wasn't set before or if the new value is the same as the old one
456  bool setValueOnce(Value *NewVal) {
457  if (CompValue && CompValue != NewVal)
458  return false;
459  CompValue = NewVal;
460  return (CompValue != nullptr);
461  }
462 
463  /// Try to match Instruction "I" as a comparison against a constant and
464  /// populates the array Vals with the set of values that match (or do not
465  /// match depending on isEQ).
466  /// Return false on failure. On success, the Value the comparison matched
467  /// against is placed in CompValue.
468  /// If CompValue is already set, the function is expected to fail if a match
469  /// is found but the value compared to is different.
470  bool matchInstruction(Instruction *I, bool isEQ) {
471  // If this is an icmp against a constant, handle this as one of the cases.
472  ICmpInst *ICI;
473  ConstantInt *C;
474  if (!((ICI = dyn_cast<ICmpInst>(I)) &&
475  (C = GetConstantInt(I->getOperand(1), DL)))) {
476  return false;
477  }
478 
479  Value *RHSVal;
480  const APInt *RHSC;
481 
482  // Pattern match a special case
483  // (x & ~2^z) == y --> x == y || x == y|2^z
484  // This undoes a transformation done by instcombine to fuse 2 compares.
485  if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
486 
487  // It's a little bit hard to see why the following transformations are
488  // correct. Here is a CVC3 program to verify them for 64-bit values:
489 
490  /*
491  ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63);
492  x : BITVECTOR(64);
493  y : BITVECTOR(64);
494  z : BITVECTOR(64);
495  mask : BITVECTOR(64) = BVSHL(ONE, z);
496  QUERY( (y & ~mask = y) =>
497  ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
498  );
499  QUERY( (y | mask = y) =>
500  ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
501  );
502  */
503 
504  // Please note that each pattern must be a dual implication (<--> or
505  // iff). One directional implication can create spurious matches. If the
506  // implication is only one-way, an unsatisfiable condition on the left
507  // side can imply a satisfiable condition on the right side. Dual
508  // implication ensures that satisfiable conditions are transformed to
509  // other satisfiable conditions and unsatisfiable conditions are
510  // transformed to other unsatisfiable conditions.
511 
512  // Here is a concrete example of a unsatisfiable condition on the left
513  // implying a satisfiable condition on the right:
514  //
515  // mask = (1 << z)
516  // (x & ~mask) == y --> (x == y || x == (y | mask))
517  //
518  // Substituting y = 3, z = 0 yields:
519  // (x & -2) == 3 --> (x == 3 || x == 2)
520 
521  // Pattern match a special case:
522  /*
523  QUERY( (y & ~mask = y) =>
524  ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
525  );
526  */
527  if (match(ICI->getOperand(0),
528  m_And(m_Value(RHSVal), m_APInt(RHSC)))) {
529  APInt Mask = ~*RHSC;
530  if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) {
531  // If we already have a value for the switch, it has to match!
532  if (!setValueOnce(RHSVal))
533  return false;
534 
535  Vals.push_back(C);
536  Vals.push_back(
538  C->getValue() | Mask));
539  UsedICmps++;
540  return true;
541  }
542  }
543 
544  // Pattern match a special case:
545  /*
546  QUERY( (y | mask = y) =>
547  ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
548  );
549  */
550  if (match(ICI->getOperand(0),
551  m_Or(m_Value(RHSVal), m_APInt(RHSC)))) {
552  APInt Mask = *RHSC;
553  if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) {
554  // If we already have a value for the switch, it has to match!
555  if (!setValueOnce(RHSVal))
556  return false;
557 
558  Vals.push_back(C);
559  Vals.push_back(ConstantInt::get(C->getContext(),
560  C->getValue() & ~Mask));
561  UsedICmps++;
562  return true;
563  }
564  }
565 
566  // If we already have a value for the switch, it has to match!
567  if (!setValueOnce(ICI->getOperand(0)))
568  return false;
569 
570  UsedICmps++;
571  Vals.push_back(C);
572  return ICI->getOperand(0);
573  }
574 
575  // If we have "x ult 3", for example, then we can add 0,1,2 to the set.
577  ICI->getPredicate(), C->getValue());
578 
579  // Shift the range if the compare is fed by an add. This is the range
580  // compare idiom as emitted by instcombine.
581  Value *CandidateVal = I->getOperand(0);
582  if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) {
583  Span = Span.subtract(*RHSC);
584  CandidateVal = RHSVal;
585  }
586 
587  // If this is an and/!= check, then we are looking to build the set of
588  // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into
589  // x != 0 && x != 1.
590  if (!isEQ)
591  Span = Span.inverse();
592 
593  // If there are a ton of values, we don't want to make a ginormous switch.
594  if (Span.getSetSize().ugt(8) || Span.isEmptySet()) {
595  return false;
596  }
597 
598  // If we already have a value for the switch, it has to match!
599  if (!setValueOnce(CandidateVal))
600  return false;
601 
602  // Add all values from the range to the set
603  for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp)
604  Vals.push_back(ConstantInt::get(I->getContext(), Tmp));
605 
606  UsedICmps++;
607  return true;
608  }
609 
610  /// Given a potentially 'or'd or 'and'd together collection of icmp
611  /// eq/ne/lt/gt instructions that compare a value against a constant, extract
612  /// the value being compared, and stick the list constants into the Vals
613  /// vector.
614  /// One "Extra" case is allowed to differ from the other.
615  void gather(Value *V) {
617  bool isEQ = (I->getOpcode() == Instruction::Or);
618 
619  // Keep a stack (SmallVector for efficiency) for depth-first traversal
621  SmallPtrSet<Value *, 8> Visited;
622 
623  // Initialize
624  Visited.insert(V);
625  DFT.push_back(V);
626 
627  while (!DFT.empty()) {
628  V = DFT.pop_back_val();
629 
630  if (Instruction *I = dyn_cast<Instruction>(V)) {
631  // If it is a || (or && depending on isEQ), process the operands.
632  if (I->getOpcode() == (isEQ ? Instruction::Or : Instruction::And)) {
633  if (Visited.insert(I->getOperand(1)).second)
634  DFT.push_back(I->getOperand(1));
635  if (Visited.insert(I->getOperand(0)).second)
636  DFT.push_back(I->getOperand(0));
637  continue;
638  }
639 
640  // Try to match the current instruction
641  if (matchInstruction(I, isEQ))
642  // Match succeed, continue the loop
643  continue;
644  }
645 
646  // One element of the sequence of || (or &&) could not be match as a
647  // comparison against the same value as the others.
648  // We allow only one "Extra" case to be checked before the switch
649  if (!Extra) {
650  Extra = V;
651  continue;
652  }
653  // Failed to parse a proper sequence, abort now
654  CompValue = nullptr;
655  break;
656  }
657  }
658 };
659 
660 } // end anonymous namespace
661 
663  Instruction *Cond = nullptr;
664  if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
665  Cond = dyn_cast<Instruction>(SI->getCondition());
666  } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
667  if (BI->isConditional())
668  Cond = dyn_cast<Instruction>(BI->getCondition());
669  } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) {
670  Cond = dyn_cast<Instruction>(IBI->getAddress());
671  }
672 
673  TI->eraseFromParent();
674  if (Cond)
676 }
677 
678 /// Return true if the specified terminator checks
679 /// to see if a value is equal to constant integer value.
680 Value *SimplifyCFGOpt::isValueEqualityComparison(TerminatorInst *TI) {
681  Value *CV = nullptr;
682  if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
683  // Do not permit merging of large switch instructions into their
684  // predecessors unless there is only one predecessor.
685  if (SI->getNumSuccessors() * std::distance(pred_begin(SI->getParent()),
686  pred_end(SI->getParent())) <=
687  128)
688  CV = SI->getCondition();
689  } else if (BranchInst *BI = dyn_cast<BranchInst>(TI))
690  if (BI->isConditional() && BI->getCondition()->hasOneUse())
691  if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) {
692  if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL))
693  CV = ICI->getOperand(0);
694  }
695 
696  // Unwrap any lossless ptrtoint cast.
697  if (CV) {
698  if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) {
699  Value *Ptr = PTII->getPointerOperand();
700  if (PTII->getType() == DL.getIntPtrType(Ptr->getType()))
701  CV = Ptr;
702  }
703  }
704  return CV;
705 }
706 
707 /// Given a value comparison instruction,
708 /// decode all of the 'cases' that it represents and return the 'default' block.
709 BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases(
710  TerminatorInst *TI, std::vector<ValueEqualityComparisonCase> &Cases) {
711  if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
712  Cases.reserve(SI->getNumCases());
713  for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end(); i != e;
714  ++i)
715  Cases.push_back(
716  ValueEqualityComparisonCase(i.getCaseValue(), i.getCaseSuccessor()));
717  return SI->getDefaultDest();
718  }
719 
720  BranchInst *BI = cast<BranchInst>(TI);
721  ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
722  BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE);
723  Cases.push_back(ValueEqualityComparisonCase(
724  GetConstantInt(ICI->getOperand(1), DL), Succ));
725  return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ);
726 }
727 
728 /// Given a vector of bb/value pairs, remove any entries
729 /// in the list that match the specified block.
730 static void
732  std::vector<ValueEqualityComparisonCase> &Cases) {
733  Cases.erase(std::remove(Cases.begin(), Cases.end(), BB), Cases.end());
734 }
735 
736 /// Return true if there are any keys in C1 that exist in C2 as well.
737 static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
738  std::vector<ValueEqualityComparisonCase> &C2) {
739  std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2;
740 
741  // Make V1 be smaller than V2.
742  if (V1->size() > V2->size())
743  std::swap(V1, V2);
744 
745  if (V1->empty())
746  return false;
747  if (V1->size() == 1) {
748  // Just scan V2.
749  ConstantInt *TheVal = (*V1)[0].Value;
750  for (unsigned i = 0, e = V2->size(); i != e; ++i)
751  if (TheVal == (*V2)[i].Value)
752  return true;
753  }
754 
755  // Otherwise, just sort both lists and compare element by element.
756  array_pod_sort(V1->begin(), V1->end());
757  array_pod_sort(V2->begin(), V2->end());
758  unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size();
759  while (i1 != e1 && i2 != e2) {
760  if ((*V1)[i1].Value == (*V2)[i2].Value)
761  return true;
762  if ((*V1)[i1].Value < (*V2)[i2].Value)
763  ++i1;
764  else
765  ++i2;
766  }
767  return false;
768 }
769 
770 /// If TI is known to be a terminator instruction and its block is known to
771 /// only have a single predecessor block, check to see if that predecessor is
772 /// also a value comparison with the same value, and if that comparison
773 /// determines the outcome of this comparison. If so, simplify TI. This does a
774 /// very limited form of jump threading.
775 bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
776  TerminatorInst *TI, BasicBlock *Pred, IRBuilder<> &Builder) {
777  Value *PredVal = isValueEqualityComparison(Pred->getTerminator());
778  if (!PredVal)
779  return false; // Not a value comparison in predecessor.
780 
781  Value *ThisVal = isValueEqualityComparison(TI);
782  assert(ThisVal && "This isn't a value comparison!!");
783  if (ThisVal != PredVal)
784  return false; // Different predicates.
785 
786  // TODO: Preserve branch weight metadata, similarly to how
787  // FoldValueComparisonIntoPredecessors preserves it.
788 
789  // Find out information about when control will move from Pred to TI's block.
790  std::vector<ValueEqualityComparisonCase> PredCases;
791  BasicBlock *PredDef =
792  GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases);
793  EliminateBlockCases(PredDef, PredCases); // Remove default from cases.
794 
795  // Find information about how control leaves this block.
796  std::vector<ValueEqualityComparisonCase> ThisCases;
797  BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases);
798  EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases.
799 
800  // If TI's block is the default block from Pred's comparison, potentially
801  // simplify TI based on this knowledge.
802  if (PredDef == TI->getParent()) {
803  // If we are here, we know that the value is none of those cases listed in
804  // PredCases. If there are any cases in ThisCases that are in PredCases, we
805  // can simplify TI.
806  if (!ValuesOverlap(PredCases, ThisCases))
807  return false;
808 
809  if (isa<BranchInst>(TI)) {
810  // Okay, one of the successors of this condbr is dead. Convert it to a
811  // uncond br.
812  assert(ThisCases.size() == 1 && "Branch can only have one case!");
813  // Insert the new branch.
814  Instruction *NI = Builder.CreateBr(ThisDef);
815  (void)NI;
816 
817  // Remove PHI node entries for the dead edge.
818  ThisCases[0].Dest->removePredecessor(TI->getParent());
819 
820  DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
821  << "Through successor TI: " << *TI << "Leaving: " << *NI
822  << "\n");
823 
825  return true;
826  }
827 
828  SwitchInst *SI = cast<SwitchInst>(TI);
829  // Okay, TI has cases that are statically dead, prune them away.
830  SmallPtrSet<Constant *, 16> DeadCases;
831  for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
832  DeadCases.insert(PredCases[i].Value);
833 
834  DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
835  << "Through successor TI: " << *TI);
836 
837  // Collect branch weights into a vector.
838  SmallVector<uint32_t, 8> Weights;
840  bool HasWeight = MD && (MD->getNumOperands() == 2 + SI->getNumCases());
841  if (HasWeight)
842  for (unsigned MD_i = 1, MD_e = MD->getNumOperands(); MD_i < MD_e;
843  ++MD_i) {
844  ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(MD_i));
845  Weights.push_back(CI->getValue().getZExtValue());
846  }
847  for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
848  --i;
849  if (DeadCases.count(i.getCaseValue())) {
850  if (HasWeight) {
851  std::swap(Weights[i.getCaseIndex() + 1], Weights.back());
852  Weights.pop_back();
853  }
854  i.getCaseSuccessor()->removePredecessor(TI->getParent());
855  SI->removeCase(i);
856  }
857  }
858  if (HasWeight && Weights.size() >= 2)
860  MDBuilder(SI->getParent()->getContext())
861  .createBranchWeights(Weights));
862 
863  DEBUG(dbgs() << "Leaving: " << *TI << "\n");
864  return true;
865  }
866 
867  // Otherwise, TI's block must correspond to some matched value. Find out
868  // which value (or set of values) this is.
869  ConstantInt *TIV = nullptr;
870  BasicBlock *TIBB = TI->getParent();
871  for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
872  if (PredCases[i].Dest == TIBB) {
873  if (TIV)
874  return false; // Cannot handle multiple values coming to this block.
875  TIV = PredCases[i].Value;
876  }
877  assert(TIV && "No edge from pred to succ?");
878 
879  // Okay, we found the one constant that our value can be if we get into TI's
880  // BB. Find out which successor will unconditionally be branched to.
881  BasicBlock *TheRealDest = nullptr;
882  for (unsigned i = 0, e = ThisCases.size(); i != e; ++i)
883  if (ThisCases[i].Value == TIV) {
884  TheRealDest = ThisCases[i].Dest;
885  break;
886  }
887 
888  // If not handled by any explicit cases, it is handled by the default case.
889  if (!TheRealDest)
890  TheRealDest = ThisDef;
891 
892  // Remove PHI node entries for dead edges.
893  BasicBlock *CheckEdge = TheRealDest;
894  for (BasicBlock *Succ : successors(TIBB))
895  if (Succ != CheckEdge)
896  Succ->removePredecessor(TIBB);
897  else
898  CheckEdge = nullptr;
899 
900  // Insert the new branch.
901  Instruction *NI = Builder.CreateBr(TheRealDest);
902  (void)NI;
903 
904  DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
905  << "Through successor TI: " << *TI << "Leaving: " << *NI
906  << "\n");
907 
909  return true;
910 }
911 
912 namespace {
913 
914 /// This class implements a stable ordering of constant
915 /// integers that does not depend on their address. This is important for
916 /// applications that sort ConstantInt's to ensure uniqueness.
917 struct ConstantIntOrdering {
918  bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const {
919  return LHS->getValue().ult(RHS->getValue());
920  }
921 };
922 
923 } // end anonymous namespace
924 
925 static int ConstantIntSortPredicate(ConstantInt *const *P1,
926  ConstantInt *const *P2) {
927  const ConstantInt *LHS = *P1;
928  const ConstantInt *RHS = *P2;
929  if (LHS == RHS)
930  return 0;
931  return LHS->getValue().ult(RHS->getValue()) ? 1 : -1;
932 }
933 
934 static inline bool HasBranchWeights(const Instruction *I) {
936  if (ProfMD && ProfMD->getOperand(0))
937  if (MDString *MDS = dyn_cast<MDString>(ProfMD->getOperand(0)))
938  return MDS->getString().equals("branch_weights");
939 
940  return false;
941 }
942 
943 /// Get Weights of a given TerminatorInst, the default weight is at the front
944 /// of the vector. If TI is a conditional eq, we need to swap the branch-weight
945 /// metadata.
947  SmallVectorImpl<uint64_t> &Weights) {
949  assert(MD);
950  for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) {
951  ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(i));
952  Weights.push_back(CI->getValue().getZExtValue());
953  }
954 
955  // If TI is a conditional eq, the default case is the false case,
956  // and the corresponding branch-weight data is at index 2. We swap the
957  // default weight to be the first entry.
958  if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
959  assert(Weights.size() == 2);
960  ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
961  if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
962  std::swap(Weights.front(), Weights.back());
963  }
964 }
965 
966 /// Keep halving the weights until all can fit in uint32_t.
967 static void FitWeights(MutableArrayRef<uint64_t> Weights) {
968  uint64_t Max = *std::max_element(Weights.begin(), Weights.end());
969  if (Max > UINT_MAX) {
970  unsigned Offset = 32 - countLeadingZeros(Max);
971  for (uint64_t &I : Weights)
972  I >>= Offset;
973  }
974 }
975 
976 /// The specified terminator is a value equality comparison instruction
977 /// (either a switch or a branch on "X == c").
978 /// See if any of the predecessors of the terminator block are value comparisons
979 /// on the same value. If so, and if safe to do so, fold them together.
980 bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
981  IRBuilder<> &Builder) {
982  BasicBlock *BB = TI->getParent();
983  Value *CV = isValueEqualityComparison(TI); // CondVal
984  assert(CV && "Not a comparison?");
985  bool Changed = false;
986 
988  while (!Preds.empty()) {
989  BasicBlock *Pred = Preds.pop_back_val();
990 
991  // See if the predecessor is a comparison with the same value.
992  TerminatorInst *PTI = Pred->getTerminator();
993  Value *PCV = isValueEqualityComparison(PTI); // PredCondVal
994 
995  if (PCV == CV && TI != PTI) {
997  if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) {
998  for (auto *Succ : FailBlocks) {
999  std::vector<BasicBlock*> Blocks = { TI->getParent() };
1000  if (!SplitBlockPredecessors(Succ, Blocks, ".fold.split"))
1001  return false;
1002  }
1003  }
1004 
1005  // Figure out which 'cases' to copy from SI to PSI.
1006  std::vector<ValueEqualityComparisonCase> BBCases;
1007  BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases);
1008 
1009  std::vector<ValueEqualityComparisonCase> PredCases;
1010  BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases);
1011 
1012  // Based on whether the default edge from PTI goes to BB or not, fill in
1013  // PredCases and PredDefault with the new switch cases we would like to
1014  // build.
1015  SmallVector<BasicBlock *, 8> NewSuccessors;
1016 
1017  // Update the branch weight metadata along the way
1018  SmallVector<uint64_t, 8> Weights;
1019  bool PredHasWeights = HasBranchWeights(PTI);
1020  bool SuccHasWeights = HasBranchWeights(TI);
1021 
1022  if (PredHasWeights) {
1023  GetBranchWeights(PTI, Weights);
1024  // branch-weight metadata is inconsistent here.
1025  if (Weights.size() != 1 + PredCases.size())
1026  PredHasWeights = SuccHasWeights = false;
1027  } else if (SuccHasWeights)
1028  // If there are no predecessor weights but there are successor weights,
1029  // populate Weights with 1, which will later be scaled to the sum of
1030  // successor's weights
1031  Weights.assign(1 + PredCases.size(), 1);
1032 
1033  SmallVector<uint64_t, 8> SuccWeights;
1034  if (SuccHasWeights) {
1035  GetBranchWeights(TI, SuccWeights);
1036  // branch-weight metadata is inconsistent here.
1037  if (SuccWeights.size() != 1 + BBCases.size())
1038  PredHasWeights = SuccHasWeights = false;
1039  } else if (PredHasWeights)
1040  SuccWeights.assign(1 + BBCases.size(), 1);
1041 
1042  if (PredDefault == BB) {
1043  // If this is the default destination from PTI, only the edges in TI
1044  // that don't occur in PTI, or that branch to BB will be activated.
1045  std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1046  for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1047  if (PredCases[i].Dest != BB)
1048  PTIHandled.insert(PredCases[i].Value);
1049  else {
1050  // The default destination is BB, we don't need explicit targets.
1051  std::swap(PredCases[i], PredCases.back());
1052 
1053  if (PredHasWeights || SuccHasWeights) {
1054  // Increase weight for the default case.
1055  Weights[0] += Weights[i + 1];
1056  std::swap(Weights[i + 1], Weights.back());
1057  Weights.pop_back();
1058  }
1059 
1060  PredCases.pop_back();
1061  --i;
1062  --e;
1063  }
1064 
1065  // Reconstruct the new switch statement we will be building.
1066  if (PredDefault != BBDefault) {
1067  PredDefault->removePredecessor(Pred);
1068  PredDefault = BBDefault;
1069  NewSuccessors.push_back(BBDefault);
1070  }
1071 
1072  unsigned CasesFromPred = Weights.size();
1073  uint64_t ValidTotalSuccWeight = 0;
1074  for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1075  if (!PTIHandled.count(BBCases[i].Value) &&
1076  BBCases[i].Dest != BBDefault) {
1077  PredCases.push_back(BBCases[i]);
1078  NewSuccessors.push_back(BBCases[i].Dest);
1079  if (SuccHasWeights || PredHasWeights) {
1080  // The default weight is at index 0, so weight for the ith case
1081  // should be at index i+1. Scale the cases from successor by
1082  // PredDefaultWeight (Weights[0]).
1083  Weights.push_back(Weights[0] * SuccWeights[i + 1]);
1084  ValidTotalSuccWeight += SuccWeights[i + 1];
1085  }
1086  }
1087 
1088  if (SuccHasWeights || PredHasWeights) {
1089  ValidTotalSuccWeight += SuccWeights[0];
1090  // Scale the cases from predecessor by ValidTotalSuccWeight.
1091  for (unsigned i = 1; i < CasesFromPred; ++i)
1092  Weights[i] *= ValidTotalSuccWeight;
1093  // Scale the default weight by SuccDefaultWeight (SuccWeights[0]).
1094  Weights[0] *= SuccWeights[0];
1095  }
1096  } else {
1097  // If this is not the default destination from PSI, only the edges
1098  // in SI that occur in PSI with a destination of BB will be
1099  // activated.
1100  std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1101  std::map<ConstantInt *, uint64_t> WeightsForHandled;
1102  for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1103  if (PredCases[i].Dest == BB) {
1104  PTIHandled.insert(PredCases[i].Value);
1105 
1106  if (PredHasWeights || SuccHasWeights) {
1107  WeightsForHandled[PredCases[i].Value] = Weights[i + 1];
1108  std::swap(Weights[i + 1], Weights.back());
1109  Weights.pop_back();
1110  }
1111 
1112  std::swap(PredCases[i], PredCases.back());
1113  PredCases.pop_back();
1114  --i;
1115  --e;
1116  }
1117 
1118  // Okay, now we know which constants were sent to BB from the
1119  // predecessor. Figure out where they will all go now.
1120  for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1121  if (PTIHandled.count(BBCases[i].Value)) {
1122  // If this is one we are capable of getting...
1123  if (PredHasWeights || SuccHasWeights)
1124  Weights.push_back(WeightsForHandled[BBCases[i].Value]);
1125  PredCases.push_back(BBCases[i]);
1126  NewSuccessors.push_back(BBCases[i].Dest);
1127  PTIHandled.erase(
1128  BBCases[i].Value); // This constant is taken care of
1129  }
1130 
1131  // If there are any constants vectored to BB that TI doesn't handle,
1132  // they must go to the default destination of TI.
1133  for (ConstantInt *I : PTIHandled) {
1134  if (PredHasWeights || SuccHasWeights)
1135  Weights.push_back(WeightsForHandled[I]);
1136  PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault));
1137  NewSuccessors.push_back(BBDefault);
1138  }
1139  }
1140 
1141  // Okay, at this point, we know which new successor Pred will get. Make
1142  // sure we update the number of entries in the PHI nodes for these
1143  // successors.
1144  for (BasicBlock *NewSuccessor : NewSuccessors)
1145  AddPredecessorToBlock(NewSuccessor, Pred, BB);
1146 
1147  Builder.SetInsertPoint(PTI);
1148  // Convert pointer to int before we switch.
1149  if (CV->getType()->isPointerTy()) {
1150  CV = Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()),
1151  "magicptr");
1152  }
1153 
1154  // Now that the successors are updated, create the new Switch instruction.
1155  SwitchInst *NewSI =
1156  Builder.CreateSwitch(CV, PredDefault, PredCases.size());
1157  NewSI->setDebugLoc(PTI->getDebugLoc());
1158  for (ValueEqualityComparisonCase &V : PredCases)
1159  NewSI->addCase(V.Value, V.Dest);
1160 
1161  if (PredHasWeights || SuccHasWeights) {
1162  // Halve the weights if any of them cannot fit in an uint32_t
1163  FitWeights(Weights);
1164 
1165  SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
1166 
1167  NewSI->setMetadata(
1169  MDBuilder(BB->getContext()).createBranchWeights(MDWeights));
1170  }
1171 
1173 
1174  // Okay, last check. If BB is still a successor of PSI, then we must
1175  // have an infinite loop case. If so, add an infinitely looping block
1176  // to handle the case to preserve the behavior of the code.
1177  BasicBlock *InfLoopBlock = nullptr;
1178  for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i)
1179  if (NewSI->getSuccessor(i) == BB) {
1180  if (!InfLoopBlock) {
1181  // Insert it at the end of the function, because it's either code,
1182  // or it won't matter if it's hot. :)
1183  InfLoopBlock = BasicBlock::Create(BB->getContext(), "infloop",
1184  BB->getParent());
1185  BranchInst::Create(InfLoopBlock, InfLoopBlock);
1186  }
1187  NewSI->setSuccessor(i, InfLoopBlock);
1188  }
1189 
1190  Changed = true;
1191  }
1192  }
1193  return Changed;
1194 }
1195 
1196 // If we would need to insert a select that uses the value of this invoke
1197 // (comments in HoistThenElseCodeToIf explain why we would need to do this), we
1198 // can't hoist the invoke, as there is nowhere to put the select in this case.
1200  Instruction *I1, Instruction *I2) {
1201  for (BasicBlock *Succ : successors(BB1)) {
1202  PHINode *PN;
1203  for (BasicBlock::iterator BBI = Succ->begin();
1204  (PN = dyn_cast<PHINode>(BBI)); ++BBI) {
1205  Value *BB1V = PN->getIncomingValueForBlock(BB1);
1206  Value *BB2V = PN->getIncomingValueForBlock(BB2);
1207  if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) {
1208  return false;
1209  }
1210  }
1211  }
1212  return true;
1213 }
1214 
1215 static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I);
1216 
1217 /// Given a conditional branch that goes to BB1 and BB2, hoist any common code
1218 /// in the two blocks up into the branch block. The caller of this function
1219 /// guarantees that BI's block dominates BB1 and BB2.
1221  const TargetTransformInfo &TTI) {
1222  // This does very trivial matching, with limited scanning, to find identical
1223  // instructions in the two blocks. In particular, we don't want to get into
1224  // O(M*N) situations here where M and N are the sizes of BB1 and BB2. As
1225  // such, we currently just scan for obviously identical instructions in an
1226  // identical order.
1227  BasicBlock *BB1 = BI->getSuccessor(0); // The true destination.
1228  BasicBlock *BB2 = BI->getSuccessor(1); // The false destination
1229 
1230  BasicBlock::iterator BB1_Itr = BB1->begin();
1231  BasicBlock::iterator BB2_Itr = BB2->begin();
1232 
1233  Instruction *I1 = &*BB1_Itr++, *I2 = &*BB2_Itr++;
1234  // Skip debug info if it is not identical.
1237  if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1238  while (isa<DbgInfoIntrinsic>(I1))
1239  I1 = &*BB1_Itr++;
1240  while (isa<DbgInfoIntrinsic>(I2))
1241  I2 = &*BB2_Itr++;
1242  }
1243  if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) ||
1244  (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)))
1245  return false;
1246 
1247  BasicBlock *BIParent = BI->getParent();
1248 
1249  bool Changed = false;
1250  do {
1251  // If we are hoisting the terminator instruction, don't move one (making a
1252  // broken BB), instead clone it, and remove BI.
1253  if (isa<TerminatorInst>(I1))
1254  goto HoistTerminator;
1255 
1256  if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2))
1257  return Changed;
1258 
1259  // For a normal instruction, we just move one to right before the branch,
1260  // then replace all uses of the other with the first. Finally, we remove
1261  // the now redundant second instruction.
1262  BIParent->getInstList().splice(BI->getIterator(), BB1->getInstList(), I1);
1263  if (!I2->use_empty())
1264  I2->replaceAllUsesWith(I1);
1265  I1->andIRFlags(I2);
1266  unsigned KnownIDs[] = {LLVMContext::MD_tbaa,
1276  combineMetadata(I1, I2, KnownIDs);
1277 
1278  // I1 and I2 are being combined into a single instruction. Its debug
1279  // location is the merged locations of the original instructions.
1280  if (!isa<CallInst>(I1))
1281  I1->setDebugLoc(
1282  DILocation::getMergedLocation(I1->getDebugLoc(), I2->getDebugLoc()));
1283 
1284  I2->eraseFromParent();
1285  Changed = true;
1286 
1287  I1 = &*BB1_Itr++;
1288  I2 = &*BB2_Itr++;
1289  // Skip debug info if it is not identical.
1292  if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1293  while (isa<DbgInfoIntrinsic>(I1))
1294  I1 = &*BB1_Itr++;
1295  while (isa<DbgInfoIntrinsic>(I2))
1296  I2 = &*BB2_Itr++;
1297  }
1298  } while (I1->isIdenticalToWhenDefined(I2));
1299 
1300  return true;
1301 
1302 HoistTerminator:
1303  // It may not be possible to hoist an invoke.
1304  if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))
1305  return Changed;
1306 
1307  for (BasicBlock *Succ : successors(BB1)) {
1308  PHINode *PN;
1309  for (BasicBlock::iterator BBI = Succ->begin();
1310  (PN = dyn_cast<PHINode>(BBI)); ++BBI) {
1311  Value *BB1V = PN->getIncomingValueForBlock(BB1);
1312  Value *BB2V = PN->getIncomingValueForBlock(BB2);
1313  if (BB1V == BB2V)
1314  continue;
1315 
1316  // Check for passingValueIsAlwaysUndefined here because we would rather
1317  // eliminate undefined control flow then converting it to a select.
1318  if (passingValueIsAlwaysUndefined(BB1V, PN) ||
1320  return Changed;
1321 
1322  if (isa<ConstantExpr>(BB1V) && !isSafeToSpeculativelyExecute(BB1V))
1323  return Changed;
1324  if (isa<ConstantExpr>(BB2V) && !isSafeToSpeculativelyExecute(BB2V))
1325  return Changed;
1326  }
1327  }
1328 
1329  // Okay, it is safe to hoist the terminator.
1330  Instruction *NT = I1->clone();
1331  BIParent->getInstList().insert(BI->getIterator(), NT);
1332  if (!NT->getType()->isVoidTy()) {
1333  I1->replaceAllUsesWith(NT);
1334  I2->replaceAllUsesWith(NT);
1335  NT->takeName(I1);
1336  }
1337 
1338  IRBuilder<NoFolder> Builder(NT);
1339  // Hoisting one of the terminators from our successor is a great thing.
1340  // Unfortunately, the successors of the if/else blocks may have PHI nodes in
1341  // them. If they do, all PHI entries for BB1/BB2 must agree for all PHI
1342  // nodes, so we insert select instruction to compute the final result.
1343  std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects;
1344  for (BasicBlock *Succ : successors(BB1)) {
1345  PHINode *PN;
1346  for (BasicBlock::iterator BBI = Succ->begin();
1347  (PN = dyn_cast<PHINode>(BBI)); ++BBI) {
1348  Value *BB1V = PN->getIncomingValueForBlock(BB1);
1349  Value *BB2V = PN->getIncomingValueForBlock(BB2);
1350  if (BB1V == BB2V)
1351  continue;
1352 
1353  // These values do not agree. Insert a select instruction before NT
1354  // that determines the right value.
1355  SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)];
1356  if (!SI)
1357  SI = cast<SelectInst>(
1358  Builder.CreateSelect(BI->getCondition(), BB1V, BB2V,
1359  BB1V->getName() + "." + BB2V->getName(), BI));
1360 
1361  // Make the PHI node use the select for all incoming values for BB1/BB2
1362  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
1363  if (PN->getIncomingBlock(i) == BB1 || PN->getIncomingBlock(i) == BB2)
1364  PN->setIncomingValue(i, SI);
1365  }
1366  }
1367 
1368  // Update any PHI nodes in our new successors.
1369  for (BasicBlock *Succ : successors(BB1))
1370  AddPredecessorToBlock(Succ, BIParent, BB1);
1371 
1373  return true;
1374 }
1375 
1376 // Is it legal to place a variable in operand \c OpIdx of \c I?
1377 // FIXME: This should be promoted to Instruction.
1379  unsigned OpIdx) {
1380  // We can't have a PHI with a metadata type.
1381  if (I->getOperand(OpIdx)->getType()->isMetadataTy())
1382  return false;
1383 
1384  // Early exit.
1385  if (!isa<Constant>(I->getOperand(OpIdx)))
1386  return true;
1387 
1388  switch (I->getOpcode()) {
1389  default:
1390  return true;
1391  case Instruction::Call:
1392  case Instruction::Invoke:
1393  // FIXME: many arithmetic intrinsics have no issue taking a
1394  // variable, however it's hard to distingish these from
1395  // specials such as @llvm.frameaddress that require a constant.
1396  if (isa<IntrinsicInst>(I))
1397  return false;
1398 
1399  // Constant bundle operands may need to retain their constant-ness for
1400  // correctness.
1401  if (ImmutableCallSite(I).isBundleOperand(OpIdx))
1402  return false;
1403 
1404  return true;
1405 
1406  case Instruction::ShuffleVector:
1407  // Shufflevector masks are constant.
1408  return OpIdx != 2;
1409  case Instruction::ExtractValue:
1410  case Instruction::InsertValue:
1411  // All operands apart from the first are constant.
1412  return OpIdx == 0;
1413  case Instruction::Alloca:
1414  return false;
1415  case Instruction::GetElementPtr:
1416  if (OpIdx == 0)
1417  return true;
1418  gep_type_iterator It = std::next(gep_type_begin(I), OpIdx - 1);
1419  return It.isSequential();
1420  }
1421 }
1422 
1423 // All instructions in Insts belong to different blocks that all unconditionally
1424 // branch to a common successor. Analyze each instruction and return true if it
1425 // would be possible to sink them into their successor, creating one common
1426 // instruction instead. For every value that would be required to be provided by
1427 // PHI node (because an operand varies in each input block), add to PHIOperands.
1430  DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) {
1431  // Prune out obviously bad instructions to move. Any non-store instruction
1432  // must have exactly one use, and we check later that use is by a single,
1433  // common PHI instruction in the successor.
1434  for (auto *I : Insts) {
1435  // These instructions may change or break semantics if moved.
1436  if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) ||
1437  I->getType()->isTokenTy())
1438  return false;
1439 
1440  // Conservatively return false if I is an inline-asm instruction. Sinking
1441  // and merging inline-asm instructions can potentially create arguments
1442  // that cannot satisfy the inline-asm constraints.
1443  if (const auto *C = dyn_cast<CallInst>(I))
1444  if (C->isInlineAsm())
1445  return false;
1446 
1447  // Everything must have only one use too, apart from stores which
1448  // have no uses.
1449  if (!isa<StoreInst>(I) && !I->hasOneUse())
1450  return false;
1451  }
1452 
1453  const Instruction *I0 = Insts.front();
1454  for (auto *I : Insts)
1455  if (!I->isSameOperationAs(I0))
1456  return false;
1457 
1458  // All instructions in Insts are known to be the same opcode. If they aren't
1459  // stores, check the only user of each is a PHI or in the same block as the
1460  // instruction, because if a user is in the same block as an instruction
1461  // we're contemplating sinking, it must already be determined to be sinkable.
1462  if (!isa<StoreInst>(I0)) {
1463  auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1464  auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0);
1465  if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool {
1466  auto *U = cast<Instruction>(*I->user_begin());
1467  return (PNUse &&
1468  PNUse->getParent() == Succ &&
1469  PNUse->getIncomingValueForBlock(I->getParent()) == I) ||
1470  U->getParent() == I->getParent();
1471  }))
1472  return false;
1473  }
1474 
1475  for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) {
1476  if (I0->getOperand(OI)->getType()->isTokenTy())
1477  // Don't touch any operand of token type.
1478  return false;
1479 
1480  // Because SROA can't handle speculating stores of selects, try not
1481  // to sink loads or stores of allocas when we'd have to create a PHI for
1482  // the address operand. Also, because it is likely that loads or stores
1483  // of allocas will disappear when Mem2Reg/SROA is run, don't sink them.
1484  // This can cause code churn which can have unintended consequences down
1485  // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244.
1486  // FIXME: This is a workaround for a deficiency in SROA - see
1487  // https://llvm.org/bugs/show_bug.cgi?id=30188
1488  if (OI == 1 && isa<StoreInst>(I0) &&
1489  any_of(Insts, [](const Instruction *I) {
1490  return isa<AllocaInst>(I->getOperand(1));
1491  }))
1492  return false;
1493  if (OI == 0 && isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) {
1494  return isa<AllocaInst>(I->getOperand(0));
1495  }))
1496  return false;
1497 
1498  auto SameAsI0 = [&I0, OI](const Instruction *I) {
1499  assert(I->getNumOperands() == I0->getNumOperands());
1500  return I->getOperand(OI) == I0->getOperand(OI);
1501  };
1502  if (!all_of(Insts, SameAsI0)) {
1503  if (!canReplaceOperandWithVariable(I0, OI))
1504  // We can't create a PHI from this GEP.
1505  return false;
1506  // Don't create indirect calls! The called value is the final operand.
1507  if ((isa<CallInst>(I0) || isa<InvokeInst>(I0)) && OI == OE - 1) {
1508  // FIXME: if the call was *already* indirect, we should do this.
1509  return false;
1510  }
1511  for (auto *I : Insts)
1512  PHIOperands[I].push_back(I->getOperand(OI));
1513  }
1514  }
1515  return true;
1516 }
1517 
1518 // Assuming canSinkLastInstruction(Blocks) has returned true, sink the last
1519 // instruction of every block in Blocks to their common successor, commoning
1520 // into one instruction.
1522  auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0);
1523 
1524  // canSinkLastInstruction returning true guarantees that every block has at
1525  // least one non-terminator instruction.
1527  for (auto *BB : Blocks) {
1528  Instruction *I = BB->getTerminator();
1529  do {
1530  I = I->getPrevNode();
1531  } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front());
1532  if (!isa<DbgInfoIntrinsic>(I))
1533  Insts.push_back(I);
1534  }
1535 
1536  // The only checking we need to do now is that all users of all instructions
1537  // are the same PHI node. canSinkLastInstruction should have checked this but
1538  // it is slightly over-aggressive - it gets confused by commutative instructions
1539  // so double-check it here.
1540  Instruction *I0 = Insts.front();
1541  if (!isa<StoreInst>(I0)) {
1542  auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1543  if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool {
1544  auto *U = cast<Instruction>(*I->user_begin());
1545  return U == PNUse;
1546  }))
1547  return false;
1548  }
1549 
1550  // We don't need to do any more checking here; canSinkLastInstruction should
1551  // have done it all for us.
1552  SmallVector<Value*, 4> NewOperands;
1553  for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) {
1554  // This check is different to that in canSinkLastInstruction. There, we
1555  // cared about the global view once simplifycfg (and instcombine) have
1556  // completed - it takes into account PHIs that become trivially
1557  // simplifiable. However here we need a more local view; if an operand
1558  // differs we create a PHI and rely on instcombine to clean up the very
1559  // small mess we may make.
1560  bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) {
1561  return I->getOperand(O) != I0->getOperand(O);
1562  });
1563  if (!NeedPHI) {
1564  NewOperands.push_back(I0->getOperand(O));
1565  continue;
1566  }
1567 
1568  // Create a new PHI in the successor block and populate it.
1569  auto *Op = I0->getOperand(O);
1570  assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!");
1571  auto *PN = PHINode::Create(Op->getType(), Insts.size(),
1572  Op->getName() + ".sink", &BBEnd->front());
1573  for (auto *I : Insts)
1574  PN->addIncoming(I->getOperand(O), I->getParent());
1575  NewOperands.push_back(PN);
1576  }
1577 
1578  // Arbitrarily use I0 as the new "common" instruction; remap its operands
1579  // and move it to the start of the successor block.
1580  for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O)
1581  I0->getOperandUse(O).set(NewOperands[O]);
1582  I0->moveBefore(&*BBEnd->getFirstInsertionPt());
1583 
1584  // The debug location for the "common" instruction is the merged locations of
1585  // all the commoned instructions. We start with the original location of the
1586  // "common" instruction and iteratively merge each location in the loop below.
1587  const DILocation *Loc = I0->getDebugLoc();
1588 
1589  // Update metadata and IR flags, and merge debug locations.
1590  for (auto *I : Insts)
1591  if (I != I0) {
1592  Loc = DILocation::getMergedLocation(Loc, I->getDebugLoc());
1593  combineMetadataForCSE(I0, I);
1594  I0->andIRFlags(I);
1595  }
1596  if (!isa<CallInst>(I0))
1597  I0->setDebugLoc(Loc);
1598 
1599  if (!isa<StoreInst>(I0)) {
1600  // canSinkLastInstruction checked that all instructions were used by
1601  // one and only one PHI node. Find that now, RAUW it to our common
1602  // instruction and nuke it.
1603  assert(I0->hasOneUse());
1604  auto *PN = cast<PHINode>(*I0->user_begin());
1605  PN->replaceAllUsesWith(I0);
1606  PN->eraseFromParent();
1607  }
1608 
1609  // Finally nuke all instructions apart from the common instruction.
1610  for (auto *I : Insts)
1611  if (I != I0)
1612  I->eraseFromParent();
1613 
1614  return true;
1615 }
1616 
1617 namespace {
1618 
1619  // LockstepReverseIterator - Iterates through instructions
1620  // in a set of blocks in reverse order from the first non-terminator.
1621  // For example (assume all blocks have size n):
1622  // LockstepReverseIterator I([B1, B2, B3]);
1623  // *I-- = [B1[n], B2[n], B3[n]];
1624  // *I-- = [B1[n-1], B2[n-1], B3[n-1]];
1625  // *I-- = [B1[n-2], B2[n-2], B3[n-2]];
1626  // ...
1627  class LockstepReverseIterator {
1628  ArrayRef<BasicBlock*> Blocks;
1630  bool Fail;
1631  public:
1632  LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) :
1633  Blocks(Blocks) {
1634  reset();
1635  }
1636 
1637  void reset() {
1638  Fail = false;
1639  Insts.clear();
1640  for (auto *BB : Blocks) {
1641  Instruction *Inst = BB->getTerminator();
1642  for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1643  Inst = Inst->getPrevNode();
1644  if (!Inst) {
1645  // Block wasn't big enough.
1646  Fail = true;
1647  return;
1648  }
1649  Insts.push_back(Inst);
1650  }
1651  }
1652 
1653  bool isValid() const {
1654  return !Fail;
1655  }
1656 
1657  void operator -- () {
1658  if (Fail)
1659  return;
1660  for (auto *&Inst : Insts) {
1661  for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1662  Inst = Inst->getPrevNode();
1663  // Already at beginning of block.
1664  if (!Inst) {
1665  Fail = true;
1666  return;
1667  }
1668  }
1669  }
1670 
1672  return Insts;
1673  }
1674  };
1675 
1676 } // end anonymous namespace
1677 
1678 /// Given an unconditional branch that goes to BBEnd,
1679 /// check whether BBEnd has only two predecessors and the other predecessor
1680 /// ends with an unconditional branch. If it is true, sink any common code
1681 /// in the two predecessors to BBEnd.
1683  assert(BI1->isUnconditional());
1684  BasicBlock *BBEnd = BI1->getSuccessor(0);
1685 
1686  // We support two situations:
1687  // (1) all incoming arcs are unconditional
1688  // (2) one incoming arc is conditional
1689  //
1690  // (2) is very common in switch defaults and
1691  // else-if patterns;
1692  //
1693  // if (a) f(1);
1694  // else if (b) f(2);
1695  //
1696  // produces:
1697  //
1698  // [if]
1699  // / \
1700  // [f(1)] [if]
1701  // | | \
1702  // | | \
1703  // | [f(2)]|
1704  // \ | /
1705  // [ end ]
1706  //
1707  // [end] has two unconditional predecessor arcs and one conditional. The
1708  // conditional refers to the implicit empty 'else' arc. This conditional
1709  // arc can also be caused by an empty default block in a switch.
1710  //
1711  // In this case, we attempt to sink code from all *unconditional* arcs.
1712  // If we can sink instructions from these arcs (determined during the scan
1713  // phase below) we insert a common successor for all unconditional arcs and
1714  // connect that to [end], to enable sinking:
1715  //
1716  // [if]
1717  // / \
1718  // [x(1)] [if]
1719  // | | \
1720  // | | \
1721  // | [x(2)] |
1722  // \ / |
1723  // [sink.split] |
1724  // \ /
1725  // [ end ]
1726  //
1727  SmallVector<BasicBlock*,4> UnconditionalPreds;
1728  Instruction *Cond = nullptr;
1729  for (auto *B : predecessors(BBEnd)) {
1730  auto *T = B->getTerminator();
1731  if (isa<BranchInst>(T) && cast<BranchInst>(T)->isUnconditional())
1732  UnconditionalPreds.push_back(B);
1733  else if ((isa<BranchInst>(T) || isa<SwitchInst>(T)) && !Cond)
1734  Cond = T;
1735  else
1736  return false;
1737  }
1738  if (UnconditionalPreds.size() < 2)
1739  return false;
1740 
1741  bool Changed = false;
1742  // We take a two-step approach to tail sinking. First we scan from the end of
1743  // each block upwards in lockstep. If the n'th instruction from the end of each
1744  // block can be sunk, those instructions are added to ValuesToSink and we
1745  // carry on. If we can sink an instruction but need to PHI-merge some operands
1746  // (because they're not identical in each instruction) we add these to
1747  // PHIOperands.
1748  unsigned ScanIdx = 0;
1749  SmallPtrSet<Value*,4> InstructionsToSink;
1751  LockstepReverseIterator LRI(UnconditionalPreds);
1752  while (LRI.isValid() &&
1753  canSinkInstructions(*LRI, PHIOperands)) {
1754  DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0] << "\n");
1755  InstructionsToSink.insert((*LRI).begin(), (*LRI).end());
1756  ++ScanIdx;
1757  --LRI;
1758  }
1759 
1760  auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) {
1761  unsigned NumPHIdValues = 0;
1762  for (auto *I : *LRI)
1763  for (auto *V : PHIOperands[I])
1764  if (InstructionsToSink.count(V) == 0)
1765  ++NumPHIdValues;
1766  DEBUG(dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n");
1767  unsigned NumPHIInsts = NumPHIdValues / UnconditionalPreds.size();
1768  if ((NumPHIdValues % UnconditionalPreds.size()) != 0)
1769  NumPHIInsts++;
1770 
1771  return NumPHIInsts <= 1;
1772  };
1773 
1774  if (ScanIdx > 0 && Cond) {
1775  // Check if we would actually sink anything first! This mutates the CFG and
1776  // adds an extra block. The goal in doing this is to allow instructions that
1777  // couldn't be sunk before to be sunk - obviously, speculatable instructions
1778  // (such as trunc, add) can be sunk and predicated already. So we check that
1779  // we're going to sink at least one non-speculatable instruction.
1780  LRI.reset();
1781  unsigned Idx = 0;
1782  bool Profitable = false;
1783  while (ProfitableToSinkInstruction(LRI) && Idx < ScanIdx) {
1784  if (!isSafeToSpeculativelyExecute((*LRI)[0])) {
1785  Profitable = true;
1786  break;
1787  }
1788  --LRI;
1789  ++Idx;
1790  }
1791  if (!Profitable)
1792  return false;
1793 
1794  DEBUG(dbgs() << "SINK: Splitting edge\n");
1795  // We have a conditional edge and we're going to sink some instructions.
1796  // Insert a new block postdominating all blocks we're going to sink from.
1797  if (!SplitBlockPredecessors(BI1->getSuccessor(0), UnconditionalPreds,
1798  ".sink.split"))
1799  // Edges couldn't be split.
1800  return false;
1801  Changed = true;
1802  }
1803 
1804  // Now that we've analyzed all potential sinking candidates, perform the
1805  // actual sink. We iteratively sink the last non-terminator of the source
1806  // blocks into their common successor unless doing so would require too
1807  // many PHI instructions to be generated (currently only one PHI is allowed
1808  // per sunk instruction).
1809  //
1810  // We can use InstructionsToSink to discount values needing PHI-merging that will
1811  // actually be sunk in a later iteration. This allows us to be more
1812  // aggressive in what we sink. This does allow a false positive where we
1813  // sink presuming a later value will also be sunk, but stop half way through
1814  // and never actually sink it which means we produce more PHIs than intended.
1815  // This is unlikely in practice though.
1816  for (unsigned SinkIdx = 0; SinkIdx != ScanIdx; ++SinkIdx) {
1817  DEBUG(dbgs() << "SINK: Sink: "
1818  << *UnconditionalPreds[0]->getTerminator()->getPrevNode()
1819  << "\n");
1820 
1821  // Because we've sunk every instruction in turn, the current instruction to
1822  // sink is always at index 0.
1823  LRI.reset();
1824  if (!ProfitableToSinkInstruction(LRI)) {
1825  // Too many PHIs would be created.
1826  DEBUG(dbgs() << "SINK: stopping here, too many PHIs would be created!\n");
1827  break;
1828  }
1829 
1830  if (!sinkLastInstruction(UnconditionalPreds))
1831  return Changed;
1832  NumSinkCommons++;
1833  Changed = true;
1834  }
1835  return Changed;
1836 }
1837 
1838 /// \brief Determine if we can hoist sink a sole store instruction out of a
1839 /// conditional block.
1840 ///
1841 /// We are looking for code like the following:
1842 /// BrBB:
1843 /// store i32 %add, i32* %arrayidx2
1844 /// ... // No other stores or function calls (we could be calling a memory
1845 /// ... // function).
1846 /// %cmp = icmp ult %x, %y
1847 /// br i1 %cmp, label %EndBB, label %ThenBB
1848 /// ThenBB:
1849 /// store i32 %add5, i32* %arrayidx2
1850 /// br label EndBB
1851 /// EndBB:
1852 /// ...
1853 /// We are going to transform this into:
1854 /// BrBB:
1855 /// store i32 %add, i32* %arrayidx2
1856 /// ... //
1857 /// %cmp = icmp ult %x, %y
1858 /// %add.add5 = select i1 %cmp, i32 %add, %add5
1859 /// store i32 %add.add5, i32* %arrayidx2
1860 /// ...
1861 ///
1862 /// \return The pointer to the value of the previous store if the store can be
1863 /// hoisted into the predecessor block. 0 otherwise.
1865  BasicBlock *StoreBB, BasicBlock *EndBB) {
1866  StoreInst *StoreToHoist = dyn_cast<StoreInst>(I);
1867  if (!StoreToHoist)
1868  return nullptr;
1869 
1870  // Volatile or atomic.
1871  if (!StoreToHoist->isSimple())
1872  return nullptr;
1873 
1874  Value *StorePtr = StoreToHoist->getPointerOperand();
1875 
1876  // Look for a store to the same pointer in BrBB.
1877  unsigned MaxNumInstToLookAt = 9;
1878  for (Instruction &CurI : reverse(*BrBB)) {
1879  if (!MaxNumInstToLookAt)
1880  break;
1881  // Skip debug info.
1882  if (isa<DbgInfoIntrinsic>(CurI))
1883  continue;
1884  --MaxNumInstToLookAt;
1885 
1886  // Could be calling an instruction that affects memory like free().
1887  if (CurI.mayHaveSideEffects() && !isa<StoreInst>(CurI))
1888  return nullptr;
1889 
1890  if (auto *SI = dyn_cast<StoreInst>(&CurI)) {
1891  // Found the previous store make sure it stores to the same location.
1892  if (SI->getPointerOperand() == StorePtr)
1893  // Found the previous store, return its value operand.
1894  return SI->getValueOperand();
1895  return nullptr; // Unknown store.
1896  }
1897  }
1898 
1899  return nullptr;
1900 }
1901 
1902 /// \brief Speculate a conditional basic block flattening the CFG.
1903 ///
1904 /// Note that this is a very risky transform currently. Speculating
1905 /// instructions like this is most often not desirable. Instead, there is an MI
1906 /// pass which can do it with full awareness of the resource constraints.
1907 /// However, some cases are "obvious" and we should do directly. An example of
1908 /// this is speculating a single, reasonably cheap instruction.
1909 ///
1910 /// There is only one distinct advantage to flattening the CFG at the IR level:
1911 /// it makes very common but simplistic optimizations such as are common in
1912 /// instcombine and the DAG combiner more powerful by removing CFG edges and
1913 /// modeling their effects with easier to reason about SSA value graphs.
1914 ///
1915 ///
1916 /// An illustration of this transform is turning this IR:
1917 /// \code
1918 /// BB:
1919 /// %cmp = icmp ult %x, %y
1920 /// br i1 %cmp, label %EndBB, label %ThenBB
1921 /// ThenBB:
1922 /// %sub = sub %x, %y
1923 /// br label BB2
1924 /// EndBB:
1925 /// %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ]
1926 /// ...
1927 /// \endcode
1928 ///
1929 /// Into this IR:
1930 /// \code
1931 /// BB:
1932 /// %cmp = icmp ult %x, %y
1933 /// %sub = sub %x, %y
1934 /// %cond = select i1 %cmp, 0, %sub
1935 /// ...
1936 /// \endcode
1937 ///
1938 /// \returns true if the conditional block is removed.
1940  const TargetTransformInfo &TTI) {
1941  // Be conservative for now. FP select instruction can often be expensive.
1942  Value *BrCond = BI->getCondition();
1943  if (isa<FCmpInst>(BrCond))
1944  return false;
1945 
1946  BasicBlock *BB = BI->getParent();
1947  BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0);
1948 
1949  // If ThenBB is actually on the false edge of the conditional branch, remember
1950  // to swap the select operands later.
1951  bool Invert = false;
1952  if (ThenBB != BI->getSuccessor(0)) {
1953  assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?");
1954  Invert = true;
1955  }
1956  assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block");
1957 
1958  // Keep a count of how many times instructions are used within CondBB when
1959  // they are candidates for sinking into CondBB. Specifically:
1960  // - They are defined in BB, and
1961  // - They have no side effects, and
1962  // - All of their uses are in CondBB.
1963  SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts;
1964 
1965  unsigned SpeculationCost = 0;
1966  Value *SpeculatedStoreValue = nullptr;
1967  StoreInst *SpeculatedStore = nullptr;
1968  for (BasicBlock::iterator BBI = ThenBB->begin(),
1969  BBE = std::prev(ThenBB->end());
1970  BBI != BBE; ++BBI) {
1971  Instruction *I = &*BBI;
1972  // Skip debug info.
1973  if (isa<DbgInfoIntrinsic>(I))
1974  continue;
1975 
1976  // Only speculatively execute a single instruction (not counting the
1977  // terminator) for now.
1978  ++SpeculationCost;
1979  if (SpeculationCost > 1)
1980  return false;
1981 
1982  // Don't hoist the instruction if it's unsafe or expensive.
1983  if (!isSafeToSpeculativelyExecute(I) &&
1984  !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore(
1985  I, BB, ThenBB, EndBB))))
1986  return false;
1987  if (!SpeculatedStoreValue &&
1988  ComputeSpeculationCost(I, TTI) >
1990  return false;
1991 
1992  // Store the store speculation candidate.
1993  if (SpeculatedStoreValue)
1994  SpeculatedStore = cast<StoreInst>(I);
1995 
1996  // Do not hoist the instruction if any of its operands are defined but not
1997  // used in BB. The transformation will prevent the operand from
1998  // being sunk into the use block.
1999  for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i) {
2000  Instruction *OpI = dyn_cast<Instruction>(*i);
2001  if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects())
2002  continue; // Not a candidate for sinking.
2003 
2004  ++SinkCandidateUseCounts[OpI];
2005  }
2006  }
2007 
2008  // Consider any sink candidates which are only used in CondBB as costs for
2009  // speculation. Note, while we iterate over a DenseMap here, we are summing
2010  // and so iteration order isn't significant.
2012  I = SinkCandidateUseCounts.begin(),
2013  E = SinkCandidateUseCounts.end();
2014  I != E; ++I)
2015  if (I->first->getNumUses() == I->second) {
2016  ++SpeculationCost;
2017  if (SpeculationCost > 1)
2018  return false;
2019  }
2020 
2021  // Check that the PHI nodes can be converted to selects.
2022  bool HaveRewritablePHIs = false;
2023  for (BasicBlock::iterator I = EndBB->begin();
2024  PHINode *PN = dyn_cast<PHINode>(I); ++I) {
2025  Value *OrigV = PN->getIncomingValueForBlock(BB);
2026  Value *ThenV = PN->getIncomingValueForBlock(ThenBB);
2027 
2028  // FIXME: Try to remove some of the duplication with HoistThenElseCodeToIf.
2029  // Skip PHIs which are trivial.
2030  if (ThenV == OrigV)
2031  continue;
2032 
2033  // Don't convert to selects if we could remove undefined behavior instead.
2034  if (passingValueIsAlwaysUndefined(OrigV, PN) ||
2035  passingValueIsAlwaysUndefined(ThenV, PN))
2036  return false;
2037 
2038  HaveRewritablePHIs = true;
2039  ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV);
2040  ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV);
2041  if (!OrigCE && !ThenCE)
2042  continue; // Known safe and cheap.
2043 
2044  if ((ThenCE && !isSafeToSpeculativelyExecute(ThenCE)) ||
2045  (OrigCE && !isSafeToSpeculativelyExecute(OrigCE)))
2046  return false;
2047  unsigned OrigCost = OrigCE ? ComputeSpeculationCost(OrigCE, TTI) : 0;
2048  unsigned ThenCost = ThenCE ? ComputeSpeculationCost(ThenCE, TTI) : 0;
2049  unsigned MaxCost =
2051  if (OrigCost + ThenCost > MaxCost)
2052  return false;
2053 
2054  // Account for the cost of an unfolded ConstantExpr which could end up
2055  // getting expanded into Instructions.
2056  // FIXME: This doesn't account for how many operations are combined in the
2057  // constant expression.
2058  ++SpeculationCost;
2059  if (SpeculationCost > 1)
2060  return false;
2061  }
2062 
2063  // If there are no PHIs to process, bail early. This helps ensure idempotence
2064  // as well.
2065  if (!HaveRewritablePHIs && !(HoistCondStores && SpeculatedStoreValue))
2066  return false;
2067 
2068  // If we get here, we can hoist the instruction and if-convert.
2069  DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";);
2070 
2071  // Insert a select of the value of the speculated store.
2072  if (SpeculatedStoreValue) {
2073  IRBuilder<NoFolder> Builder(BI);
2074  Value *TrueV = SpeculatedStore->getValueOperand();
2075  Value *FalseV = SpeculatedStoreValue;
2076  if (Invert)
2077  std::swap(TrueV, FalseV);
2078  Value *S = Builder.CreateSelect(
2079  BrCond, TrueV, FalseV, TrueV->getName() + "." + FalseV->getName(), BI);
2080  SpeculatedStore->setOperand(0, S);
2081  }
2082 
2083  // Metadata can be dependent on the condition we are hoisting above.
2084  // Conservatively strip all metadata on the instruction.
2085  for (auto &I : *ThenBB)
2086  I.dropUnknownNonDebugMetadata();
2087 
2088  // Hoist the instructions.
2089  BB->getInstList().splice(BI->getIterator(), ThenBB->getInstList(),
2090  ThenBB->begin(), std::prev(ThenBB->end()));
2091 
2092  // Insert selects and rewrite the PHI operands.
2093  IRBuilder<NoFolder> Builder(BI);
2094  for (BasicBlock::iterator I = EndBB->begin();
2095  PHINode *PN = dyn_cast<PHINode>(I); ++I) {
2096  unsigned OrigI = PN->getBasicBlockIndex(BB);
2097  unsigned ThenI = PN->getBasicBlockIndex(ThenBB);
2098  Value *OrigV = PN->getIncomingValue(OrigI);
2099  Value *ThenV = PN->getIncomingValue(ThenI);
2100 
2101  // Skip PHIs which are trivial.
2102  if (OrigV == ThenV)
2103  continue;
2104 
2105  // Create a select whose true value is the speculatively executed value and
2106  // false value is the preexisting value. Swap them if the branch
2107  // destinations were inverted.
2108  Value *TrueV = ThenV, *FalseV = OrigV;
2109  if (Invert)
2110  std::swap(TrueV, FalseV);
2111  Value *V = Builder.CreateSelect(
2112  BrCond, TrueV, FalseV, TrueV->getName() + "." + FalseV->getName(), BI);
2113  PN->setIncomingValue(OrigI, V);
2114  PN->setIncomingValue(ThenI, V);
2115  }
2116 
2117  ++NumSpeculations;
2118  return true;
2119 }
2120 
2121 /// Return true if we can thread a branch across this block.
2123  BranchInst *BI = cast<BranchInst>(BB->getTerminator());
2124  unsigned Size = 0;
2125 
2126  for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
2127  if (isa<DbgInfoIntrinsic>(BBI))
2128  continue;
2129  if (Size > 10)
2130  return false; // Don't clone large BB's.
2131  ++Size;
2132 
2133  // We can only support instructions that do not define values that are
2134  // live outside of the current basic block.
2135  for (User *U : BBI->users()) {
2136  Instruction *UI = cast<Instruction>(U);
2137  if (UI->getParent() != BB || isa<PHINode>(UI))
2138  return false;
2139  }
2140 
2141  // Looks ok, continue checking.
2142  }
2143 
2144  return true;
2145 }
2146 
2147 /// If we have a conditional branch on a PHI node value that is defined in the
2148 /// same block as the branch and if any PHI entries are constants, thread edges
2149 /// corresponding to that entry to be branches to their ultimate destination.
2150 static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout &DL) {
2151  BasicBlock *BB = BI->getParent();
2152  PHINode *PN = dyn_cast<PHINode>(BI->getCondition());
2153  // NOTE: we currently cannot transform this case if the PHI node is used
2154  // outside of the block.
2155  if (!PN || PN->getParent() != BB || !PN->hasOneUse())
2156  return false;
2157 
2158  // Degenerate case of a single entry PHI.
2159  if (PN->getNumIncomingValues() == 1) {
2161  return true;
2162  }
2163 
2164  // Now we know that this block has multiple preds and two succs.
2166  return false;
2167 
2168  // Can't fold blocks that contain noduplicate or convergent calls.
2169  if (any_of(*BB, [](const Instruction &I) {
2170  const CallInst *CI = dyn_cast<CallInst>(&I);
2171  return CI && (CI->cannotDuplicate() || CI->isConvergent());
2172  }))
2173  return false;
2174 
2175  // Okay, this is a simple enough basic block. See if any phi values are
2176  // constants.
2177  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
2179  if (!CB || !CB->getType()->isIntegerTy(1))
2180  continue;
2181 
2182  // Okay, we now know that all edges from PredBB should be revectored to
2183  // branch to RealDest.
2184  BasicBlock *PredBB = PN->getIncomingBlock(i);
2185  BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue());
2186 
2187  if (RealDest == BB)
2188  continue; // Skip self loops.
2189  // Skip if the predecessor's terminator is an indirect branch.
2190  if (isa<IndirectBrInst>(PredBB->getTerminator()))
2191  continue;
2192 
2193  // The dest block might have PHI nodes, other predecessors and other
2194  // difficult cases. Instead of being smart about this, just insert a new
2195  // block that jumps to the destination block, effectively splitting
2196  // the edge we are about to create.
2197  BasicBlock *EdgeBB =
2198  BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge",
2199  RealDest->getParent(), RealDest);
2200  BranchInst::Create(RealDest, EdgeBB);
2201 
2202  // Update PHI nodes.
2203  AddPredecessorToBlock(RealDest, EdgeBB, BB);
2204 
2205  // BB may have instructions that are being threaded over. Clone these
2206  // instructions into EdgeBB. We know that there will be no uses of the
2207  // cloned instructions outside of EdgeBB.
2208  BasicBlock::iterator InsertPt = EdgeBB->begin();
2209  DenseMap<Value *, Value *> TranslateMap; // Track translated values.
2210  for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
2211  if (PHINode *PN = dyn_cast<PHINode>(BBI)) {
2212  TranslateMap[PN] = PN->getIncomingValueForBlock(PredBB);
2213  continue;
2214  }
2215  // Clone the instruction.
2216  Instruction *N = BBI->clone();
2217  if (BBI->hasName())
2218  N->setName(BBI->getName() + ".c");
2219 
2220  // Update operands due to translation.
2221  for (User::op_iterator i = N->op_begin(), e = N->op_end(); i != e; ++i) {
2222  DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(*i);
2223  if (PI != TranslateMap.end())
2224  *i = PI->second;
2225  }
2226 
2227  // Check for trivial simplification.
2228  if (Value *V = SimplifyInstruction(N, DL)) {
2229  if (!BBI->use_empty())
2230  TranslateMap[&*BBI] = V;
2231  if (!N->mayHaveSideEffects()) {
2232  delete N; // Instruction folded away, don't need actual inst
2233  N = nullptr;
2234  }
2235  } else {
2236  if (!BBI->use_empty())
2237  TranslateMap[&*BBI] = N;
2238  }
2239  // Insert the new instruction into its new home.
2240  if (N)
2241  EdgeBB->getInstList().insert(InsertPt, N);
2242  }
2243 
2244  // Loop over all of the edges from PredBB to BB, changing them to branch
2245  // to EdgeBB instead.
2246  TerminatorInst *PredBBTI = PredBB->getTerminator();
2247  for (unsigned i = 0, e = PredBBTI->getNumSuccessors(); i != e; ++i)
2248  if (PredBBTI->getSuccessor(i) == BB) {
2249  BB->removePredecessor(PredBB);
2250  PredBBTI->setSuccessor(i, EdgeBB);
2251  }
2252 
2253  // Recurse, simplifying any other constants.
2254  return FoldCondBranchOnPHI(BI, DL) | true;
2255  }
2256 
2257  return false;
2258 }
2259 
2260 /// Given a BB that starts with the specified two-entry PHI node,
2261 /// see if we can eliminate it.
2263  const DataLayout &DL) {
2264  // Ok, this is a two entry PHI node. Check to see if this is a simple "if
2265  // statement", which has a very simple dominance structure. Basically, we
2266  // are trying to find the condition that is being branched on, which
2267  // subsequently causes this merge to happen. We really want control
2268  // dependence information for this check, but simplifycfg can't keep it up
2269  // to date, and this catches most of the cases we care about anyway.
2270  BasicBlock *BB = PN->getParent();
2271  BasicBlock *IfTrue, *IfFalse;
2272  Value *IfCond = GetIfCondition(BB, IfTrue, IfFalse);
2273  if (!IfCond ||
2274  // Don't bother if the branch will be constant folded trivially.
2275  isa<ConstantInt>(IfCond))
2276  return false;
2277 
2278  // Okay, we found that we can merge this two-entry phi node into a select.
2279  // Doing so would require us to fold *all* two entry phi nodes in this block.
2280  // At some point this becomes non-profitable (particularly if the target
2281  // doesn't support cmov's). Only do this transformation if there are two or
2282  // fewer PHI nodes in this block.
2283  unsigned NumPhis = 0;
2284  for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I)
2285  if (NumPhis > 2)
2286  return false;
2287 
2288  // Loop over the PHI's seeing if we can promote them all to select
2289  // instructions. While we are at it, keep track of the instructions
2290  // that need to be moved to the dominating block.
2291  SmallPtrSet<Instruction *, 4> AggressiveInsts;
2292  unsigned MaxCostVal0 = PHINodeFoldingThreshold,
2293  MaxCostVal1 = PHINodeFoldingThreshold;
2294  MaxCostVal0 *= TargetTransformInfo::TCC_Basic;
2295  MaxCostVal1 *= TargetTransformInfo::TCC_Basic;
2296 
2297  for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
2298  PHINode *PN = cast<PHINode>(II++);
2299  if (Value *V = SimplifyInstruction(PN, DL)) {
2300  PN->replaceAllUsesWith(V);
2301  PN->eraseFromParent();
2302  continue;
2303  }
2304 
2305  if (!DominatesMergePoint(PN->getIncomingValue(0), BB, &AggressiveInsts,
2306  MaxCostVal0, TTI) ||
2307  !DominatesMergePoint(PN->getIncomingValue(1), BB, &AggressiveInsts,
2308  MaxCostVal1, TTI))
2309  return false;
2310  }
2311 
2312  // If we folded the first phi, PN dangles at this point. Refresh it. If
2313  // we ran out of PHIs then we simplified them all.
2314  PN = dyn_cast<PHINode>(BB->begin());
2315  if (!PN)
2316  return true;
2317 
2318  // Don't fold i1 branches on PHIs which contain binary operators. These can
2319  // often be turned into switches and other things.
2320  if (PN->getType()->isIntegerTy(1) &&
2321  (isa<BinaryOperator>(PN->getIncomingValue(0)) ||
2322  isa<BinaryOperator>(PN->getIncomingValue(1)) ||
2323  isa<BinaryOperator>(IfCond)))
2324  return false;
2325 
2326  // If all PHI nodes are promotable, check to make sure that all instructions
2327  // in the predecessor blocks can be promoted as well. If not, we won't be able
2328  // to get rid of the control flow, so it's not worth promoting to select
2329  // instructions.
2330  BasicBlock *DomBlock = nullptr;
2331  BasicBlock *IfBlock1 = PN->getIncomingBlock(0);
2332  BasicBlock *IfBlock2 = PN->getIncomingBlock(1);
2333  if (cast<BranchInst>(IfBlock1->getTerminator())->isConditional()) {
2334  IfBlock1 = nullptr;
2335  } else {
2336  DomBlock = *pred_begin(IfBlock1);
2337  for (BasicBlock::iterator I = IfBlock1->begin(); !isa<TerminatorInst>(I);
2338  ++I)
2339  if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I)) {
2340  // This is not an aggressive instruction that we can promote.
2341  // Because of this, we won't be able to get rid of the control flow, so
2342  // the xform is not worth it.
2343  return false;
2344  }
2345  }
2346 
2347  if (cast<BranchInst>(IfBlock2->getTerminator())->isConditional()) {
2348  IfBlock2 = nullptr;
2349  } else {
2350  DomBlock = *pred_begin(IfBlock2);
2351  for (BasicBlock::iterator I = IfBlock2->begin(); !isa<TerminatorInst>(I);
2352  ++I)
2353  if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I)) {
2354  // This is not an aggressive instruction that we can promote.
2355  // Because of this, we won't be able to get rid of the control flow, so
2356  // the xform is not worth it.
2357  return false;
2358  }
2359  }
2360 
2361  DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfCond << " T: "
2362  << IfTrue->getName() << " F: " << IfFalse->getName() << "\n");
2363 
2364  // If we can still promote the PHI nodes after this gauntlet of tests,
2365  // do all of the PHI's now.
2366  Instruction *InsertPt = DomBlock->getTerminator();
2367  IRBuilder<NoFolder> Builder(InsertPt);
2368 
2369  // Move all 'aggressive' instructions, which are defined in the
2370  // conditional parts of the if's up to the dominating block.
2371  if (IfBlock1) {
2372  for (auto &I : *IfBlock1)
2373  I.dropUnknownNonDebugMetadata();
2374  DomBlock->getInstList().splice(InsertPt->getIterator(),
2375  IfBlock1->getInstList(), IfBlock1->begin(),
2376  IfBlock1->getTerminator()->getIterator());
2377  }
2378  if (IfBlock2) {
2379  for (auto &I : *IfBlock2)
2380  I.dropUnknownNonDebugMetadata();
2381  DomBlock->getInstList().splice(InsertPt->getIterator(),
2382  IfBlock2->getInstList(), IfBlock2->begin(),
2383  IfBlock2->getTerminator()->getIterator());
2384  }
2385 
2386  while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
2387  // Change the PHI node into a select instruction.
2388  Value *TrueVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfFalse);
2389  Value *FalseVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfTrue);
2390 
2391  Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", InsertPt);
2392  PN->replaceAllUsesWith(Sel);
2393  Sel->takeName(PN);
2394  PN->eraseFromParent();
2395  }
2396 
2397  // At this point, IfBlock1 and IfBlock2 are both empty, so our if statement
2398  // has been flattened. Change DomBlock to jump directly to our new block to
2399  // avoid other simplifycfg's kicking in on the diamond.
2400  TerminatorInst *OldTI = DomBlock->getTerminator();
2401  Builder.SetInsertPoint(OldTI);
2402  Builder.CreateBr(BB);
2403  OldTI->eraseFromParent();
2404  return true;
2405 }
2406 
2407 /// If we found a conditional branch that goes to two returning blocks,
2408 /// try to merge them together into one return,
2409 /// introducing a select if the return values disagree.
2411  IRBuilder<> &Builder) {
2412  assert(BI->isConditional() && "Must be a conditional branch");
2413  BasicBlock *TrueSucc = BI->getSuccessor(0);
2414  BasicBlock *FalseSucc = BI->getSuccessor(1);
2415  ReturnInst *TrueRet = cast<ReturnInst>(TrueSucc->getTerminator());
2416  ReturnInst *FalseRet = cast<ReturnInst>(FalseSucc->getTerminator());
2417 
2418  // Check to ensure both blocks are empty (just a return) or optionally empty
2419  // with PHI nodes. If there are other instructions, merging would cause extra
2420  // computation on one path or the other.
2421  if (!TrueSucc->getFirstNonPHIOrDbg()->isTerminator())
2422  return false;
2423  if (!FalseSucc->getFirstNonPHIOrDbg()->isTerminator())
2424  return false;
2425 
2426  Builder.SetInsertPoint(BI);
2427  // Okay, we found a branch that is going to two return nodes. If
2428  // there is no return value for this function, just change the
2429  // branch into a return.
2430  if (FalseRet->getNumOperands() == 0) {
2431  TrueSucc->removePredecessor(BI->getParent());
2432  FalseSucc->removePredecessor(BI->getParent());
2433  Builder.CreateRetVoid();
2435  return true;
2436  }
2437 
2438  // Otherwise, figure out what the true and false return values are
2439  // so we can insert a new select instruction.
2440  Value *TrueValue = TrueRet->getReturnValue();
2441  Value *FalseValue = FalseRet->getReturnValue();
2442 
2443  // Unwrap any PHI nodes in the return blocks.
2444  if (PHINode *TVPN = dyn_cast_or_null<PHINode>(TrueValue))
2445  if (TVPN->getParent() == TrueSucc)
2446  TrueValue = TVPN->getIncomingValueForBlock(BI->getParent());
2447  if (PHINode *FVPN = dyn_cast_or_null<PHINode>(FalseValue))
2448  if (FVPN->getParent() == FalseSucc)
2449  FalseValue = FVPN->getIncomingValueForBlock(BI->getParent());
2450 
2451  // In order for this transformation to be safe, we must be able to
2452  // unconditionally execute both operands to the return. This is
2453  // normally the case, but we could have a potentially-trapping
2454  // constant expression that prevents this transformation from being
2455  // safe.
2456  if (ConstantExpr *TCV = dyn_cast_or_null<ConstantExpr>(TrueValue))
2457  if (TCV->canTrap())
2458  return false;
2459  if (ConstantExpr *FCV = dyn_cast_or_null<ConstantExpr>(FalseValue))
2460  if (FCV->canTrap())
2461  return false;
2462 
2463  // Okay, we collected all the mapped values and checked them for sanity, and
2464  // defined to really do this transformation. First, update the CFG.
2465  TrueSucc->removePredecessor(BI->getParent());
2466  FalseSucc->removePredecessor(BI->getParent());
2467 
2468  // Insert select instructions where needed.
2469  Value *BrCond = BI->getCondition();
2470  if (TrueValue) {
2471  // Insert a select if the results differ.
2472  if (TrueValue == FalseValue || isa<UndefValue>(FalseValue)) {
2473  } else if (isa<UndefValue>(TrueValue)) {
2474  TrueValue = FalseValue;
2475  } else {
2476  TrueValue =
2477  Builder.CreateSelect(BrCond, TrueValue, FalseValue, "retval", BI);
2478  }
2479  }
2480 
2481  Value *RI =
2482  !TrueValue ? Builder.CreateRetVoid() : Builder.CreateRet(TrueValue);
2483 
2484  (void)RI;
2485 
2486  DEBUG(dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
2487  << "\n " << *BI << "NewRet = " << *RI
2488  << "TRUEBLOCK: " << *TrueSucc << "FALSEBLOCK: " << *FalseSucc);
2489 
2491 
2492  return true;
2493 }
2494 
2495 /// Return true if the given instruction is available
2496 /// in its predecessor block. If yes, the instruction will be removed.
2498  if (!isa<BinaryOperator>(Inst) && !isa<CmpInst>(Inst))
2499  return false;
2500  for (Instruction &I : *PB) {
2501  Instruction *PBI = &I;
2502  // Check whether Inst and PBI generate the same value.
2503  if (Inst->isIdenticalTo(PBI)) {
2504  Inst->replaceAllUsesWith(PBI);
2505  Inst->eraseFromParent();
2506  return true;
2507  }
2508  }
2509  return false;
2510 }
2511 
2512 /// Return true if either PBI or BI has branch weight available, and store
2513 /// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does
2514 /// not have branch weight, use 1:1 as its weight.
2516  uint64_t &PredTrueWeight,
2517  uint64_t &PredFalseWeight,
2518  uint64_t &SuccTrueWeight,
2519  uint64_t &SuccFalseWeight) {
2520  bool PredHasWeights =
2521  PBI->extractProfMetadata(PredTrueWeight, PredFalseWeight);
2522  bool SuccHasWeights =
2523  BI->extractProfMetadata(SuccTrueWeight, SuccFalseWeight);
2524  if (PredHasWeights || SuccHasWeights) {
2525  if (!PredHasWeights)
2526  PredTrueWeight = PredFalseWeight = 1;
2527  if (!SuccHasWeights)
2528  SuccTrueWeight = SuccFalseWeight = 1;
2529  return true;
2530  } else {
2531  return false;
2532  }
2533 }
2534 
2535 /// If this basic block is simple enough, and if a predecessor branches to us
2536 /// and one of our successors, fold the block into the predecessor and use
2537 /// logical operations to pick the right destination.
2538 bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
2539  BasicBlock *BB = BI->getParent();
2540 
2541  Instruction *Cond = nullptr;
2542  if (BI->isConditional())
2543  Cond = dyn_cast<Instruction>(BI->getCondition());
2544  else {
2545  // For unconditional branch, check for a simple CFG pattern, where
2546  // BB has a single predecessor and BB's successor is also its predecessor's
2547  // successor. If such pattern exisits, check for CSE between BB and its
2548  // predecessor.
2549  if (BasicBlock *PB = BB->getSinglePredecessor())
2550  if (BranchInst *PBI = dyn_cast<BranchInst>(PB->getTerminator()))
2551  if (PBI->isConditional() &&
2552  (BI->getSuccessor(0) == PBI->getSuccessor(0) ||
2553  BI->getSuccessor(0) == PBI->getSuccessor(1))) {
2554  for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
2555  Instruction *Curr = &*I++;
2556  if (isa<CmpInst>(Curr)) {
2557  Cond = Curr;
2558  break;
2559  }
2560  // Quit if we can't remove this instruction.
2561  if (!checkCSEInPredecessor(Curr, PB))
2562  return false;
2563  }
2564  }
2565 
2566  if (!Cond)
2567  return false;
2568  }
2569 
2570  if (!Cond || (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond)) ||
2571  Cond->getParent() != BB || !Cond->hasOneUse())
2572  return false;
2573 
2574  // Make sure the instruction after the condition is the cond branch.
2575  BasicBlock::iterator CondIt = ++Cond->getIterator();
2576 
2577  // Ignore dbg intrinsics.
2578  while (isa<DbgInfoIntrinsic>(CondIt))
2579  ++CondIt;
2580 
2581  if (&*CondIt != BI)
2582  return false;
2583 
2584  // Only allow this transformation if computing the condition doesn't involve
2585  // too many instructions and these involved instructions can be executed
2586  // unconditionally. We denote all involved instructions except the condition
2587  // as "bonus instructions", and only allow this transformation when the
2588  // number of the bonus instructions does not exceed a certain threshold.
2589  unsigned NumBonusInsts = 0;
2590  for (auto I = BB->begin(); Cond != &*I; ++I) {
2591  // Ignore dbg intrinsics.
2592  if (isa<DbgInfoIntrinsic>(I))
2593  continue;
2594  if (!I->hasOneUse() || !isSafeToSpeculativelyExecute(&*I))
2595  return false;
2596  // I has only one use and can be executed unconditionally.
2597  Instruction *User = dyn_cast<Instruction>(I->user_back());
2598  if (User == nullptr || User->getParent() != BB)
2599  return false;
2600  // I is used in the same BB. Since BI uses Cond and doesn't have more slots
2601  // to use any other instruction, User must be an instruction between next(I)
2602  // and Cond.
2603  ++NumBonusInsts;
2604  // Early exits once we reach the limit.
2605  if (NumBonusInsts > BonusInstThreshold)
2606  return false;
2607  }
2608 
2609  // Cond is known to be a compare or binary operator. Check to make sure that
2610  // neither operand is a potentially-trapping constant expression.
2611  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(0)))
2612  if (CE->canTrap())
2613  return false;
2614  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(1)))
2615  if (CE->canTrap())
2616  return false;
2617 
2618  // Finally, don't infinitely unroll conditional loops.
2619  BasicBlock *TrueDest = BI->getSuccessor(0);
2620  BasicBlock *FalseDest = (BI->isConditional()) ? BI->getSuccessor(1) : nullptr;
2621  if (TrueDest == BB || FalseDest == BB)
2622  return false;
2623 
2624  for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
2625  BasicBlock *PredBlock = *PI;
2626  BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
2627 
2628  // Check that we have two conditional branches. If there is a PHI node in
2629  // the common successor, verify that the same value flows in from both
2630  // blocks.
2632  if (!PBI || PBI->isUnconditional() ||
2633  (BI->isConditional() && !SafeToMergeTerminators(BI, PBI)) ||
2634  (!BI->isConditional() &&
2635  !isProfitableToFoldUnconditional(BI, PBI, Cond, PHIs)))
2636  continue;
2637 
2638  // Determine if the two branches share a common destination.
2639  Instruction::BinaryOps Opc = Instruction::BinaryOpsEnd;
2640  bool InvertPredCond = false;
2641 
2642  if (BI->isConditional()) {
2643  if (PBI->getSuccessor(0) == TrueDest) {
2644  Opc = Instruction::Or;
2645  } else if (PBI->getSuccessor(1) == FalseDest) {
2646  Opc = Instruction::And;
2647  } else if (PBI->getSuccessor(0) == FalseDest) {
2648  Opc = Instruction::And;
2649  InvertPredCond = true;
2650  } else if (PBI->getSuccessor(1) == TrueDest) {
2651  Opc = Instruction::Or;
2652  InvertPredCond = true;
2653  } else {
2654  continue;
2655  }
2656  } else {
2657  if (PBI->getSuccessor(0) != TrueDest && PBI->getSuccessor(1) != TrueDest)
2658  continue;
2659  }
2660 
2661  DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB);
2662  IRBuilder<> Builder(PBI);
2663 
2664  // If we need to invert the condition in the pred block to match, do so now.
2665  if (InvertPredCond) {
2666  Value *NewCond = PBI->getCondition();
2667 
2668  if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) {
2669  CmpInst *CI = cast<CmpInst>(NewCond);
2670  CI->setPredicate(CI->getInversePredicate());
2671  } else {
2672  NewCond =
2673  Builder.CreateNot(NewCond, PBI->getCondition()->getName() + ".not");
2674  }
2675 
2676  PBI->setCondition(NewCond);
2677  PBI->swapSuccessors();
2678  }
2679 
2680  // If we have bonus instructions, clone them into the predecessor block.
2681  // Note that there may be multiple predecessor blocks, so we cannot move
2682  // bonus instructions to a predecessor block.
2683  ValueToValueMapTy VMap; // maps original values to cloned values
2684  // We already make sure Cond is the last instruction before BI. Therefore,
2685  // all instructions before Cond other than DbgInfoIntrinsic are bonus
2686  // instructions.
2687  for (auto BonusInst = BB->begin(); Cond != &*BonusInst; ++BonusInst) {
2688  if (isa<DbgInfoIntrinsic>(BonusInst))
2689  continue;
2690  Instruction *NewBonusInst = BonusInst->clone();
2691  RemapInstruction(NewBonusInst, VMap,
2693  VMap[&*BonusInst] = NewBonusInst;
2694 
2695  // If we moved a load, we cannot any longer claim any knowledge about
2696  // its potential value. The previous information might have been valid
2697  // only given the branch precondition.
2698  // For an analogous reason, we must also drop all the metadata whose
2699  // semantics we don't understand.
2700  NewBonusInst->dropUnknownNonDebugMetadata();
2701 
2702  PredBlock->getInstList().insert(PBI->getIterator(), NewBonusInst);
2703  NewBonusInst->takeName(&*BonusInst);
2704  BonusInst->setName(BonusInst->getName() + ".old");
2705  }
2706 
2707  // Clone Cond into the predecessor basic block, and or/and the
2708  // two conditions together.
2709  Instruction *New = Cond->clone();
2710  RemapInstruction(New, VMap,
2712  PredBlock->getInstList().insert(PBI->getIterator(), New);
2713  New->takeName(Cond);
2714  Cond->setName(New->getName() + ".old");
2715 
2716  if (BI->isConditional()) {
2717  Instruction *NewCond = cast<Instruction>(
2718  Builder.CreateBinOp(Opc, PBI->getCondition(), New, "or.cond"));
2719  PBI->setCondition(NewCond);
2720 
2721  uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
2722  bool HasWeights =
2723  extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
2724  SuccTrueWeight, SuccFalseWeight);
2725  SmallVector<uint64_t, 8> NewWeights;
2726 
2727  if (PBI->getSuccessor(0) == BB) {
2728  if (HasWeights) {
2729  // PBI: br i1 %x, BB, FalseDest
2730  // BI: br i1 %y, TrueDest, FalseDest
2731  // TrueWeight is TrueWeight for PBI * TrueWeight for BI.
2732  NewWeights.push_back(PredTrueWeight * SuccTrueWeight);
2733  // FalseWeight is FalseWeight for PBI * TotalWeight for BI +
2734  // TrueWeight for PBI * FalseWeight for BI.
2735  // We assume that total weights of a BranchInst can fit into 32 bits.
2736  // Therefore, we will not have overflow using 64-bit arithmetic.
2737  NewWeights.push_back(PredFalseWeight *
2738  (SuccFalseWeight + SuccTrueWeight) +
2739  PredTrueWeight * SuccFalseWeight);
2740  }
2741  AddPredecessorToBlock(TrueDest, PredBlock, BB);
2742  PBI->setSuccessor(0, TrueDest);
2743  }
2744  if (PBI->getSuccessor(1) == BB) {
2745  if (HasWeights) {
2746  // PBI: br i1 %x, TrueDest, BB
2747  // BI: br i1 %y, TrueDest, FalseDest
2748  // TrueWeight is TrueWeight for PBI * TotalWeight for BI +
2749  // FalseWeight for PBI * TrueWeight for BI.
2750  NewWeights.push_back(PredTrueWeight *
2751  (SuccFalseWeight + SuccTrueWeight) +
2752  PredFalseWeight * SuccTrueWeight);
2753  // FalseWeight is FalseWeight for PBI * FalseWeight for BI.
2754  NewWeights.push_back(PredFalseWeight * SuccFalseWeight);
2755  }
2756  AddPredecessorToBlock(FalseDest, PredBlock, BB);
2757  PBI->setSuccessor(1, FalseDest);
2758  }
2759  if (NewWeights.size() == 2) {
2760  // Halve the weights if any of them cannot fit in an uint32_t
2761  FitWeights(NewWeights);
2762 
2763  SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(),
2764  NewWeights.end());
2765  PBI->setMetadata(
2767  MDBuilder(BI->getContext()).createBranchWeights(MDWeights));
2768  } else
2769  PBI->setMetadata(LLVMContext::MD_prof, nullptr);
2770  } else {
2771  // Update PHI nodes in the common successors.
2772  for (unsigned i = 0, e = PHIs.size(); i != e; ++i) {
2773  ConstantInt *PBI_C = cast<ConstantInt>(
2774  PHIs[i]->getIncomingValueForBlock(PBI->getParent()));
2775  assert(PBI_C->getType()->isIntegerTy(1));
2776  Instruction *MergedCond = nullptr;
2777  if (PBI->getSuccessor(0) == TrueDest) {
2778  // Create (PBI_Cond and PBI_C) or (!PBI_Cond and BI_Value)
2779  // PBI_C is true: PBI_Cond or (!PBI_Cond and BI_Value)
2780  // is false: !PBI_Cond and BI_Value
2781  Instruction *NotCond = cast<Instruction>(
2782  Builder.CreateNot(PBI->getCondition(), "not.cond"));
2783  MergedCond = cast<Instruction>(
2784  Builder.CreateBinOp(Instruction::And, NotCond, New, "and.cond"));
2785  if (PBI_C->isOne())
2786  MergedCond = cast<Instruction>(Builder.CreateBinOp(
2787  Instruction::Or, PBI->getCondition(), MergedCond, "or.cond"));
2788  } else {
2789  // Create (PBI_Cond and BI_Value) or (!PBI_Cond and PBI_C)
2790  // PBI_C is true: (PBI_Cond and BI_Value) or (!PBI_Cond)
2791  // is false: PBI_Cond and BI_Value
2792  MergedCond = cast<Instruction>(Builder.CreateBinOp(
2793  Instruction::And, PBI->getCondition(), New, "and.cond"));
2794  if (PBI_C->isOne()) {
2795  Instruction *NotCond = cast<Instruction>(
2796  Builder.CreateNot(PBI->getCondition(), "not.cond"));
2797  MergedCond = cast<Instruction>(Builder.CreateBinOp(
2798  Instruction::Or, NotCond, MergedCond, "or.cond"));
2799  }
2800  }
2801  // Update PHI Node.
2802  PHIs[i]->setIncomingValue(PHIs[i]->getBasicBlockIndex(PBI->getParent()),
2803  MergedCond);
2804  }
2805  // Change PBI from Conditional to Unconditional.
2806  BranchInst *New_PBI = BranchInst::Create(TrueDest, PBI);
2808  PBI = New_PBI;
2809  }
2810 
2811  // If BI was a loop latch, it may have had associated loop metadata.
2812  // We need to copy it to the new latch, that is, PBI.
2813  if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop))
2814  PBI->setMetadata(LLVMContext::MD_loop, LoopMD);
2815 
2816  // TODO: If BB is reachable from all paths through PredBlock, then we
2817  // could replace PBI's branch probabilities with BI's.
2818 
2819  // Copy any debug value intrinsics into the end of PredBlock.
2820  for (Instruction &I : *BB)
2821  if (isa<DbgInfoIntrinsic>(I))
2822  I.clone()->insertBefore(PBI);
2823 
2824  return true;
2825  }
2826  return false;
2827 }
2828 
2829 // If there is only one store in BB1 and BB2, return it, otherwise return
2830 // nullptr.
2832  StoreInst *S = nullptr;
2833  for (auto *BB : {BB1, BB2}) {
2834  if (!BB)
2835  continue;
2836  for (auto &I : *BB)
2837  if (auto *SI = dyn_cast<StoreInst>(&I)) {
2838  if (S)
2839  // Multiple stores seen.
2840  return nullptr;
2841  else
2842  S = SI;
2843  }
2844  }
2845  return S;
2846 }
2847 
2848 static Value *ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB,
2849  Value *AlternativeV = nullptr) {
2850  // PHI is going to be a PHI node that allows the value V that is defined in
2851  // BB to be referenced in BB's only successor.
2852  //
2853  // If AlternativeV is nullptr, the only value we care about in PHI is V. It
2854  // doesn't matter to us what the other operand is (it'll never get used). We
2855  // could just create a new PHI with an undef incoming value, but that could
2856  // increase register pressure if EarlyCSE/InstCombine can't fold it with some
2857  // other PHI. So here we directly look for some PHI in BB's successor with V
2858  // as an incoming operand. If we find one, we use it, else we create a new
2859  // one.
2860  //
2861  // If AlternativeV is not nullptr, we care about both incoming values in PHI.
2862  // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV]
2863  // where OtherBB is the single other predecessor of BB's only successor.
2864  PHINode *PHI = nullptr;
2865  BasicBlock *Succ = BB->getSingleSuccessor();
2866 
2867  for (auto I = Succ->begin(); isa<PHINode>(I); ++I)
2868  if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) {
2869  PHI = cast<PHINode>(I);
2870  if (!AlternativeV)
2871  break;
2872 
2873  assert(std::distance(pred_begin(Succ), pred_end(Succ)) == 2);
2874  auto PredI = pred_begin(Succ);
2875  BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI;
2876  if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV)
2877  break;
2878  PHI = nullptr;
2879  }
2880  if (PHI)
2881  return PHI;
2882 
2883  // If V is not an instruction defined in BB, just return it.
2884  if (!AlternativeV &&
2885  (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB))
2886  return V;
2887 
2888  PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge", &Succ->front());
2889  PHI->addIncoming(V, BB);
2890  for (BasicBlock *PredBB : predecessors(Succ))
2891  if (PredBB != BB)
2892  PHI->addIncoming(
2893  AlternativeV ? AlternativeV : UndefValue::get(V->getType()), PredBB);
2894  return PHI;
2895 }
2896 
2898  BasicBlock *QTB, BasicBlock *QFB,
2899  BasicBlock *PostBB, Value *Address,
2900  bool InvertPCond, bool InvertQCond) {
2901  auto IsaBitcastOfPointerType = [](const Instruction &I) {
2902  return Operator::getOpcode(&I) == Instruction::BitCast &&
2903  I.getType()->isPointerTy();
2904  };
2905 
2906  // If we're not in aggressive mode, we only optimize if we have some
2907  // confidence that by optimizing we'll allow P and/or Q to be if-converted.
2908  auto IsWorthwhile = [&](BasicBlock *BB) {
2909  if (!BB)
2910  return true;
2911  // Heuristic: if the block can be if-converted/phi-folded and the
2912  // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
2913  // thread this store.
2914  unsigned N = 0;
2915  for (auto &I : *BB) {
2916  // Cheap instructions viable for folding.
2917  if (isa<BinaryOperator>(I) || isa<GetElementPtrInst>(I) ||
2918  isa<StoreInst>(I))
2919  ++N;
2920  // Free instructions.
2921  else if (isa<TerminatorInst>(I) || isa<DbgInfoIntrinsic>(I) ||
2922  IsaBitcastOfPointerType(I))
2923  continue;
2924  else
2925  return false;
2926  }
2927  return N <= PHINodeFoldingThreshold;
2928  };
2929 
2931  (!IsWorthwhile(PTB) || !IsWorthwhile(PFB) || !IsWorthwhile(QTB) ||
2932  !IsWorthwhile(QFB)))
2933  return false;
2934 
2935  // For every pointer, there must be exactly two stores, one coming from
2936  // PTB or PFB, and the other from QTB or QFB. We don't support more than one
2937  // store (to any address) in PTB,PFB or QTB,QFB.
2938  // FIXME: We could relax this restriction with a bit more work and performance
2939  // testing.
2940  StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB);
2941  StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB);
2942  if (!PStore || !QStore)
2943  return false;
2944 
2945  // Now check the stores are compatible.
2946  if (!QStore->isUnordered() || !PStore->isUnordered())
2947  return false;
2948 
2949  // Check that sinking the store won't cause program behavior changes. Sinking
2950  // the store out of the Q blocks won't change any behavior as we're sinking
2951  // from a block to its unconditional successor. But we're moving a store from
2952  // the P blocks down through the middle block (QBI) and past both QFB and QTB.
2953  // So we need to check that there are no aliasing loads or stores in
2954  // QBI, QTB and QFB. We also need to check there are no conflicting memory
2955  // operations between PStore and the end of its parent block.
2956  //
2957  // The ideal way to do this is to query AliasAnalysis, but we don't
2958  // preserve AA currently so that is dangerous. Be super safe and just
2959  // check there are no other memory operations at all.
2960  for (auto &I : *QFB->getSinglePredecessor())
2961  if (I.mayReadOrWriteMemory())
2962  return false;
2963  for (auto &I : *QFB)
2964  if (&I != QStore && I.mayReadOrWriteMemory())
2965  return false;
2966  if (QTB)
2967  for (auto &I : *QTB)
2968  if (&I != QStore && I.mayReadOrWriteMemory())
2969  return false;
2970  for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end();
2971  I != E; ++I)
2972  if (&*I != PStore && I->mayReadOrWriteMemory())
2973  return false;
2974 
2975  // OK, we're going to sink the stores to PostBB. The store has to be
2976  // conditional though, so first create the predicate.
2977  Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator())
2978  ->getCondition();
2979  Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator())
2980  ->getCondition();
2981 
2982  Value *PPHI = ensureValueAvailableInSuccessor(PStore->getValueOperand(),
2983  PStore->getParent());
2984  Value *QPHI = ensureValueAvailableInSuccessor(QStore->getValueOperand(),
2985  QStore->getParent(), PPHI);
2986 
2987  IRBuilder<> QB(&*PostBB->getFirstInsertionPt());
2988 
2989  Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond);
2990  Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond);
2991 
2992  if (InvertPCond)
2993  PPred = QB.CreateNot(PPred);
2994  if (InvertQCond)
2995  QPred = QB.CreateNot(QPred);
2996  Value *CombinedPred = QB.CreateOr(PPred, QPred);
2997 
2998  auto *T =
2999  SplitBlockAndInsertIfThen(CombinedPred, &*QB.GetInsertPoint(), false);
3000  QB.SetInsertPoint(T);
3001  StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address));
3002  AAMDNodes AAMD;
3003  PStore->getAAMetadata(AAMD, /*Merge=*/false);
3004  PStore->getAAMetadata(AAMD, /*Merge=*/true);
3005  SI->setAAMetadata(AAMD);
3006 
3007  QStore->eraseFromParent();
3008  PStore->eraseFromParent();
3009 
3010  return true;
3011 }
3012 
3014  // The intention here is to find diamonds or triangles (see below) where each
3015  // conditional block contains a store to the same address. Both of these
3016  // stores are conditional, so they can't be unconditionally sunk. But it may
3017  // be profitable to speculatively sink the stores into one merged store at the
3018  // end, and predicate the merged store on the union of the two conditions of
3019  // PBI and QBI.
3020  //
3021  // This can reduce the number of stores executed if both of the conditions are
3022  // true, and can allow the blocks to become small enough to be if-converted.
3023  // This optimization will also chain, so that ladders of test-and-set
3024  // sequences can be if-converted away.
3025  //
3026  // We only deal with simple diamonds or triangles:
3027  //
3028  // PBI or PBI or a combination of the two
3029  // / \ | \
3030  // PTB PFB | PFB
3031  // \ / | /
3032  // QBI QBI
3033  // / \ | \
3034  // QTB QFB | QFB
3035  // \ / | /
3036  // PostBB PostBB
3037  //
3038  // We model triangles as a type of diamond with a nullptr "true" block.
3039  // Triangles are canonicalized so that the fallthrough edge is represented by
3040  // a true condition, as in the diagram above.
3041  //
3042  BasicBlock *PTB = PBI->getSuccessor(0);
3043  BasicBlock *PFB = PBI->getSuccessor(1);
3044  BasicBlock *QTB = QBI->getSuccessor(0);
3045  BasicBlock *QFB = QBI->getSuccessor(1);
3046  BasicBlock *PostBB = QFB->getSingleSuccessor();
3047 
3048  bool InvertPCond = false, InvertQCond = false;
3049  // Canonicalize fallthroughs to the true branches.
3050  if (PFB == QBI->getParent()) {
3051  std::swap(PFB, PTB);
3052  InvertPCond = true;
3053  }
3054  if (QFB == PostBB) {
3055  std::swap(QFB, QTB);
3056  InvertQCond = true;
3057  }
3058 
3059  // From this point on we can assume PTB or QTB may be fallthroughs but PFB
3060  // and QFB may not. Model fallthroughs as a nullptr block.
3061  if (PTB == QBI->getParent())
3062  PTB = nullptr;
3063  if (QTB == PostBB)
3064  QTB = nullptr;
3065 
3066  // Legality bailouts. We must have at least the non-fallthrough blocks and
3067  // the post-dominating block, and the non-fallthroughs must only have one
3068  // predecessor.
3069  auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) {
3070  return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S;
3071  };
3072  if (!PostBB ||
3073  !HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) ||
3074  !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB))
3075  return false;
3076  if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) ||
3077  (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB)))
3078  return false;
3079  if (PostBB->getNumUses() != 2 || QBI->getParent()->getNumUses() != 2)
3080  return false;
3081 
3082  // OK, this is a sequence of two diamonds or triangles.
3083  // Check if there are stores in PTB or PFB that are repeated in QTB or QFB.
3084  SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses;
3085  for (auto *BB : {PTB, PFB}) {
3086  if (!BB)
3087  continue;
3088  for (auto &I : *BB)
3089  if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3090  PStoreAddresses.insert(SI->getPointerOperand());
3091  }
3092  for (auto *BB : {QTB, QFB}) {
3093  if (!BB)
3094  continue;
3095  for (auto &I : *BB)
3096  if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3097  QStoreAddresses.insert(SI->getPointerOperand());
3098  }
3099 
3100  set_intersect(PStoreAddresses, QStoreAddresses);
3101  // set_intersect mutates PStoreAddresses in place. Rename it here to make it
3102  // clear what it contains.
3103  auto &CommonAddresses = PStoreAddresses;
3104 
3105  bool Changed = false;
3106  for (auto *Address : CommonAddresses)
3107  Changed |= mergeConditionalStoreToAddress(
3108  PTB, PFB, QTB, QFB, PostBB, Address, InvertPCond, InvertQCond);
3109  return Changed;
3110 }
3111 
3112 /// If we have a conditional branch as a predecessor of another block,
3113 /// this function tries to simplify it. We know
3114 /// that PBI and BI are both conditional branches, and BI is in one of the
3115 /// successor blocks of PBI - PBI branches to BI.
3117  const DataLayout &DL) {
3118  assert(PBI->isConditional() && BI->isConditional());
3119  BasicBlock *BB = BI->getParent();
3120 
3121  // If this block ends with a branch instruction, and if there is a
3122  // predecessor that ends on a branch of the same condition, make
3123  // this conditional branch redundant.
3124  if (PBI->getCondition() == BI->getCondition() &&
3125  PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3126  // Okay, the outcome of this conditional branch is statically
3127  // knowable. If this block had a single pred, handle specially.
3128  if (BB->getSinglePredecessor()) {
3129  // Turn this into a branch on constant.
3130  bool CondIsTrue = PBI->getSuccessor(0) == BB;
3131  BI->setCondition(
3132  ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue));
3133  return true; // Nuke the branch on constant.
3134  }
3135 
3136  // Otherwise, if there are multiple predecessors, insert a PHI that merges
3137  // in the constant and simplify the block result. Subsequent passes of
3138  // simplifycfg will thread the block.
3140  pred_iterator PB = pred_begin(BB), PE = pred_end(BB);
3141  PHINode *NewPN = PHINode::Create(
3142  Type::getInt1Ty(BB->getContext()), std::distance(PB, PE),
3143  BI->getCondition()->getName() + ".pr", &BB->front());
3144  // Okay, we're going to insert the PHI node. Since PBI is not the only
3145  // predecessor, compute the PHI'd conditional value for all of the preds.
3146  // Any predecessor where the condition is not computable we keep symbolic.
3147  for (pred_iterator PI = PB; PI != PE; ++PI) {
3148  BasicBlock *P = *PI;
3149  if ((PBI = dyn_cast<BranchInst>(P->getTerminator())) && PBI != BI &&
3150  PBI->isConditional() && PBI->getCondition() == BI->getCondition() &&
3151  PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3152  bool CondIsTrue = PBI->getSuccessor(0) == BB;
3153  NewPN->addIncoming(
3154  ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue),
3155  P);
3156  } else {
3157  NewPN->addIncoming(BI->getCondition(), P);
3158  }
3159  }
3160 
3161  BI->setCondition(NewPN);
3162  return true;
3163  }
3164  }
3165 
3166  if (auto *CE = dyn_cast<ConstantExpr>(BI->getCondition()))
3167  if (CE->canTrap())
3168  return false;
3169 
3170  // If both branches are conditional and both contain stores to the same
3171  // address, remove the stores from the conditionals and create a conditional
3172  // merged store at the end.
3173  if (MergeCondStores && mergeConditionalStores(PBI, BI))
3174  return true;
3175 
3176  // If this is a conditional branch in an empty block, and if any
3177  // predecessors are a conditional branch to one of our destinations,
3178  // fold the conditions into logical ops and one cond br.
3179  BasicBlock::iterator BBI = BB->begin();
3180  // Ignore dbg intrinsics.
3181  while (isa<DbgInfoIntrinsic>(BBI))
3182  ++BBI;
3183  if (&*BBI != BI)
3184  return false;
3185 
3186  int PBIOp, BIOp;
3187  if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
3188  PBIOp = 0;
3189  BIOp = 0;
3190  } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
3191  PBIOp = 0;
3192  BIOp = 1;
3193  } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
3194  PBIOp = 1;
3195  BIOp = 0;
3196  } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
3197  PBIOp = 1;
3198  BIOp = 1;
3199  } else {
3200  return false;
3201  }
3202 
3203  // Check to make sure that the other destination of this branch
3204  // isn't BB itself. If so, this is an infinite loop that will
3205  // keep getting unwound.
3206  if (PBI->getSuccessor(PBIOp) == BB)
3207  return false;
3208 
3209  // Do not perform this transformation if it would require
3210  // insertion of a large number of select instructions. For targets
3211  // without predication/cmovs, this is a big pessimization.
3212 
3213  // Also do not perform this transformation if any phi node in the common
3214  // destination block can trap when reached by BB or PBB (PR17073). In that
3215  // case, it would be unsafe to hoist the operation into a select instruction.
3216 
3217  BasicBlock *CommonDest = PBI->getSuccessor(PBIOp);
3218  unsigned NumPhis = 0;
3219  for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II);
3220  ++II, ++NumPhis) {
3221  if (NumPhis > 2) // Disable this xform.
3222  return false;
3223 
3224  PHINode *PN = cast<PHINode>(II);
3225  Value *BIV = PN->getIncomingValueForBlock(BB);
3226  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(BIV))
3227  if (CE->canTrap())
3228  return false;
3229 
3230  unsigned PBBIdx = PN->getBasicBlockIndex(PBI->getParent());
3231  Value *PBIV = PN->getIncomingValue(PBBIdx);
3232  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(PBIV))
3233  if (CE->canTrap())
3234  return false;
3235  }
3236 
3237  // Finally, if everything is ok, fold the branches to logical ops.
3238  BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1);
3239 
3240  DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()
3241  << "AND: " << *BI->getParent());
3242 
3243  // If OtherDest *is* BB, then BB is a basic block with a single conditional
3244  // branch in it, where one edge (OtherDest) goes back to itself but the other
3245  // exits. We don't *know* that the program avoids the infinite loop
3246  // (even though that seems likely). If we do this xform naively, we'll end up
3247  // recursively unpeeling the loop. Since we know that (after the xform is
3248  // done) that the block *is* infinite if reached, we just make it an obviously
3249  // infinite loop with no cond branch.
3250  if (OtherDest == BB) {
3251  // Insert it at the end of the function, because it's either code,
3252  // or it won't matter if it's hot. :)
3253  BasicBlock *InfLoopBlock =
3254  BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
3255  BranchInst::Create(InfLoopBlock, InfLoopBlock);
3256  OtherDest = InfLoopBlock;
3257  }
3258 
3259  DEBUG(dbgs() << *PBI->getParent()->getParent());
3260 
3261  // BI may have other predecessors. Because of this, we leave
3262  // it alone, but modify PBI.
3263 
3264  // Make sure we get to CommonDest on True&True directions.
3265  Value *PBICond = PBI->getCondition();
3266  IRBuilder<NoFolder> Builder(PBI);
3267  if (PBIOp)
3268  PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not");
3269 
3270  Value *BICond = BI->getCondition();
3271  if (BIOp)
3272  BICond = Builder.CreateNot(BICond, BICond->getName() + ".not");
3273 
3274  // Merge the conditions.
3275  Value *Cond = Builder.CreateOr(PBICond, BICond, "brmerge");
3276 
3277  // Modify PBI to branch on the new condition to the new dests.
3278  PBI->setCondition(Cond);
3279  PBI->setSuccessor(0, CommonDest);
3280  PBI->setSuccessor(1, OtherDest);
3281 
3282  // Update branch weight for PBI.
3283  uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
3284  uint64_t PredCommon, PredOther, SuccCommon, SuccOther;
3285  bool HasWeights =
3286  extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
3287  SuccTrueWeight, SuccFalseWeight);
3288  if (HasWeights) {
3289  PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3290  PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3291  SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3292  SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3293  // The weight to CommonDest should be PredCommon * SuccTotal +
3294  // PredOther * SuccCommon.
3295  // The weight to OtherDest should be PredOther * SuccOther.
3296  uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) +
3297  PredOther * SuccCommon,
3298  PredOther * SuccOther};
3299  // Halve the weights if any of them cannot fit in an uint32_t
3300  FitWeights(NewWeights);
3301 
3303  MDBuilder(BI->getContext())
3304  .createBranchWeights(NewWeights[0], NewWeights[1]));
3305  }
3306 
3307  // OtherDest may have phi nodes. If so, add an entry from PBI's
3308  // block that are identical to the entries for BI's block.
3309  AddPredecessorToBlock(OtherDest, PBI->getParent(), BB);
3310 
3311  // We know that the CommonDest already had an edge from PBI to
3312  // it. If it has PHIs though, the PHIs may have different
3313  // entries for BB and PBI's BB. If so, insert a select to make
3314  // them agree.
3315  PHINode *PN;
3316  for (BasicBlock::iterator II = CommonDest->begin();
3317  (PN = dyn_cast<PHINode>(II)); ++II) {
3318  Value *BIV = PN->getIncomingValueForBlock(BB);
3319  unsigned PBBIdx = PN->getBasicBlockIndex(PBI->getParent());
3320  Value *PBIV = PN->getIncomingValue(PBBIdx);
3321  if (BIV != PBIV) {
3322  // Insert a select in PBI to pick the right value.
3323  SelectInst *NV = cast<SelectInst>(
3324  Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux"));
3325  PN->setIncomingValue(PBBIdx, NV);
3326  // Although the select has the same condition as PBI, the original branch
3327  // weights for PBI do not apply to the new select because the select's
3328  // 'logical' edges are incoming edges of the phi that is eliminated, not
3329  // the outgoing edges of PBI.
3330  if (HasWeights) {
3331  uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3332  uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3333  uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3334  uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3335  // The weight to PredCommonDest should be PredCommon * SuccTotal.
3336  // The weight to PredOtherDest should be PredOther * SuccCommon.
3337  uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther),
3338  PredOther * SuccCommon};
3339 
3340  FitWeights(NewWeights);
3341 
3343  MDBuilder(BI->getContext())
3344  .createBranchWeights(NewWeights[0], NewWeights[1]));
3345  }
3346  }
3347  }
3348 
3349  DEBUG(dbgs() << "INTO: " << *PBI->getParent());
3350  DEBUG(dbgs() << *PBI->getParent()->getParent());
3351 
3352  // This basic block is probably dead. We know it has at least
3353  // one fewer predecessor.
3354  return true;
3355 }
3356 
3357 // Simplifies a terminator by replacing it with a branch to TrueBB if Cond is
3358 // true or to FalseBB if Cond is false.
3359 // Takes care of updating the successors and removing the old terminator.
3360 // Also makes sure not to introduce new successors by assuming that edges to
3361 // non-successor TrueBBs and FalseBBs aren't reachable.
3362 static bool SimplifyTerminatorOnSelect(TerminatorInst *OldTerm, Value *Cond,
3363  BasicBlock *TrueBB, BasicBlock *FalseBB,
3364  uint32_t TrueWeight,
3365  uint32_t FalseWeight) {
3366  // Remove any superfluous successor edges from the CFG.
3367  // First, figure out which successors to preserve.
3368  // If TrueBB and FalseBB are equal, only try to preserve one copy of that
3369  // successor.
3370  BasicBlock *KeepEdge1 = TrueBB;
3371  BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr;
3372 
3373  // Then remove the rest.
3374  for (BasicBlock *Succ : OldTerm->successors()) {
3375  // Make sure only to keep exactly one copy of each edge.
3376  if (Succ == KeepEdge1)
3377  KeepEdge1 = nullptr;
3378  else if (Succ == KeepEdge2)
3379  KeepEdge2 = nullptr;
3380  else
3381  Succ->removePredecessor(OldTerm->getParent(),
3382  /*DontDeleteUselessPHIs=*/true);
3383  }
3384 
3385  IRBuilder<> Builder(OldTerm);
3386  Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc());
3387 
3388  // Insert an appropriate new terminator.
3389  if (!KeepEdge1 && !KeepEdge2) {
3390  if (TrueBB == FalseBB)
3391  // We were only looking for one successor, and it was present.
3392  // Create an unconditional branch to it.
3393  Builder.CreateBr(TrueBB);
3394  else {
3395  // We found both of the successors we were looking for.
3396  // Create a conditional branch sharing the condition of the select.
3397  BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
3398  if (TrueWeight != FalseWeight)
3400  MDBuilder(OldTerm->getContext())
3401  .createBranchWeights(TrueWeight, FalseWeight));
3402  }
3403  } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
3404  // Neither of the selected blocks were successors, so this
3405  // terminator must be unreachable.
3406  new UnreachableInst(OldTerm->getContext(), OldTerm);
3407  } else {
3408  // One of the selected values was a successor, but the other wasn't.
3409  // Insert an unconditional branch to the one that was found;
3410  // the edge to the one that wasn't must be unreachable.
3411  if (!KeepEdge1)
3412  // Only TrueBB was found.
3413  Builder.CreateBr(TrueBB);
3414  else
3415  // Only FalseBB was found.
3416  Builder.CreateBr(FalseBB);
3417  }
3418 
3420  return true;
3421 }
3422 
3423 // Replaces
3424 // (switch (select cond, X, Y)) on constant X, Y
3425 // with a branch - conditional if X and Y lead to distinct BBs,
3426 // unconditional otherwise.
3428  // Check for constant integer values in the select.
3429  ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue());
3430  ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue());
3431  if (!TrueVal || !FalseVal)
3432  return false;
3433 
3434  // Find the relevant condition and destinations.
3435  Value *Condition = Select->getCondition();
3436  BasicBlock *TrueBB = SI->findCaseValue(TrueVal).getCaseSuccessor();
3437  BasicBlock *FalseBB = SI->findCaseValue(FalseVal).getCaseSuccessor();
3438 
3439  // Get weight for TrueBB and FalseBB.
3440  uint32_t TrueWeight = 0, FalseWeight = 0;
3441  SmallVector<uint64_t, 8> Weights;
3442  bool HasWeights = HasBranchWeights(SI);
3443  if (HasWeights) {
3444  GetBranchWeights(SI, Weights);
3445  if (Weights.size() == 1 + SI->getNumCases()) {
3446  TrueWeight =
3447  (uint32_t)Weights[SI->findCaseValue(TrueVal).getSuccessorIndex()];
3448  FalseWeight =
3449  (uint32_t)Weights[SI->findCaseValue(FalseVal).getSuccessorIndex()];
3450  }
3451  }
3452 
3453  // Perform the actual simplification.
3454  return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight,
3455  FalseWeight);
3456 }
3457 
3458 // Replaces
3459 // (indirectbr (select cond, blockaddress(@fn, BlockA),
3460 // blockaddress(@fn, BlockB)))
3461 // with
3462 // (br cond, BlockA, BlockB).
3464  // Check that both operands of the select are block addresses.
3467  if (!TBA || !FBA)
3468  return false;
3469 
3470  // Extract the actual blocks.
3471  BasicBlock *TrueBB = TBA->getBasicBlock();
3472  BasicBlock *FalseBB = FBA->getBasicBlock();
3473 
3474  // Perform the actual simplification.
3475  return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0,
3476  0);
3477 }
3478 
3479 /// This is called when we find an icmp instruction
3480 /// (a seteq/setne with a constant) as the only instruction in a
3481 /// block that ends with an uncond branch. We are looking for a very specific
3482 /// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In
3483 /// this case, we merge the first two "or's of icmp" into a switch, but then the
3484 /// default value goes to an uncond block with a seteq in it, we get something
3485 /// like:
3486 ///
3487 /// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ]
3488 /// DEFAULT:
3489 /// %tmp = icmp eq i8 %A, 92
3490 /// br label %end
3491 /// end:
3492 /// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ]
3493 ///
3494 /// We prefer to split the edge to 'end' so that there is a true/false entry to
3495 /// the PHI, merging the third icmp into the switch.
3497  ICmpInst *ICI, IRBuilder<> &Builder, const DataLayout &DL,
3498  const TargetTransformInfo &TTI, unsigned BonusInstThreshold,
3499  AssumptionCache *AC) {
3500  BasicBlock *BB = ICI->getParent();
3501 
3502  // If the block has any PHIs in it or the icmp has multiple uses, it is too
3503  // complex.
3504  if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse())
3505  return false;
3506 
3507  Value *V = ICI->getOperand(0);
3508  ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1));
3509 
3510  // The pattern we're looking for is where our only predecessor is a switch on
3511  // 'V' and this block is the default case for the switch. In this case we can
3512  // fold the compared value into the switch to simplify things.
3513  BasicBlock *Pred = BB->getSinglePredecessor();
3514  if (!Pred || !isa<SwitchInst>(Pred->getTerminator()))
3515  return false;
3516 
3517  SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator());
3518  if (SI->getCondition() != V)
3519  return false;
3520 
3521  // If BB is reachable on a non-default case, then we simply know the value of
3522  // V in this block. Substitute it and constant fold the icmp instruction
3523  // away.
3524  if (SI->getDefaultDest() != BB) {
3525  ConstantInt *VVal = SI->findCaseDest(BB);
3526  assert(VVal && "Should have a unique destination value");
3527  ICI->setOperand(0, VVal);
3528 
3529  if (Value *V = SimplifyInstruction(ICI, DL)) {
3530  ICI->replaceAllUsesWith(V);
3531  ICI->eraseFromParent();
3532  }
3533  // BB is now empty, so it is likely to simplify away.
3534  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
3535  }
3536 
3537  // Ok, the block is reachable from the default dest. If the constant we're
3538  // comparing exists in one of the other edges, then we can constant fold ICI
3539  // and zap it.
3540  if (SI->findCaseValue(Cst) != SI->case_default()) {
3541  Value *V;
3542  if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
3543  V = ConstantInt::getFalse(BB->getContext());
3544  else
3545  V = ConstantInt::getTrue(BB->getContext());
3546 
3547  ICI->replaceAllUsesWith(V);
3548  ICI->eraseFromParent();
3549  // BB is now empty, so it is likely to simplify away.
3550  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
3551  }
3552 
3553  // The use of the icmp has to be in the 'end' block, by the only PHI node in
3554  // the block.
3555  BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0);
3556  PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back());
3557  if (PHIUse == nullptr || PHIUse != &SuccBlock->front() ||
3558  isa<PHINode>(++BasicBlock::iterator(PHIUse)))
3559  return false;
3560 
3561  // If the icmp is a SETEQ, then the default dest gets false, the new edge gets
3562  // true in the PHI.
3563  Constant *DefaultCst = ConstantInt::getTrue(BB->getContext());
3564  Constant *NewCst = ConstantInt::getFalse(BB->getContext());
3565 
3566  if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
3567  std::swap(DefaultCst, NewCst);
3568 
3569  // Replace ICI (which is used by the PHI for the default value) with true or
3570  // false depending on if it is EQ or NE.
3571  ICI->replaceAllUsesWith(DefaultCst);
3572  ICI->eraseFromParent();
3573 
3574  // Okay, the switch goes to this block on a default value. Add an edge from
3575  // the switch to the merge point on the compared value.
3576  BasicBlock *NewBB =
3577  BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB);
3578  SmallVector<uint64_t, 8> Weights;
3579  bool HasWeights = HasBranchWeights(SI);
3580  if (HasWeights) {
3581  GetBranchWeights(SI, Weights);
3582  if (Weights.size() == 1 + SI->getNumCases()) {
3583  // Split weight for default case to case for "Cst".
3584  Weights[0] = (Weights[0] + 1) >> 1;
3585  Weights.push_back(Weights[0]);
3586 
3587  SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
3588  SI->setMetadata(
3590  MDBuilder(SI->getContext()).createBranchWeights(MDWeights));
3591  }
3592  }
3593  SI->addCase(Cst, NewBB);
3594 
3595  // NewBB branches to the phi block, add the uncond branch and the phi entry.
3596  Builder.SetInsertPoint(NewBB);
3597  Builder.SetCurrentDebugLocation(SI->getDebugLoc());
3598  Builder.CreateBr(SuccBlock);
3599  PHIUse->addIncoming(NewCst, NewBB);
3600  return true;
3601 }
3602 
3603 /// The specified branch is a conditional branch.
3604 /// Check to see if it is branching on an or/and chain of icmp instructions, and
3605 /// fold it into a switch instruction if so.
3607  const DataLayout &DL) {
3608  Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
3609  if (!Cond)
3610  return false;
3611 
3612  // Change br (X == 0 | X == 1), T, F into a switch instruction.
3613  // If this is a bunch of seteq's or'd together, or if it's a bunch of
3614  // 'setne's and'ed together, collect them.
3615 
3616  // Try to gather values from a chain of and/or to be turned into a switch
3617  ConstantComparesGatherer ConstantCompare(Cond, DL);
3618  // Unpack the result
3619  SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals;
3620  Value *CompVal = ConstantCompare.CompValue;
3621  unsigned UsedICmps = ConstantCompare.UsedICmps;
3622  Value *ExtraCase = ConstantCompare.Extra;
3623 
3624  // If we didn't have a multiply compared value, fail.
3625  if (!CompVal)
3626  return false;
3627 
3628  // Avoid turning single icmps into a switch.
3629  if (UsedICmps <= 1)
3630  return false;
3631 
3632  bool TrueWhenEqual = (Cond->getOpcode() == Instruction::Or);
3633 
3634  // There might be duplicate constants in the list, which the switch
3635  // instruction can't handle, remove them now.
3636  array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate);
3637  Values.erase(std::unique(Values.begin(), Values.end()), Values.end());
3638 
3639  // If Extra was used, we require at least two switch values to do the
3640  // transformation. A switch with one value is just a conditional branch.
3641  if (ExtraCase && Values.size() < 2)
3642  return false;
3643 
3644  // TODO: Preserve branch weight metadata, similarly to how
3645  // FoldValueComparisonIntoPredecessors preserves it.
3646 
3647  // Figure out which block is which destination.
3648  BasicBlock *DefaultBB = BI->getSuccessor(1);
3649  BasicBlock *EdgeBB = BI->getSuccessor(0);
3650  if (!TrueWhenEqual)
3651  std::swap(DefaultBB, EdgeBB);
3652 
3653  BasicBlock *BB = BI->getParent();
3654 
3655  DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()
3656  << " cases into SWITCH. BB is:\n"
3657  << *BB);
3658 
3659  // If there are any extra values that couldn't be folded into the switch
3660  // then we evaluate them with an explicit branch first. Split the block
3661  // right before the condbr to handle it.
3662  if (ExtraCase) {
3663  BasicBlock *NewBB =
3664  BB->splitBasicBlock(BI->getIterator(), "switch.early.test");
3665  // Remove the uncond branch added to the old block.
3666  TerminatorInst *OldTI = BB->getTerminator();
3667  Builder.SetInsertPoint(OldTI);
3668 
3669  if (TrueWhenEqual)
3670  Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB);
3671  else
3672  Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB);
3673 
3674  OldTI->eraseFromParent();
3675 
3676  // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
3677  // for the edge we just added.
3678  AddPredecessorToBlock(EdgeBB, BB, NewBB);
3679 
3680  DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCase
3681  << "\nEXTRABB = " << *BB);
3682  BB = NewBB;
3683  }
3684 
3685  Builder.SetInsertPoint(BI);
3686  // Convert pointer to int before we switch.
3687  if (CompVal->getType()->isPointerTy()) {
3688  CompVal = Builder.CreatePtrToInt(
3689  CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr");
3690  }
3691 
3692  // Create the new switch instruction now.
3693  SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size());
3694 
3695  // Add all of the 'cases' to the switch instruction.
3696  for (unsigned i = 0, e = Values.size(); i != e; ++i)
3697  New->addCase(Values[i], EdgeBB);
3698 
3699  // We added edges from PI to the EdgeBB. As such, if there were any
3700  // PHI nodes in EdgeBB, they need entries to be added corresponding to
3701  // the number of edges added.
3702  for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) {
3703  PHINode *PN = cast<PHINode>(BBI);
3704  Value *InVal = PN->getIncomingValueForBlock(BB);
3705  for (unsigned i = 0, e = Values.size() - 1; i != e; ++i)
3706  PN->addIncoming(InVal, BB);
3707  }
3708 
3709  // Erase the old branch instruction.
3711 
3712  DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n');
3713  return true;
3714 }
3715 
3716 bool SimplifyCFGOpt::SimplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
3717  if (isa<PHINode>(RI->getValue()))
3718  return SimplifyCommonResume(RI);
3719  else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) &&
3720  RI->getValue() == RI->getParent()->getFirstNonPHI())
3721  // The resume must unwind the exception that caused control to branch here.
3722  return SimplifySingleResume(RI);
3723 
3724  return false;
3725 }
3726 
3727 // Simplify resume that is shared by several landing pads (phi of landing pad).
3728 bool SimplifyCFGOpt::SimplifyCommonResume(ResumeInst *RI) {
3729  BasicBlock *BB = RI->getParent();
3730 
3731  // Check that there are no other instructions except for debug intrinsics
3732  // between the phi of landing pads (RI->getValue()) and resume instruction.
3733  BasicBlock::iterator I = cast<Instruction>(RI->getValue())->getIterator(),
3734  E = RI->getIterator();
3735  while (++I != E)
3736  if (!isa<DbgInfoIntrinsic>(I))
3737  return false;
3738 
3739  SmallSet<BasicBlock *, 4> TrivialUnwindBlocks;
3740  auto *PhiLPInst = cast<PHINode>(RI->getValue());
3741 
3742  // Check incoming blocks to see if any of them are trivial.
3743  for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End;
3744  Idx++) {
3745  auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx);
3746  auto *IncomingValue = PhiLPInst->getIncomingValue(Idx);
3747 
3748  // If the block has other successors, we can not delete it because
3749  // it has other dependents.
3750  if (IncomingBB->getUniqueSuccessor() != BB)
3751  continue;
3752 
3753  auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI());
3754  // Not the landing pad that caused the control to branch here.
3755  if (IncomingValue != LandingPad)
3756  continue;
3757 
3758  bool isTrivial = true;
3759 
3760  I = IncomingBB->getFirstNonPHI()->getIterator();
3761  E = IncomingBB->getTerminator()->getIterator();
3762  while (++I != E)
3763  if (!isa<DbgInfoIntrinsic>(I)) {
3764  isTrivial = false;
3765  break;
3766  }
3767 
3768  if (isTrivial)
3769  TrivialUnwindBlocks.insert(IncomingBB);
3770  }
3771 
3772  // If no trivial unwind blocks, don't do any simplifications.
3773  if (TrivialUnwindBlocks.empty())
3774  return false;
3775 
3776  // Turn all invokes that unwind here into calls.
3777  for (auto *TrivialBB : TrivialUnwindBlocks) {
3778  // Blocks that will be simplified should be removed from the phi node.
3779  // Note there could be multiple edges to the resume block, and we need
3780  // to remove them all.
3781  while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1)
3782  BB->removePredecessor(TrivialBB, true);
3783 
3784  for (pred_iterator PI = pred_begin(TrivialBB), PE = pred_end(TrivialBB);
3785  PI != PE;) {
3786  BasicBlock *Pred = *PI++;
3787  removeUnwindEdge(Pred);
3788  }
3789 
3790  // In each SimplifyCFG run, only the current processed block can be erased.
3791  // Otherwise, it will break the iteration of SimplifyCFG pass. So instead
3792  // of erasing TrivialBB, we only remove the branch to the common resume
3793  // block so that we can later erase the resume block since it has no
3794  // predecessors.
3795  TrivialBB->getTerminator()->eraseFromParent();
3796  new UnreachableInst(RI->getContext(), TrivialBB);
3797  }
3798 
3799  // Delete the resume block if all its predecessors have been removed.
3800  if (pred_empty(BB))
3801  BB->eraseFromParent();
3802 
3803  return !TrivialUnwindBlocks.empty();
3804 }
3805 
3806 // Simplify resume that is only used by a single (non-phi) landing pad.
3807 bool SimplifyCFGOpt::SimplifySingleResume(ResumeInst *RI) {
3808  BasicBlock *BB = RI->getParent();
3810  assert(RI->getValue() == LPInst &&
3811  "Resume must unwind the exception that caused control to here");
3812 
3813  // Check that there are no other instructions except for debug intrinsics.
3814  BasicBlock::iterator I = LPInst->getIterator(), E = RI->getIterator();
3815  while (++I != E)
3816  if (!isa<DbgInfoIntrinsic>(I))
3817  return false;
3818 
3819  // Turn all invokes that unwind here into calls and delete the basic block.
3820  for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE;) {
3821  BasicBlock *Pred = *PI++;
3822  removeUnwindEdge(Pred);
3823  }
3824 
3825  // The landingpad is now unreachable. Zap it.
3826  BB->eraseFromParent();
3827  if (LoopHeaders)
3828  LoopHeaders->erase(BB);
3829  return true;
3830 }
3831 
3833  // If this is a trivial cleanup pad that executes no instructions, it can be
3834  // eliminated. If the cleanup pad continues to the caller, any predecessor
3835  // that is an EH pad will be updated to continue to the caller and any
3836  // predecessor that terminates with an invoke instruction will have its invoke
3837  // instruction converted to a call instruction. If the cleanup pad being
3838  // simplified does not continue to the caller, each predecessor will be
3839  // updated to continue to the unwind destination of the cleanup pad being
3840  // simplified.
3841  BasicBlock *BB = RI->getParent();
3842  CleanupPadInst *CPInst = RI->getCleanupPad();
3843  if (CPInst->getParent() != BB)
3844  // This isn't an empty cleanup.
3845  return false;
3846 
3847  // We cannot kill the pad if it has multiple uses. This typically arises
3848  // from unreachable basic blocks.
3849  if (!CPInst->hasOneUse())
3850  return false;
3851 
3852  // Check that there are no other instructions except for benign intrinsics.
3853  BasicBlock::iterator I = CPInst->getIterator(), E = RI->getIterator();
3854  while (++I != E) {
3855  auto *II = dyn_cast<IntrinsicInst>(I);
3856  if (!II)
3857  return false;
3858 
3859  Intrinsic::ID IntrinsicID = II->getIntrinsicID();
3860  switch (IntrinsicID) {
3861  case Intrinsic::dbg_declare:
3862  case Intrinsic::dbg_value:
3863  case Intrinsic::lifetime_end:
3864  break;
3865  default:
3866  return false;
3867  }
3868  }
3869 
3870  // If the cleanup return we are simplifying unwinds to the caller, this will
3871  // set UnwindDest to nullptr.
3872  BasicBlock *UnwindDest = RI->getUnwindDest();
3873  Instruction *DestEHPad = UnwindDest ? UnwindDest->getFirstNonPHI() : nullptr;
3874 
3875  // We're about to remove BB from the control flow. Before we do, sink any
3876  // PHINodes into the unwind destination. Doing this before changing the
3877  // control flow avoids some potentially slow checks, since we can currently
3878  // be certain that UnwindDest and BB have no common predecessors (since they
3879  // are both EH pads).
3880  if (UnwindDest) {
3881  // First, go through the PHI nodes in UnwindDest and update any nodes that
3882  // reference the block we are removing
3883  for (BasicBlock::iterator I = UnwindDest->begin(),
3884  IE = DestEHPad->getIterator();
3885  I != IE; ++I) {
3886  PHINode *DestPN = cast<PHINode>(I);
3887 
3888  int Idx = DestPN->getBasicBlockIndex(BB);
3889  // Since BB unwinds to UnwindDest, it has to be in the PHI node.
3890  assert(Idx != -1);
3891  // This PHI node has an incoming value that corresponds to a control
3892  // path through the cleanup pad we are removing. If the incoming
3893  // value is in the cleanup pad, it must be a PHINode (because we
3894  // verified above that the block is otherwise empty). Otherwise, the
3895  // value is either a constant or a value that dominates the cleanup
3896  // pad being removed.
3897  //
3898  // Because BB and UnwindDest are both EH pads, all of their
3899  // predecessors must unwind to these blocks, and since no instruction
3900  // can have multiple unwind destinations, there will be no overlap in
3901  // incoming blocks between SrcPN and DestPN.
3902  Value *SrcVal = DestPN->getIncomingValue(Idx);
3903  PHINode *SrcPN = dyn_cast<PHINode>(SrcVal);
3904 
3905  // Remove the entry for the block we are deleting.
3906  DestPN->removeIncomingValue(Idx, false);
3907 
3908  if (SrcPN && SrcPN->getParent() == BB) {
3909  // If the incoming value was a PHI node in the cleanup pad we are
3910  // removing, we need to merge that PHI node's incoming values into
3911  // DestPN.
3912  for (unsigned SrcIdx = 0, SrcE = SrcPN->getNumIncomingValues();
3913  SrcIdx != SrcE; ++SrcIdx) {
3914  DestPN->addIncoming(SrcPN->getIncomingValue(SrcIdx),
3915  SrcPN->getIncomingBlock(SrcIdx));
3916  }
3917  } else {
3918  // Otherwise, the incoming value came from above BB and
3919  // so we can just reuse it. We must associate all of BB's
3920  // predecessors with this value.
3921  for (auto *pred : predecessors(BB)) {
3922  DestPN->addIncoming(SrcVal, pred);
3923  }
3924  }
3925  }
3926 
3927  // Sink any remaining PHI nodes directly into UnwindDest.
3928  Instruction *InsertPt = DestEHPad;
3929  for (BasicBlock::iterator I = BB->begin(),
3930  IE = BB->getFirstNonPHI()->getIterator();
3931  I != IE;) {
3932  // The iterator must be incremented here because the instructions are
3933  // being moved to another block.
3934  PHINode *PN = cast<PHINode>(I++);
3935  if (PN->use_empty())
3936  // If the PHI node has no uses, just leave it. It will be erased
3937  // when we erase BB below.
3938  continue;
3939 
3940  // Otherwise, sink this PHI node into UnwindDest.
3941  // Any predecessors to UnwindDest which are not already represented
3942  // must be back edges which inherit the value from the path through
3943  // BB. In this case, the PHI value must reference itself.
3944  for (auto *pred : predecessors(UnwindDest))
3945  if (pred != BB)
3946  PN->addIncoming(PN, pred);
3947  PN->moveBefore(InsertPt);
3948  }
3949  }
3950 
3951  for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE;) {
3952  // The iterator must be updated here because we are removing this pred.
3953  BasicBlock *PredBB = *PI++;
3954  if (UnwindDest == nullptr) {
3955  removeUnwindEdge(PredBB);
3956  } else {
3957  TerminatorInst *TI = PredBB->getTerminator();
3958  TI->replaceUsesOfWith(BB, UnwindDest);
3959  }
3960  }
3961 
3962  // The cleanup pad is now unreachable. Zap it.
3963  BB->eraseFromParent();
3964  return true;
3965 }
3966 
3967 // Try to merge two cleanuppads together.
3969  // Skip any cleanuprets which unwind to caller, there is nothing to merge
3970  // with.
3971  BasicBlock *UnwindDest = RI->getUnwindDest();
3972  if (!UnwindDest)
3973  return false;
3974 
3975  // This cleanupret isn't the only predecessor of this cleanuppad, it wouldn't
3976  // be safe to merge without code duplication.
3977  if (UnwindDest->getSinglePredecessor() != RI->getParent())
3978  return false;
3979 
3980  // Verify that our cleanuppad's unwind destination is another cleanuppad.
3981  auto *SuccessorCleanupPad = dyn_cast<CleanupPadInst>(&UnwindDest->front());
3982  if (!SuccessorCleanupPad)
3983  return false;
3984 
3985  CleanupPadInst *PredecessorCleanupPad = RI->getCleanupPad();
3986  // Replace any uses of the successor cleanupad with the predecessor pad
3987  // The only cleanuppad uses should be this cleanupret, it's cleanupret and
3988  // funclet bundle operands.
3989  SuccessorCleanupPad->replaceAllUsesWith(PredecessorCleanupPad);
3990  // Remove the old cleanuppad.
3991  SuccessorCleanupPad->eraseFromParent();
3992  // Now, we simply replace the cleanupret with a branch to the unwind
3993  // destination.
3994  BranchInst::Create(UnwindDest, RI->getParent());
3995  RI->eraseFromParent();
3996 
3997  return true;
3998 }
3999 
4000 bool SimplifyCFGOpt::SimplifyCleanupReturn(CleanupReturnInst *RI) {
4001  // It is possible to transiantly have an undef cleanuppad operand because we
4002  // have deleted some, but not all, dead blocks.
4003  // Eventually, this block will be deleted.
4004  if (isa<UndefValue>(RI->getOperand(0)))
4005  return false;
4006 
4007  if (mergeCleanupPad(RI))
4008  return true;
4009 
4010  if (removeEmptyCleanup(RI))
4011  return true;
4012 
4013  return false;
4014 }
4015 
4016 bool SimplifyCFGOpt::SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
4017  BasicBlock *BB = RI->getParent();
4018  if (!BB->getFirstNonPHIOrDbg()->isTerminator())
4019  return false;
4020 
4021  // Find predecessors that end with branches.
4022  SmallVector<BasicBlock *, 8> UncondBranchPreds;
4023  SmallVector<BranchInst *, 8> CondBranchPreds;
4024  for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
4025  BasicBlock *P = *PI;
4026  TerminatorInst *PTI = P->getTerminator();
4027  if (BranchInst *BI = dyn_cast<BranchInst>(PTI)) {
4028  if (BI->isUnconditional())
4029  UncondBranchPreds.push_back(P);
4030  else
4031  CondBranchPreds.push_back(BI);
4032  }
4033  }
4034 
4035  // If we found some, do the transformation!
4036  if (!UncondBranchPreds.empty() && DupRet) {
4037  while (!UncondBranchPreds.empty()) {
4038  BasicBlock *Pred = UncondBranchPreds.pop_back_val();
4039  DEBUG(dbgs() << "FOLDING: " << *BB
4040  << "INTO UNCOND BRANCH PRED: " << *Pred);
4041  (void)FoldReturnIntoUncondBranch(RI, BB, Pred);
4042  }
4043 
4044  // If we eliminated all predecessors of the block, delete the block now.
4045  if (pred_empty(BB)) {
4046  // We know there are no successors, so just nuke the block.
4047  BB->eraseFromParent();
4048  if (LoopHeaders)
4049  LoopHeaders->erase(BB);
4050  }
4051 
4052  return true;
4053  }
4054 
4055  // Check out all of the conditional branches going to this return
4056  // instruction. If any of them just select between returns, change the
4057  // branch itself into a select/return pair.
4058  while (!CondBranchPreds.empty()) {
4059  BranchInst *BI = CondBranchPreds.pop_back_val();
4060 
4061  // Check to see if the non-BB successor is also a return block.
4062  if (isa<ReturnInst>(BI->getSuccessor(0)->getTerminator()) &&
4063  isa<ReturnInst>(BI->getSuccessor(1)->getTerminator()) &&
4064  SimplifyCondBranchToTwoReturns(BI, Builder))
4065  return true;
4066  }
4067  return false;
4068 }
4069 
4070 bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) {
4071  BasicBlock *BB = UI->getParent();
4072 
4073  bool Changed = false;
4074 
4075  // If there are any instructions immediately before the unreachable that can
4076  // be removed, do so.
4077  while (UI->getIterator() != BB->begin()) {
4078  BasicBlock::iterator BBI = UI->getIterator();
4079  --BBI;
4080  // Do not delete instructions that can have side effects which might cause
4081  // the unreachable to not be reachable; specifically, calls and volatile
4082  // operations may have this effect.
4083  if (isa<CallInst>(BBI) && !isa<DbgInfoIntrinsic>(BBI))
4084  break;
4085 
4086  if (BBI->mayHaveSideEffects()) {
4087  if (auto *SI = dyn_cast<StoreInst>(BBI)) {
4088  if (SI->isVolatile())
4089  break;
4090  } else if (auto *LI = dyn_cast<LoadInst>(BBI)) {
4091  if (LI->isVolatile())
4092  break;
4093  } else if (auto *RMWI = dyn_cast<AtomicRMWInst>(BBI)) {
4094  if (RMWI->isVolatile())
4095  break;
4096  } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(BBI)) {
4097  if (CXI->isVolatile())
4098  break;
4099  } else if (isa<CatchPadInst>(BBI)) {
4100  // A catchpad may invoke exception object constructors and such, which
4101  // in some languages can be arbitrary code, so be conservative by
4102  // default.
4103  // For CoreCLR, it just involves a type test, so can be removed.
4106  break;
4107  } else if (!isa<FenceInst>(BBI) && !isa<VAArgInst>(BBI) &&
4108  !isa<LandingPadInst>(BBI)) {
4109  break;
4110  }
4111  // Note that deleting LandingPad's here is in fact okay, although it
4112  // involves a bit of subtle reasoning. If this inst is a LandingPad,
4113  // all the predecessors of this block will be the unwind edges of Invokes,
4114  // and we can therefore guarantee this block will be erased.
4115  }
4116 
4117  // Delete this instruction (any uses are guaranteed to be dead)
4118  if (!BBI->use_empty())
4119  BBI->replaceAllUsesWith(UndefValue::get(BBI->getType()));
4120  BBI->eraseFromParent();
4121  Changed = true;
4122  }
4123 
4124  // If the unreachable instruction is the first in the block, take a gander
4125  // at all of the predecessors of this instruction, and simplify them.
4126  if (&BB->front() != UI)
4127  return Changed;
4128 
4130  for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
4131  TerminatorInst *TI = Preds[i]->getTerminator();
4132  IRBuilder<> Builder(TI);
4133  if (auto *BI = dyn_cast<BranchInst>(TI)) {
4134  if (BI->isUnconditional()) {
4135  if (BI->getSuccessor(0) == BB) {
4136  new UnreachableInst(TI->getContext(), TI);
4137  TI->eraseFromParent();
4138  Changed = true;
4139  }
4140  } else {
4141  if (BI->getSuccessor(0) == BB) {
4142  Builder.CreateBr(BI->getSuccessor(1));
4144  } else if (BI->getSuccessor(1) == BB) {
4145  Builder.CreateBr(BI->getSuccessor(0));
4147  Changed = true;
4148  }
4149  }
4150  } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
4151  for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end(); i != e;
4152  ++i)
4153  if (i.getCaseSuccessor() == BB) {
4154  BB->removePredecessor(SI->getParent());
4155  SI->removeCase(i);
4156  --i;
4157  --e;
4158  Changed = true;
4159  }
4160  } else if (auto *II = dyn_cast<InvokeInst>(TI)) {
4161  if (II->getUnwindDest() == BB) {
4162  removeUnwindEdge(TI->getParent());
4163  Changed = true;
4164  }
4165  } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) {
4166  if (CSI->getUnwindDest() == BB) {
4167  removeUnwindEdge(TI->getParent());
4168  Changed = true;
4169  continue;
4170  }
4171 
4172  for (CatchSwitchInst::handler_iterator I = CSI->handler_begin(),
4173  E = CSI->handler_end();
4174  I != E; ++I) {
4175  if (*I == BB) {
4176  CSI->removeHandler(I);
4177  --I;
4178  --E;
4179  Changed = true;
4180  }
4181  }
4182  if (CSI->getNumHandlers() == 0) {
4183  BasicBlock *CatchSwitchBB = CSI->getParent();
4184  if (CSI->hasUnwindDest()) {
4185  // Redirect preds to the unwind dest
4186  CatchSwitchBB->replaceAllUsesWith(CSI->getUnwindDest());
4187  } else {
4188  // Rewrite all preds to unwind to caller (or from invoke to call).
4189  SmallVector<BasicBlock *, 8> EHPreds(predecessors(CatchSwitchBB));
4190  for (BasicBlock *EHPred : EHPreds)
4191  removeUnwindEdge(EHPred);
4192  }
4193  // The catchswitch is no longer reachable.
4194  new UnreachableInst(CSI->getContext(), CSI);
4195  CSI->eraseFromParent();
4196  Changed = true;
4197  }
4198  } else if (isa<CleanupReturnInst>(TI)) {
4199  new UnreachableInst(TI->getContext(), TI);
4200  TI->eraseFromParent();
4201  Changed = true;
4202  }
4203  }
4204 
4205  // If this block is now dead, remove it.
4206  if (pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) {
4207  // We know there are no successors, so just nuke the block.
4208  BB->eraseFromParent();
4209  if (LoopHeaders)
4210  LoopHeaders->erase(BB);
4211  return true;
4212  }
4213 
4214  return Changed;
4215 }
4216 
4218  assert(Cases.size() >= 1);
4219 
4220  array_pod_sort(Cases.begin(), Cases.end(), ConstantIntSortPredicate);
4221  for (size_t I = 1, E = Cases.size(); I != E; ++I) {
4222  if (Cases[I - 1]->getValue() != Cases[I]->getValue() + 1)
4223  return false;
4224  }
4225  return true;
4226 }
4227 
4228 /// Turn a switch with two reachable destinations into an integer range
4229 /// comparison and branch.
4230 static bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder) {
4231  assert(SI->getNumCases() > 1 && "Degenerate switch?");
4232 
4233  bool HasDefault =
4234  !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
4235 
4236  // Partition the cases into two sets with different destinations.
4237  BasicBlock *DestA = HasDefault ? SI->getDefaultDest() : nullptr;
4238  BasicBlock *DestB = nullptr;
4241 
4242  for (SwitchInst::CaseIt I : SI->cases()) {
4243  BasicBlock *Dest = I.getCaseSuccessor();
4244  if (!DestA)
4245  DestA = Dest;
4246  if (Dest == DestA) {
4247  CasesA.push_back(I.getCaseValue());
4248  continue;
4249  }
4250  if (!DestB)
4251  DestB = Dest;
4252  if (Dest == DestB) {
4253  CasesB.push_back(I.getCaseValue());
4254  continue;
4255  }
4256  return false; // More than two destinations.
4257  }
4258 
4259  assert(DestA && DestB &&
4260  "Single-destination switch should have been folded.");
4261  assert(DestA != DestB);
4262  assert(DestB != SI->getDefaultDest());
4263  assert(!CasesB.empty() && "There must be non-default cases.");
4264  assert(!CasesA.empty() || HasDefault);
4265 
4266  // Figure out if one of the sets of cases form a contiguous range.
4267  SmallVectorImpl<ConstantInt *> *ContiguousCases = nullptr;
4268  BasicBlock *ContiguousDest = nullptr;
4269  BasicBlock *OtherDest = nullptr;
4270  if (!CasesA.empty() && CasesAreContiguous(CasesA)) {
4271  ContiguousCases = &CasesA;
4272  ContiguousDest = DestA;
4273  OtherDest = DestB;
4274  } else if (CasesAreContiguous(CasesB)) {
4275  ContiguousCases = &CasesB;
4276  ContiguousDest = DestB;
4277  OtherDest = DestA;
4278  } else
4279  return false;
4280 
4281  // Start building the compare and branch.
4282 
4283  Constant *Offset = ConstantExpr::getNeg(ContiguousCases->back());
4284  Constant *NumCases =
4285  ConstantInt::get(Offset->getType(), ContiguousCases->size());
4286 
4287  Value *Sub = SI->getCondition();
4288  if (!Offset->isNullValue())
4289  Sub = Builder.CreateAdd(Sub, Offset, Sub->getName() + ".off");
4290 
4291  Value *Cmp;
4292  // If NumCases overflowed, then all possible values jump to the successor.
4293  if (NumCases->isNullValue() && !ContiguousCases->empty())
4294  Cmp = ConstantInt::getTrue(SI->getContext());
4295  else
4296  Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch");
4297  BranchInst *NewBI = Builder.CreateCondBr(Cmp, ContiguousDest, OtherDest);
4298 
4299  // Update weight for the newly-created conditional branch.
4300  if (HasBranchWeights(SI)) {
4301  SmallVector<uint64_t, 8> Weights;
4302  GetBranchWeights(SI, Weights);
4303  if (Weights.size() == 1 + SI->getNumCases()) {
4304  uint64_t TrueWeight = 0;
4305  uint64_t FalseWeight = 0;
4306  for (size_t I = 0, E = Weights.size(); I != E; ++I) {
4307  if (SI->getSuccessor(I) == ContiguousDest)
4308  TrueWeight += Weights[I];
4309  else
4310  FalseWeight += Weights[I];
4311  }
4312  while (TrueWeight > UINT32_MAX || FalseWeight > UINT32_MAX) {
4313  TrueWeight /= 2;
4314  FalseWeight /= 2;
4315  }
4317  MDBuilder(SI->getContext())
4318  .createBranchWeights((uint32_t)TrueWeight,
4319  (uint32_t)FalseWeight));
4320  }
4321  }
4322 
4323  // Prune obsolete incoming values off the successors' PHI nodes.
4324  for (auto BBI = ContiguousDest->begin(); isa<PHINode>(BBI); ++BBI) {
4325  unsigned PreviousEdges = ContiguousCases->size();
4326  if (ContiguousDest == SI->getDefaultDest())
4327  ++PreviousEdges;
4328  for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4329  cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4330  }
4331  for (auto BBI = OtherDest->begin(); isa<PHINode>(BBI); ++BBI) {
4332  unsigned PreviousEdges = SI->getNumCases() - ContiguousCases->size();
4333  if (OtherDest == SI->getDefaultDest())
4334  ++PreviousEdges;
4335  for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4336  cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4337  }
4338 
4339  // Drop the switch.
4340  SI->eraseFromParent();
4341 
4342  return true;
4343 }
4344 
4345 /// Compute masked bits for the condition of a switch
4346 /// and use it to remove dead cases.
4348  const DataLayout &DL) {
4349  Value *Cond = SI->getCondition();
4350  unsigned Bits = Cond->getType()->getIntegerBitWidth();
4351  APInt KnownZero(Bits, 0), KnownOne(Bits, 0);
4352  computeKnownBits(Cond, KnownZero, KnownOne, DL, 0, AC, SI);
4353 
4354  // We can also eliminate cases by determining that their values are outside of
4355  // the limited range of the condition based on how many significant (non-sign)
4356  // bits are in the condition value.
4357  unsigned ExtraSignBits = ComputeNumSignBits(Cond, DL, 0, AC, SI) - 1;
4358  unsigned MaxSignificantBitsInCond = Bits - ExtraSignBits;
4359 
4360  // Gather dead cases.
4362  for (auto &Case : SI->cases()) {
4363  APInt CaseVal = Case.getCaseValue()->getValue();
4364  if ((CaseVal & KnownZero) != 0 || (CaseVal & KnownOne) != KnownOne ||
4365  (CaseVal.getMinSignedBits() > MaxSignificantBitsInCond)) {
4366  DeadCases.push_back(Case.getCaseValue());
4367  DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseVal << " is dead.\n");
4368  }
4369  }
4370 
4371  // If we can prove that the cases must cover all possible values, the
4372  // default destination becomes dead and we can remove it. If we know some
4373  // of the bits in the value, we can use that to more precisely compute the
4374  // number of possible unique case values.
4375  bool HasDefault =
4376  !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
4377  const unsigned NumUnknownBits =
4378  Bits - (KnownZero.Or(KnownOne)).countPopulation();
4379  assert(NumUnknownBits <= Bits);
4380  if (HasDefault && DeadCases.empty() &&
4381  NumUnknownBits < 64 /* avoid overflow */ &&
4382  SI->getNumCases() == (1ULL << NumUnknownBits)) {
4383  DEBUG(dbgs() << "SimplifyCFG: switch default is dead.\n");
4384  BasicBlock *NewDefault =
4386  SI->setDefaultDest(&*NewDefault);
4387  SplitBlock(&*NewDefault, &NewDefault->front());
4388  auto *OldTI = NewDefault->getTerminator();
4389  new UnreachableInst(SI->getContext(), OldTI);
4391  return true;
4392  }
4393 
4394  SmallVector<uint64_t, 8> Weights;
4395  bool HasWeight = HasBranchWeights(SI);
4396  if (HasWeight) {
4397  GetBranchWeights(SI, Weights);
4398  HasWeight = (Weights.size() == 1 + SI->getNumCases());
4399  }
4400 
4401  // Remove dead cases from the switch.
4402  for (ConstantInt *DeadCase : DeadCases) {
4403  SwitchInst::CaseIt Case = SI->findCaseValue(DeadCase);
4404  assert(Case != SI->case_default() &&
4405  "Case was not found. Probably mistake in DeadCases forming.");
4406  if (HasWeight) {
4407  std::swap(Weights[Case.getCaseIndex() + 1], Weights.back());
4408  Weights.pop_back();
4409  }
4410 
4411  // Prune unused values from PHI nodes.
4413  SI->removeCase(Case);
4414  }
4415  if (HasWeight && Weights.size() >= 2) {
4416  SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
4418  MDBuilder(SI->getParent()->getContext())
4419  .createBranchWeights(MDWeights));
4420  }
4421 
4422  return !DeadCases.empty();
4423 }
4424 
4425 /// If BB would be eligible for simplification by
4426 /// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated
4427 /// by an unconditional branch), look at the phi node for BB in the successor
4428 /// block and see if the incoming value is equal to CaseValue. If so, return
4429 /// the phi node, and set PhiIndex to BB's index in the phi node.
4431  BasicBlock *BB, int *PhiIndex) {
4432  if (BB->getFirstNonPHIOrDbg() != BB->getTerminator())
4433  return nullptr; // BB must be empty to be a candidate for simplification.
4434  if (!BB->getSinglePredecessor())
4435  return nullptr; // BB must be dominated by the switch.
4436 
4438  if (!Branch || !Branch->isUnconditional())
4439  return nullptr; // Terminator must be unconditional branch.
4440 
4441  BasicBlock *Succ = Branch->getSuccessor(0);
4442 
4443  BasicBlock::iterator I = Succ->begin();
4444  while (PHINode *PHI = dyn_cast<PHINode>(I++)) {
4445  int Idx = PHI->getBasicBlockIndex(BB);
4446  assert(Idx >= 0 && "PHI has no entry for predecessor?");
4447 
4448  Value *InValue = PHI->getIncomingValue(Idx);
4449  if (InValue != CaseValue)
4450  continue;
4451 
4452  *PhiIndex = Idx;
4453  return PHI;
4454  }
4455 
4456  return nullptr;
4457 }
4458 
4459 /// Try to forward the condition of a switch instruction to a phi node
4460 /// dominated by the switch, if that would mean that some of the destination
4461 /// blocks of the switch can be folded away.
4462 /// Returns true if a change is made.
4464  typedef DenseMap<PHINode *, SmallVector<int, 4>> ForwardingNodesMap;
4465  ForwardingNodesMap ForwardingNodes;
4466 
4467  for (SwitchInst::CaseIt I = SI->case_begin(), E = SI->case_end(); I != E;
4468  ++I) {
4469  ConstantInt *CaseValue = I.getCaseValue();
4470  BasicBlock *CaseDest = I.getCaseSuccessor();
4471 
4472  int PhiIndex;
4473  PHINode *PHI =
4474  FindPHIForConditionForwarding(CaseValue, CaseDest, &PhiIndex);
4475  if (!PHI)
4476  continue;
4477 
4478  ForwardingNodes[PHI].push_back(PhiIndex);
4479  }
4480 
4481  bool Changed = false;
4482 
4483  for (ForwardingNodesMap::iterator I = ForwardingNodes.begin(),
4484  E = ForwardingNodes.end();
4485  I != E; ++I) {
4486  PHINode *Phi = I->first;
4487  SmallVectorImpl<int> &Indexes = I->second;
4488 
4489  if (Indexes.size() < 2)
4490  continue;
4491 
4492  for (size_t I = 0, E = Indexes.size(); I != E; ++I)
4493  Phi->setIncomingValue(Indexes[I], SI->getCondition());
4494  Changed = true;
4495  }
4496 
4497  return Changed;
4498 }
4499 
4500 /// Return true if the backend will be able to handle
4501 /// initializing an array of constants like C.
4503  if (C->isThreadDependent())
4504  return false;
4505  if (C->isDLLImportDependent())
4506  return false;
4507 
4508  if (!isa<ConstantFP>(C) && !isa<ConstantInt>(C) &&
4509  !isa<ConstantPointerNull>(C) && !isa<GlobalValue>(C) &&
4510  !isa<UndefValue>(C) && !isa<ConstantExpr>(C))
4511  return false;
4512 
4513  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
4514  if (!CE->isGEPWithNoNotionalOverIndexing())
4515  return false;
4516  if (!ValidLookupTableConstant(CE->getOperand(0), TTI))
4517  return false;
4518  }
4519 
4521  return false;
4522 
4523  return true;
4524 }
4525 
4526 /// If V is a Constant, return it. Otherwise, try to look up
4527 /// its constant value in ConstantPool, returning 0 if it's not there.
4528 static Constant *
4531  if (Constant *C = dyn_cast<Constant>(V))
4532  return C;
4533  return ConstantPool.lookup(V);
4534 }
4535 
4536 /// Try to fold instruction I into a constant. This works for
4537 /// simple instructions such as binary operations where both operands are
4538 /// constant or can be replaced by constants from the ConstantPool. Returns the
4539 /// resulting constant on success, 0 otherwise.
4540 static Constant *
4543  if (SelectInst *Select = dyn_cast<SelectInst>(I)) {
4544  Constant *A = LookupConstant(Select->getCondition(), ConstantPool);
4545  if (!A)
4546  return nullptr;
4547  if (A->isAllOnesValue())
4548  return LookupConstant(Select->getTrueValue(), ConstantPool);
4549  if (A->isNullValue())
4550  return LookupConstant(Select->getFalseValue(), ConstantPool);
4551  return nullptr;
4552  }
4553 
4555  for (unsigned N = 0, E = I->getNumOperands(); N != E; ++N) {
4557  COps.push_back(A);
4558  else
4559  return nullptr;
4560  }
4561 
4562  if (CmpInst *Cmp = dyn_cast<CmpInst>(I)) {
4563  return ConstantFoldCompareInstOperands(Cmp->getPredicate(), COps[0],
4564  COps[1], DL);
4565  }
4566 
4567  return ConstantFoldInstOperands(I, COps, DL);
4568 }
4569 
4570 /// Try to determine the resulting constant values in phi nodes
4571 /// at the common destination basic block, *CommonDest, for one of the case
4572 /// destionations CaseDest corresponding to value CaseVal (0 for the default
4573 /// case), of a switch instruction SI.
4574 static bool
4576  BasicBlock **CommonDest,
4577  SmallVectorImpl<std::pair<PHINode *, Constant *>> &Res,
4578  const DataLayout &DL, const TargetTransformInfo &TTI) {
4579  // The block from which we enter the common destination.
4580  BasicBlock *Pred = SI->getParent();
4581 
4582  // If CaseDest is empty except for some side-effect free instructions through
4583  // which we can constant-propagate the CaseVal, continue to its successor.
4585  ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal));
4586  for (BasicBlock::iterator I = CaseDest->begin(), E = CaseDest->end(); I != E;
4587  ++I) {
4588  if (TerminatorInst *T = dyn_cast<TerminatorInst>(I)) {
4589  // If the terminator is a simple branch, continue to the next block.
4590  if (T->getNumSuccessors() != 1 || T->isExceptional())
4591  return false;
4592  Pred = CaseDest;
4593  CaseDest = T->getSuccessor(0);
4594  } else if (isa<DbgInfoIntrinsic>(I)) {
4595  // Skip debug intrinsic.
4596  continue;
4597  } else if (Constant *C = ConstantFold(&*I, DL, ConstantPool)) {
4598  // Instruction is side-effect free and constant.
4599 
4600  // If the instruction has uses outside this block or a phi node slot for
4601  // the block, it is not safe to bypass the instruction since it would then
4602  // no longer dominate all its uses.
4603  for (auto &Use : I->uses()) {
4604  User *User = Use.getUser();
4605  if (Instruction *I = dyn_cast<Instruction>(User))
4606  if (I->getParent() == CaseDest)
4607  continue;
4608  if (PHINode *Phi = dyn_cast<PHINode>(User))
4609  if (Phi->getIncomingBlock(Use) == CaseDest)
4610  continue;
4611  return false;
4612  }
4613 
4614  ConstantPool.insert(std::make_pair(&*I, C));
4615  } else {
4616  break;
4617  }
4618  }
4619 
4620  // If we did not have a CommonDest before, use the current one.
4621  if (!*CommonDest)
4622  *CommonDest = CaseDest;
4623  // If the destination isn't the common one, abort.
4624  if (CaseDest != *CommonDest)
4625  return false;
4626 
4627  // Get the values for this case from phi nodes in the destination block.
4628  BasicBlock::iterator I = (*CommonDest)->begin();
4629  while (PHINode *PHI = dyn_cast<PHINode>(I++)) {
4630  int Idx = PHI->getBasicBlockIndex(Pred);
4631  if (Idx == -1)
4632  continue;
4633 
4634  Constant *ConstVal =
4635  LookupConstant(PHI->getIncomingValue(Idx), ConstantPool);
4636  if (!ConstVal)
4637  return false;
4638 
4639  // Be conservative about which kinds of constants we support.
4640  if (!ValidLookupTableConstant(ConstVal, TTI))
4641  return false;
4642 
4643  Res.push_back(std::make_pair(PHI, ConstVal));
4644  }
4645 
4646  return Res.size() > 0;
4647 }
4648 
4649 // Helper function used to add CaseVal to the list of cases that generate
4650 // Result.
4651 static void MapCaseToResult(ConstantInt *CaseVal,
4652  SwitchCaseResultVectorTy &UniqueResults,
4653  Constant *Result) {
4654  for (auto &I : UniqueResults) {
4655  if (I.first == Result) {
4656  I.second.push_back(CaseVal);
4657  return;
4658  }
4659  }
4660  UniqueResults.push_back(
4661  std::make_pair(Result, SmallVector<ConstantInt *, 4>(1, CaseVal)));
4662 }
4663 
4664 // Helper function that initializes a map containing
4665 // results for the PHI node of the common destination block for a switch
4666 // instruction. Returns false if multiple PHI nodes have been found or if
4667 // there is not a common destination block for the switch.
4669  BasicBlock *&CommonDest,
4670  SwitchCaseResultVectorTy &UniqueResults,
4671  Constant *&DefaultResult,
4672  const DataLayout &DL,
4673  const TargetTransformInfo &TTI) {
4674  for (auto &I : SI->cases()) {
4675  ConstantInt *CaseVal = I.getCaseValue();
4676 
4677  // Resulting value at phi nodes for this case value.
4678  SwitchCaseResultsTy Results;
4679  if (!GetCaseResults(SI, CaseVal, I.getCaseSuccessor(), &CommonDest, Results,
4680  DL, TTI))
4681  return false;
4682 
4683  // Only one value per case is permitted
4684  if (Results.size() > 1)
4685  return false;
4686  MapCaseToResult(CaseVal, UniqueResults, Results.begin()->second);
4687 
4688  // Check the PHI consistency.
4689  if (!PHI)
4690  PHI = Results[0].first;
4691  else if (PHI != Results[0].first)
4692  return false;
4693  }
4694  // Find the default result value.
4696  BasicBlock *DefaultDest = SI->getDefaultDest();
4697  GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, DefaultResults,
4698  DL, TTI);
4699  // If the default value is not found abort unless the default destination
4700  // is unreachable.
4701  DefaultResult =
4702  DefaultResults.size() == 1 ? DefaultResults.begin()->second : nullptr;
4703  if ((!DefaultResult &&
4704  !isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg())))
4705  return false;
4706 
4707  return true;
4708 }
4709 
4710 // Helper function that checks if it is possible to transform a switch with only
4711 // two cases (or two cases + default) that produces a result into a select.
4712 // Example:
4713 // switch (a) {
4714 // case 10: %0 = icmp eq i32 %a, 10
4715 // return 10; %1 = select i1 %0, i32 10, i32 4
4716 // case 20: ----> %2 = icmp eq i32 %a, 20
4717 // return 2; %3 = select i1 %2, i32 2, i32 %1
4718 // default:
4719 // return 4;
4720 // }
4721 static Value *ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector,
4722  Constant *DefaultResult, Value *Condition,
4723  IRBuilder<> &Builder) {
4724  assert(ResultVector.size() == 2 &&
4725  "We should have exactly two unique results at this point");
4726  // If we are selecting between only two cases transform into a simple
4727  // select or a two-way select if default is possible.
4728  if (ResultVector[0].second.size() == 1 &&
4729  ResultVector[1].second.size() == 1) {
4730  ConstantInt *const FirstCase = ResultVector[0].second[0];
4731  ConstantInt *const SecondCase = ResultVector[1].second[0];
4732 
4733  bool DefaultCanTrigger = DefaultResult;
4734  Value *SelectValue = ResultVector[1].first;
4735  if (DefaultCanTrigger) {
4736  Value *const ValueCompare =
4737  Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp");
4738  SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first,
4739  DefaultResult, "switch.select");
4740  }
4741  Value *const ValueCompare =
4742  Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp");
4743  return Builder.CreateSelect(ValueCompare, ResultVector[0].first,
4744  SelectValue, "switch.select");
4745  }
4746 
4747  return nullptr;
4748 }
4749 
4750 // Helper function to cleanup a switch instruction that has been converted into
4751 // a select, fixing up PHI nodes and basic blocks.
4753  Value *SelectValue,
4754  IRBuilder<> &Builder) {
4755  BasicBlock *SelectBB = SI->getParent();
4756  while (PHI->getBasicBlockIndex(SelectBB) >= 0)
4757  PHI->removeIncomingValue(SelectBB);
4758  PHI->addIncoming(SelectValue, SelectBB);
4759 
4760  Builder.CreateBr(PHI->getParent());
4761 
4762  // Remove the switch.
4763  for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
4764  BasicBlock *Succ = SI->getSuccessor(i);
4765 
4766  if (Succ == PHI->getParent())
4767  continue;
4768  Succ->removePredecessor(SelectBB);
4769  }
4770  SI->eraseFromParent();
4771 }
4772 
4773 /// If the switch is only used to initialize one or more
4774 /// phi nodes in a common successor block with only two different
4775 /// constant values, replace the switch with select.
4776 static bool SwitchToSelect(SwitchInst *SI, IRBuilder<> &Builder,
4777  AssumptionCache *AC, const DataLayout &DL,
4778  const TargetTransformInfo &TTI) {
4779  Value *const Cond = SI->getCondition();
4780  PHINode *PHI = nullptr;
4781  BasicBlock *CommonDest = nullptr;
4782  Constant *DefaultResult;
4783  SwitchCaseResultVectorTy UniqueResults;
4784  // Collect all the cases that will deliver the same value from the switch.
4785  if (!InitializeUniqueCases(SI, PHI, CommonDest, UniqueResults, DefaultResult,
4786  DL, TTI))
4787  return false;
4788  // Selects choose between maximum two values.
4789  if (UniqueResults.size() != 2)
4790  return false;
4791  assert(PHI != nullptr && "PHI for value select not found");
4792 
4793  Builder.SetInsertPoint(SI);
4794  Value *SelectValue =
4795  ConvertTwoCaseSwitch(UniqueResults, DefaultResult, Cond, Builder);
4796  if (SelectValue) {
4797  RemoveSwitchAfterSelectConversion(SI, PHI, SelectValue, Builder);
4798  return true;
4799  }
4800  // The switch couldn't be converted into a select.
4801  return false;
4802 }
4803 
4804 namespace {
4805 
4806 /// This class represents a lookup table that can be used to replace a switch.
4807 class SwitchLookupTable {
4808 public:
4809  /// Create a lookup table to use as a switch replacement with the contents
4810  /// of Values, using DefaultValue to fill any holes in the table.
4811  SwitchLookupTable(
4812  Module &M, uint64_t TableSize, ConstantInt *Offset,
4813  const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
4814  Constant *DefaultValue, const DataLayout &DL);
4815 
4816  /// Build instructions with Builder to retrieve the value at
4817  /// the position given by Index in the lookup table.
4818  Value *BuildLookup(Value *Index, IRBuilder<> &Builder);
4819 
4820  /// Return true if a table with TableSize elements of
4821  /// type ElementType would fit in a target-legal register.
4822  static bool WouldFitInRegister(const DataLayout &DL, uint64_t TableSize,
4823  Type *ElementType);
4824 
4825 private:
4826  // Depending on the contents of the table, it can be represented in
4827  // different ways.
4828  enum {
4829  // For tables where each element contains the same value, we just have to
4830  // store that single value and return it for each lookup.
4831  SingleValueKind,
4832 
4833  // For tables where there is a linear relationship between table index
4834  // and values. We calculate the result with a simple multiplication
4835  // and addition instead of a table lookup.
4836  LinearMapKind,
4837 
4838  // For small tables with integer elements, we can pack them into a bitmap
4839  // that fits into a target-legal register. Values are retrieved by
4840  // shift and mask operations.
4841  BitMapKind,
4842 
4843  // The table is stored as an array of values. Values are retrieved by load
4844  // instructions from the table.
4845  ArrayKind
4846  } Kind;
4847 
4848  // For SingleValueKind, this is the single value.
4849  Constant *SingleValue;
4850 
4851  // For BitMapKind, this is the bitmap.
4852  ConstantInt *BitMap;
4853  IntegerType *BitMapElementTy;
4854 
4855  // For LinearMapKind, these are the constants used to derive the value.
4856  ConstantInt *LinearOffset;
4857  ConstantInt *LinearMultiplier;
4858 
4859  // For ArrayKind, this is the array.
4860  GlobalVariable *Array;
4861 };
4862 
4863 } // end anonymous namespace
4864 
4865 SwitchLookupTable::SwitchLookupTable(
4866  Module &M, uint64_t TableSize, ConstantInt *Offset,
4867  const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
4868  Constant *DefaultValue, const DataLayout &DL)
4869  : SingleValue(nullptr), BitMap(nullptr), BitMapElementTy(nullptr),
4870  LinearOffset(nullptr), LinearMultiplier(nullptr), Array(nullptr) {
4871  assert(Values.size() && "Can't build lookup table without values!");
4872  assert(TableSize >= Values.size() && "Can't fit values in table!");
4873 
4874  // If all values in the table are equal, this is that value.
4875  SingleValue = Values.begin()->second;
4876 
4877  Type *ValueType = Values.begin()->second->getType();
4878 
4879  // Build up the table contents.
4880  SmallVector<Constant *, 64> TableContents(TableSize);
4881  for (size_t I = 0, E = Values.size(); I != E; ++I) {
4882  ConstantInt *CaseVal = Values[I].first;
4883  Constant *CaseRes = Values[I].second;
4884  assert(CaseRes->getType() == ValueType);
4885 
4886  uint64_t Idx = (CaseVal->getValue() - Offset->getValue()).getLimitedValue();
4887  TableContents[Idx] = CaseRes;
4888 
4889  if (CaseRes != SingleValue)
4890  SingleValue = nullptr;
4891  }
4892 
4893  // Fill in any holes in the table with the default result.
4894  if (Values.size() < TableSize) {
4895  assert(DefaultValue &&
4896  "Need a default value to fill the lookup table holes.");
4897  assert(DefaultValue->getType() == ValueType);
4898  for (uint64_t I = 0; I < TableSize; ++I) {
4899  if (!TableContents[I])
4900  TableContents[I] = DefaultValue;
4901  }
4902 
4903  if (DefaultValue != SingleValue)
4904  SingleValue = nullptr;
4905  }
4906 
4907  // If each element in the table contains the same value, we only need to store
4908  // that single value.
4909  if (SingleValue) {
4910  Kind = SingleValueKind;
4911  return;
4912  }
4913 
4914  // Check if we can derive the value with a linear transformation from the
4915  // table index.
4916  if (isa<IntegerType>(ValueType)) {
4917  bool LinearMappingPossible = true;
4918  APInt PrevVal;
4919  APInt DistToPrev;
4920  assert(TableSize >= 2 && "Should be a SingleValue table.");
4921  // Check if there is the same distance between two consecutive values.
4922  for (uint64_t I = 0; I < TableSize; ++I) {
4923  ConstantInt *ConstVal = dyn_cast<ConstantInt>(TableContents[I]);
4924  if (!ConstVal) {
4925  // This is an undef. We could deal with it, but undefs in lookup tables
4926  // are very seldom. It's probably not worth the additional complexity.
4927  LinearMappingPossible = false;
4928  break;
4929  }
4930  APInt Val = ConstVal->getValue();
4931  if (I != 0) {
4932  APInt Dist = Val - PrevVal;
4933  if (I == 1) {
4934  DistToPrev = Dist;
4935  } else if (Dist != DistToPrev) {
4936  LinearMappingPossible = false;
4937  break;
4938  }
4939  }
4940  PrevVal = Val;
4941  }
4942  if (LinearMappingPossible) {
4943  LinearOffset = cast<ConstantInt>(TableContents[0]);
4944  LinearMultiplier = ConstantInt::get(M.getContext(), DistToPrev);
4945  Kind = LinearMapKind;
4946  ++NumLinearMaps;
4947  return;
4948  }
4949  }
4950 
4951  // If the type is integer and the table fits in a register, build a bitmap.
4952  if (WouldFitInRegister(DL, TableSize, ValueType)) {
4953  IntegerType *IT = cast<IntegerType>(ValueType);
4954  APInt TableInt(TableSize * IT->getBitWidth(), 0);
4955  for (uint64_t I = TableSize; I > 0; --I) {
4956  TableInt <<= IT->getBitWidth();
4957  // Insert values into the bitmap. Undef values are set to zero.
4958  if (!isa<UndefValue>(TableContents[I - 1])) {
4959  ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]);
4960  TableInt |= Val->getValue().zext(TableInt.getBitWidth());
4961  }
4962  }
4963  BitMap = ConstantInt::get(M.getContext(), TableInt);
4964  BitMapElementTy = IT;
4965  Kind = BitMapKind;
4966  ++NumBitMaps;
4967  return;
4968  }
4969 
4970  // Store the table in an array.
4971  ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize);
4972  Constant *Initializer = ConstantArray::get(ArrayTy, TableContents);
4973 
4974  Array = new GlobalVariable(M, ArrayTy, /*constant=*/true,
4975  GlobalVariable::PrivateLinkage, Initializer,
4976  "switch.table");
4977  Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
4978  Kind = ArrayKind;
4979 }
4980 
4981 Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) {
4982  switch (Kind) {
4983  case SingleValueKind:
4984  return SingleValue;
4985  case LinearMapKind: {
4986  // Derive the result value from the input value.
4987  Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(),
4988  false, "switch.idx.cast");
4989  if (!LinearMultiplier->isOne())
4990  Result = Builder.CreateMul(Result, LinearMultiplier, "switch.idx.mult");
4991  if (!LinearOffset->isZero())
4992  Result = Builder.CreateAdd(Result, LinearOffset, "switch.offset");
4993  return Result;
4994  }
4995  case BitMapKind: {
4996  // Type of the bitmap (e.g. i59).
4997  IntegerType *MapTy = BitMap->getType();
4998 
4999  // Cast Index to the same type as the bitmap.
5000  // Note: The Index is <= the number of elements in the table, so
5001  // truncating it to the width of the bitmask is safe.
5002  Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast");
5003 
5004  // Multiply the shift amount by the element width.
5005  ShiftAmt = Builder.CreateMul(
5006  ShiftAmt, ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()),
5007  "switch.shiftamt");
5008 
5009  // Shift down.
5010  Value *DownShifted =
5011  Builder.CreateLShr(BitMap, ShiftAmt, "switch.downshift");
5012  // Mask off.
5013  return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked");
5014  }
5015  case ArrayKind: {
5016  // Make sure the table index will not overflow when treated as signed.
5017  IntegerType *IT = cast<IntegerType>(Index->getType());
5018  uint64_t TableSize =
5019  Array->getInitializer()->getType()->getArrayNumElements();
5020  if (TableSize > (1ULL << (IT->getBitWidth() - 1)))
5021  Index = Builder.CreateZExt(
5022  Index, IntegerType::get(IT->getContext(), IT->getBitWidth() + 1),
5023  "switch.tableidx.zext");
5024 
5025  Value *GEPIndices[] = {Builder.getInt32(0), Index};
5026  Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array,
5027  GEPIndices, "switch.gep");
5028  return Builder.CreateLoad(GEP, "switch.load");
5029  }
5030  }
5031  llvm_unreachable("Unknown lookup table kind!");
5032 }
5033 
5034 bool SwitchLookupTable::WouldFitInRegister(const DataLayout &DL,
5035  uint64_t TableSize,
5036  Type *ElementType) {
5037  auto *IT = dyn_cast<IntegerType>(ElementType);
5038  if (!IT)
5039  return false;
5040  // FIXME: If the type is wider than it needs to be, e.g. i8 but all values
5041  // are <= 15, we could try to narrow the type.
5042 
5043  // Avoid overflow, fitsInLegalInteger uses unsigned int for the width.
5044  if (TableSize >= UINT_MAX / IT->getBitWidth())
5045  return false;
5046  return DL.fitsInLegalInteger(TableSize * IT->getBitWidth());
5047 }
5048 
5049 /// Determine whether a lookup table should be built for this switch, based on
5050 /// the number of cases, size of the table, and the types of the results.
5051 static bool
5052 ShouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize,
5053  const TargetTransformInfo &TTI, const DataLayout &DL,
5054  const SmallDenseMap<PHINode *, Type *> &ResultTypes) {
5055  if (SI->getNumCases() > TableSize || TableSize >= UINT64_MAX / 10)
5056  return false; // TableSize overflowed, or mul below might overflow.
5057 
5058  bool AllTablesFitInRegister = true;
5059  bool HasIllegalType = false;
5060  for (const auto &I : ResultTypes) {
5061  Type *Ty = I.second;
5062 
5063  // Saturate this flag to true.
5064  HasIllegalType = HasIllegalType || !TTI.isTypeLegal(Ty);
5065 
5066  // Saturate this flag to false.
5067  AllTablesFitInRegister =
5068  AllTablesFitInRegister &&
5069  SwitchLookupTable::WouldFitInRegister(DL, TableSize, Ty);
5070 
5071  // If both flags saturate, we're done. NOTE: This *only* works with
5072  // saturating flags, and all flags have to saturate first due to the
5073  // non-deterministic behavior of iterating over a dense map.
5074  if (HasIllegalType && !AllTablesFitInRegister)
5075  break;
5076  }
5077 
5078  // If each table would fit in a register, we should build it anyway.
5079  if (AllTablesFitInRegister)
5080  return true;
5081 
5082  // Don't build a table that doesn't fit in-register if it has illegal types.
5083  if (HasIllegalType)
5084  return false;
5085 
5086  // The table density should be at least 40%. This is the same criterion as for
5087  // jump tables, see SelectionDAGBuilder::handleJTSwitchCase.
5088  // FIXME: Find the best cut-off.
5089  return SI->getNumCases() * 10 >= TableSize * 4;
5090 }
5091 
5092 /// Try to reuse the switch table index compare. Following pattern:
5093 /// \code
5094 /// if (idx < tablesize)
5095 /// r = table[idx]; // table does not contain default_value
5096 /// else
5097 /// r = default_value;
5098 /// if (r != default_value)
5099 /// ...
5100 /// \endcode
5101 /// Is optimized to:
5102 /// \code
5103 /// cond = idx < tablesize;
5104 /// if (cond)
5105 /// r = table[idx];
5106 /// else
5107 /// r = default_value;
5108 /// if (cond)
5109 /// ...
5110 /// \endcode
5111 /// Jump threading will then eliminate the second if(cond).
5112 static void reuseTableCompare(
5113  User *PhiUser, BasicBlock *PhiBlock, BranchInst *RangeCheckBranch,
5114  Constant *DefaultValue,
5115  const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values) {
5116 
5117  ICmpInst *CmpInst = dyn_cast<ICmpInst>(PhiUser);
5118  if (!CmpInst)
5119  return;
5120 
5121  // We require that the compare is in the same block as the phi so that jump
5122  // threading can do its work afterwards.
5123  if (CmpInst->getParent() != PhiBlock)
5124  return;
5125 
5126  Constant *CmpOp1 = dyn_cast<Constant>(CmpInst->getOperand(1));
5127  if (!CmpOp1)
5128  return;
5129 
5130  Value *RangeCmp = RangeCheckBranch->getCondition();
5131  Constant *TrueConst = ConstantInt::getTrue(RangeCmp->getType());
5132  Constant *FalseConst = ConstantInt::getFalse(RangeCmp->getType());
5133 
5134  // Check if the compare with the default value is constant true or false.
5135  Constant *DefaultConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
5136  DefaultValue, CmpOp1, true);
5137  if (DefaultConst != TrueConst && DefaultConst != FalseConst)
5138  return;
5139 
5140  // Check if the compare with the case values is distinct from the default
5141  // compare result.
5142  for (auto ValuePair : Values) {
5143  Constant *CaseConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
5144  ValuePair.second, CmpOp1, true);
5145  if (!CaseConst || CaseConst == DefaultConst)
5146  return;
5147  assert((CaseConst == TrueConst || CaseConst == FalseConst) &&
5148  "Expect true or false as compare result.");
5149  }
5150 
5151  // Check if the branch instruction dominates the phi node. It's a simple
5152  // dominance check, but sufficient for our needs.
5153  // Although this check is invariant in the calling loops, it's better to do it
5154  // at this late stage. Practically we do it at most once for a switch.
5155  BasicBlock *BranchBlock = RangeCheckBranch->getParent();
5156  for (auto PI = pred_begin(PhiBlock), E = pred_end(PhiBlock); PI != E; ++PI) {
5157  BasicBlock *Pred = *PI;
5158  if (Pred != BranchBlock && Pred->getUniquePredecessor() != BranchBlock)
5159  return;
5160  }
5161 
5162  if (DefaultConst == FalseConst) {
5163  // The compare yields the same result. We can replace it.
5164  CmpInst->replaceAllUsesWith(RangeCmp);
5165  ++NumTableCmpReuses;
5166  } else {
5167  // The compare yields the same result, just inverted. We can replace it.
5168  Value *InvertedTableCmp = BinaryOperator::CreateXor(
5169  RangeCmp, ConstantInt::get(RangeCmp->getType(), 1), "inverted.cmp",
5170  RangeCheckBranch);
5171  CmpInst->replaceAllUsesWith(InvertedTableCmp);
5172  ++NumTableCmpReuses;
5173  }
5174 }
5175 
5176 /// If the switch is only used to initialize one or more phi nodes in a common
5177 /// successor block with different constant values, replace the switch with
5178 /// lookup tables.
5179 static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
5180  const DataLayout &DL,
5181  const TargetTransformInfo &TTI) {
5182  assert(SI->getNumCases() > 1 && "Degenerate switch?");
5183 
5184  // Only build lookup table when we have a target that supports it.
5185  if (!TTI.shouldBuildLookupTables())
5186  return false;
5187 
5188  // FIXME: If the switch is too sparse for a lookup table, perhaps we could
5189  // split off a dense part and build a lookup table for that.
5190 
5191  // FIXME: This creates arrays of GEPs to constant strings, which means each
5192  // GEP needs a runtime relocation in PIC code. We should just build one big
5193  // string and lookup indices into that.
5194 
5195  // Ignore switches with less than three cases. Lookup tables will not make
5196  // them
5197  // faster, so we don't analyze them.
5198  if (SI->getNumCases() < 3)
5199  return false;
5200 
5201  // Figure out the corresponding result for each case value and phi node in the
5202  // common destination, as well as the min and max case values.
5203  assert(SI->case_begin() != SI->case_end());
5204  SwitchInst::CaseIt CI = SI->case_begin();
5205  ConstantInt *MinCaseVal = CI.getCaseValue();
5206  ConstantInt *MaxCaseVal = CI.getCaseValue();
5207 
5208  BasicBlock *CommonDest = nullptr;
5209  typedef SmallVector<std::pair<ConstantInt *, Constant *>, 4> ResultListTy;
5211  SmallDenseMap<PHINode *, Constant *> DefaultResults;
5214 
5215  for (SwitchInst::CaseIt E = SI->case_end(); CI != E; ++CI) {
5216  ConstantInt *CaseVal = CI.getCaseValue();
5217  if (CaseVal->getValue().slt(MinCaseVal->getValue()))
5218  MinCaseVal = CaseVal;
5219  if (CaseVal->getValue().sgt(MaxCaseVal->getValue()))
5220  MaxCaseVal = CaseVal;
5221 
5222  // Resulting value at phi nodes for this case value.
5223  typedef SmallVector<std::pair<PHINode *, Constant *>, 4> ResultsTy;
5224  ResultsTy Results;
5225  if (!GetCaseResults(SI, CaseVal, CI.getCaseSuccessor(), &CommonDest,
5226  Results, DL, TTI))
5227  return false;
5228 
5229  // Append the result from this case to the list for each phi.
5230  for (const auto &I : Results) {
5231  PHINode *PHI = I.first;
5232  Constant *Value = I.second;
5233  if (!ResultLists.count(PHI))
5234  PHIs.push_back(PHI);
5235  ResultLists[PHI].push_back(std::make_pair(CaseVal, Value));
5236  }
5237  }
5238 
5239  // Keep track of the result types.
5240  for (PHINode *PHI : PHIs) {
5241  ResultTypes[PHI] = ResultLists[PHI][0].second->getType();
5242  }
5243 
5244  uint64_t NumResults = ResultLists[PHIs[0]].size();
5245  APInt RangeSpread = MaxCaseVal->getValue() - MinCaseVal->getValue();
5246  uint64_t TableSize = RangeSpread.getLimitedValue() + 1;
5247  bool TableHasHoles = (NumResults < TableSize);
5248 
5249  // If the table has holes, we need a constant result for the default case
5250  // or a bitmask that fits in a register.
5251  SmallVector<std::pair<PHINode *, Constant *>, 4> DefaultResultsList;
5252  bool HasDefaultResults =
5253  GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest,
5254  DefaultResultsList, DL, TTI);
5255 
5256  bool NeedMask = (TableHasHoles && !HasDefaultResults);
5257  if (NeedMask) {
5258  // As an extra penalty for the validity test we require more cases.
5259  if (SI->getNumCases() < 4) // FIXME: Find best threshold value (benchmark).
5260  return false;
5261  if (!DL.fitsInLegalInteger(TableSize))
5262  return false;
5263  }
5264 
5265  for (const auto &I : DefaultResultsList) {
5266  PHINode *PHI = I.first;
5267  Constant *Result = I.second;
5268  DefaultResults[PHI] = Result;
5269  }
5270 
5271  if (!ShouldBuildLookupTable(SI, TableSize, TTI, DL, ResultTypes))
5272  return false;
5273 
5274  // Create the BB that does the lookups.
5275  Module &Mod = *CommonDest->getParent()->getParent();
5276  BasicBlock *LookupBB = BasicBlock::Create(
5277  Mod.getContext(), "switch.lookup", CommonDest->getParent(), CommonDest);
5278 
5279  // Compute the table index value.
5280  Builder.SetInsertPoint(SI);
5281  Value *TableIndex =
5282  Builder.CreateSub(SI->getCondition(), MinCaseVal, "switch.tableidx");
5283 
5284  // Compute the maximum table size representable by the integer type we are
5285  // switching upon.
5286  unsigned CaseSize = MinCaseVal->getType()->getPrimitiveSizeInBits();
5287  uint64_t MaxTableSize = CaseSize > 63 ? UINT64_MAX : 1ULL << CaseSize;
5288  assert(MaxTableSize >= TableSize &&
5289  "It is impossible for a switch to have more entries than the max "
5290  "representable value of its input integer type's size.");
5291 
5292  // If the default destination is unreachable, or if the lookup table covers
5293  // all values of the conditional variable, branch directly to the lookup table
5294  // BB. Otherwise, check that the condition is within the case range.
5295  const bool DefaultIsReachable =
5296  !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
5297  const bool GeneratingCoveredLookupTable = (MaxTableSize == TableSize);
5298  BranchInst *RangeCheckBranch = nullptr;
5299 
5300  if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
5301  Builder.CreateBr(LookupBB);
5302  // Note: We call removeProdecessor later since we need to be able to get the
5303  // PHI value for the default case in case we're using a bit mask.
5304  } else {
5305  Value *Cmp = Builder.CreateICmpULT(
5306  TableIndex, ConstantInt::get(MinCaseVal->getType(), TableSize));
5307  RangeCheckBranch =
5308  Builder.CreateCondBr(Cmp, LookupBB, SI->getDefaultDest());
5309  }
5310 
5311  // Populate the BB that does the lookups.
5312  Builder.SetInsertPoint(LookupBB);
5313 
5314  if (NeedMask) {
5315  // Before doing the lookup we do the hole check.
5316  // The LookupBB is therefore re-purposed to do the hole check
5317  // and we create a new LookupBB.
5318  BasicBlock *MaskBB = LookupBB;
5319  MaskBB->setName("switch.hole_check");
5320  LookupBB = BasicBlock::Create(Mod.getContext(), "switch.lookup",
5321  CommonDest->getParent(), CommonDest);
5322 
5323  // Make the mask's bitwidth at least 8bit and a power-of-2 to avoid
5324  // unnecessary illegal types.
5325  uint64_t TableSizePowOf2 = NextPowerOf2(std::max(7ULL, TableSize - 1ULL));
5326  APInt MaskInt(TableSizePowOf2, 0);
5327  APInt One(TableSizePowOf2, 1);
5328  // Build bitmask; fill in a 1 bit for every case.
5329  const ResultListTy &ResultList = ResultLists[PHIs[0]];
5330  for (size_t I = 0, E = ResultList.size(); I != E; ++I) {
5331  uint64_t Idx = (ResultList[I].first->getValue() - MinCaseVal->getValue())
5332  .getLimitedValue();
5333  MaskInt |= One << Idx;
5334  }
5335  ConstantInt *TableMask = ConstantInt::get(Mod.getContext(), MaskInt);
5336 
5337  // Get the TableIndex'th bit of the bitmask.
5338  // If this bit is 0 (meaning hole) jump to the default destination,
5339  // else continue with table lookup.
5340  IntegerType *MapTy = TableMask->getType();
5341  Value *MaskIndex =
5342  Builder.CreateZExtOrTrunc(TableIndex, MapTy, "switch.maskindex");
5343  Value *Shifted = Builder.CreateLShr(TableMask, MaskIndex, "switch.shifted");
5344  Value *LoBit = Builder.CreateTrunc(
5345  Shifted, Type::getInt1Ty(Mod.getContext()), "switch.lobit");
5346  Builder.CreateCondBr(LoBit, LookupBB, SI->getDefaultDest());
5347 
5348  Builder.SetInsertPoint(LookupBB);
5349  AddPredecessorToBlock(SI->getDefaultDest(), MaskBB, SI->getParent());
5350  }
5351 
5352  if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
5353  // We cached PHINodes in PHIs, to avoid accessing deleted PHINodes later,
5354  // do not delete PHINodes here.
5356  /*DontDeleteUselessPHIs=*/true);
5357  }
5358 
5359  bool ReturnedEarly = false;
5360  for (size_t I = 0, E = PHIs.size(); I != E; ++I) {
5361  PHINode *PHI = PHIs[I];
5362  const ResultListTy &ResultList = ResultLists[PHI];
5363 
5364  // If using a bitmask, use any value to fill the lookup table holes.
5365  Constant *DV = NeedMask ? ResultLists[PHI][0].second : DefaultResults[PHI];
5366  SwitchLookupTable Table(Mod, TableSize, MinCaseVal, ResultList, DV, DL);
5367 
5368  Value *Result = Table.BuildLookup(TableIndex, Builder);
5369 
5370  // If the result is used to return immediately from the function, we want to
5371  // do that right here.
5372  if (PHI->hasOneUse() && isa<ReturnInst>(*PHI->user_begin()) &&
5373  PHI->user_back() == CommonDest->getFirstNonPHIOrDbg()) {
5374  Builder.CreateRet(Result);
5375  ReturnedEarly = true;
5376  break;
5377  }
5378 
5379  // Do a small peephole optimization: re-use the switch table compare if
5380  // possible.
5381  if (!TableHasHoles && HasDefaultResults && RangeCheckBranch) {
5382  BasicBlock *PhiBlock = PHI->getParent();
5383  // Search for compare instructions which use the phi.
5384  for (auto *User : PHI->users()) {
5385  reuseTableCompare(User, PhiBlock, RangeCheckBranch, DV, ResultList);
5386  }
5387  }
5388 
5389  PHI->addIncoming(Result, LookupBB);
5390  }
5391 
5392  if (!ReturnedEarly)
5393  Builder.CreateBr(CommonDest);
5394 
5395  // Remove the switch.
5396  for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
5397  BasicBlock *Succ = SI->getSuccessor(i);
5398 
5399  if (Succ == SI->getDefaultDest())
5400  continue;
5401  Succ->removePredecessor(SI->getParent());
5402  }
5403  SI->eraseFromParent();
5404 
5405  ++NumLookupTables;
5406  if (NeedMask)
5407  ++NumLookupTablesHoles;
5408  return true;
5409 }
5410 
5411 static bool isSwitchDense(ArrayRef<int64_t> Values) {
5412  // See also SelectionDAGBuilder::isDense(), which this function was based on.
5413  uint64_t Diff = (uint64_t)Values.back() - (uint64_t)Values.front();
5414  uint64_t Range = Diff + 1;
5415  uint64_t NumCases = Values.size();
5416  // 40% is the default density for building a jump table in optsize/minsize mode.
5417  uint64_t MinDensity = 40;
5418 
5419  return NumCases * 100 >= Range * MinDensity;
5420 }
5421 
5422 // Try and transform a switch that has "holes" in it to a contiguous sequence
5423 // of cases.
5424 //
5425 // A switch such as: switch(i) {case 5: case 9: case 13: case 17:} can be
5426 // range-reduced to: switch ((i-5) / 4) {case 0: case 1: case 2: case 3:}.
5427 //
5428 // This converts a sparse switch into a dense switch which allows better
5429 // lowering and could also allow transforming into a lookup table.
5430 static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder,
5431  const DataLayout &DL,
5432  const TargetTransformInfo &TTI) {
5433  auto *CondTy = cast<IntegerType>(SI->getCondition()->getType());
5434  if (CondTy->getIntegerBitWidth() > 64 ||
5435  !DL.fitsInLegalInteger(CondTy->getIntegerBitWidth()))
5436  return false;
5437  // Only bother with this optimization if there are more than 3 switch cases;
5438  // SDAG will only bother creating jump tables for 4 or more cases.
5439  if (SI->getNumCases() < 4)
5440  return false;
5441 
5442  // This transform is agnostic to the signedness of the input or case values. We
5443  // can treat the case values as signed or unsigned. We can optimize more common
5444  // cases such as a sequence crossing zero {-4,0,4,8} if we interpret case values
5445  // as signed.
5446  SmallVector<int64_t,4> Values;
5447  for (auto &C : SI->cases())
5448  Values.push_back(C.getCaseValue()->getValue().getSExtValue());
5449  std::sort(Values.begin(), Values.end());
5450 
5451  // If the switch is already dense, there's nothing useful to do here.
5452  if (isSwitchDense(Values))
5453  return false;
5454 
5455  // First, transform the values such that they start at zero and ascend.
5456  int64_t Base = Values[0];
5457  for (auto &V : Values)
5458  V -= Base;
5459 
5460  // Now we have signed numbers that have been shifted so that, given enough
5461  // precision, there are no negative values. Since the rest of the transform
5462  // is bitwise only, we switch now to an unsigned representation.
5463  uint64_t GCD = 0;
5464  for (auto &V : Values)
5465  GCD = GreatestCommonDivisor64(GCD, (uint64_t)V);
5466 
5467  // This transform can be done speculatively because it is so cheap - it results
5468  // in a single rotate operation being inserted. This can only happen if the
5469  // factor extracted is a power of 2.
5470  // FIXME: If the GCD is an odd number we can multiply by the multiplicative
5471  // inverse of GCD and then perform this transform.
5472  // FIXME: It's possible that optimizing a switch on powers of two might also
5473  // be beneficial - flag values are often powers of two and we could use a CLZ
5474  // as the key function.
5475  if (GCD <= 1 || !isPowerOf2_64(GCD))
5476  // No common divisor found or too expensive to compute key function.
5477  return false;
5478 
5479  unsigned Shift = Log2_64(GCD);
5480  for (auto &V : Values)
5481  V = (int64_t)((uint64_t)V >> Shift);
5482 
5483  if (!isSwitchDense(Values))
5484  // Transform didn't create a dense switch.
5485  return false;
5486 
5487  // The obvious transform is to shift the switch condition right and emit a
5488  // check that the condition actually cleanly divided by GCD, i.e.
5489  // C & (1 << Shift - 1) == 0
5490  // inserting a new CFG edge to handle the case where it didn't divide cleanly.
5491  //
5492  // A cheaper way of doing this is a simple ROTR(C, Shift). This performs the
5493  // shift and puts the shifted-off bits in the uppermost bits. If any of these
5494  // are nonzero then the switch condition will be very large and will hit the
5495  // default case.
5496 
5497  auto *Ty = cast<IntegerType>(SI->getCondition()->getType());
5498  Builder.SetInsertPoint(SI);
5499  auto *ShiftC = ConstantInt::get(Ty, Shift);
5500  auto *Sub = Builder.CreateSub(SI->getCondition(), ConstantInt::get(Ty, Base));
5501  auto *LShr = Builder.CreateLShr(Sub, ShiftC);
5502  auto *Shl = Builder.CreateShl(Sub, Ty->getBitWidth() - Shift);
5503  auto *Rot = Builder.CreateOr(LShr, Shl);
5504  SI->replaceUsesOfWith(SI->getCondition(), Rot);
5505 
5506  for (SwitchInst::CaseIt C = SI->case_begin(), E = SI->case_end(); C != E;
5507  ++C) {
5508  auto *Orig = C.getCaseValue();
5509  auto Sub = Orig->getValue() - APInt(Ty->getBitWidth(), Base);
5510  C.setValue(
5511  cast<ConstantInt>(ConstantInt::get(Ty, Sub.lshr(ShiftC->getValue()))));
5512  }
5513  return true;
5514 }
5515 
5516 bool SimplifyCFGOpt::SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
5517  BasicBlock *BB = SI->getParent();
5518 
5519  if (isValueEqualityComparison(SI)) {
5520  // If we only have one predecessor, and if it is a branch on this value,
5521  // see if that predecessor totally determines the outcome of this switch.
5522  if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
5523  if (SimplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder))
5524  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5525 
5526  Value *Cond = SI->getCondition();
5527  if (SelectInst *Select = dyn_cast<SelectInst>(Cond))
5528  if (SimplifySwitchOnSelect(SI, Select))
5529  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5530 
5531  // If the block only contains the switch, see if we can fold the block
5532  // away into any preds.
5533  BasicBlock::iterator BBI = BB->begin();
5534  // Ignore dbg intrinsics.
5535  while (isa<DbgInfoIntrinsic>(BBI))
5536  ++BBI;
5537  if (SI == &*BBI)
5538  if (FoldValueComparisonIntoPredecessors(SI, Builder))
5539  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5540  }
5541 
5542  // Try to transform the switch into an icmp and a branch.
5543  if (TurnSwitchRangeIntoICmp(SI, Builder))
5544  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5545 
5546  // Remove unreachable cases.
5547  if (EliminateDeadSwitchCases(SI, AC, DL))
5548  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5549 
5550  if (SwitchToSelect(SI, Builder, AC, DL, TTI))
5551  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5552 
5554  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5555 
5556  if (SwitchToLookupTable(SI, Builder, DL, TTI))
5557  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5558 
5559  if (ReduceSwitchRange(SI, Builder, DL, TTI))
5560  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5561 
5562  return false;
5563 }
5564 
5565 bool SimplifyCFGOpt::SimplifyIndirectBr(IndirectBrInst *IBI) {
5566  BasicBlock *BB = IBI->getParent();
5567  bool Changed = false;
5568 
5569  // Eliminate redundant destinations.
5571  for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; ++i) {
5572  BasicBlock *Dest = IBI->getDestination(i);
5573  if (!Dest->hasAddressTaken() || !Succs.insert(Dest).second) {
5574  Dest->removePredecessor(BB);
5575  IBI->removeDestination(i);
5576  --i;
5577  --e;
5578  Changed = true;
5579  }
5580  }
5581 
5582  if (IBI->getNumDestinations() == 0) {
5583  // If the indirectbr has no successors, change it to unreachable.
5584  new UnreachableInst(IBI->getContext(), IBI);
5586  return true;
5587  }
5588 
5589  if (IBI->getNumDestinations() == 1) {
5590  // If the indirectbr has one successor, change it to a direct branch.
5591  BranchInst::Create(IBI->getDestination(0), IBI);
5593  return true;
5594  }
5595 
5596  if (SelectInst *SI = dyn_cast<SelectInst>(IBI->getAddress())) {
5597  if (SimplifyIndirectBrOnSelect(IBI, SI))
5598  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5599  }
5600  return Changed;
5601 }
5602 
5603 /// Given an block with only a single landing pad and a unconditional branch
5604 /// try to find another basic block which this one can be merged with. This
5605 /// handles cases where we have multiple invokes with unique landing pads, but
5606 /// a shared handler.
5607 ///
5608 /// We specifically choose to not worry about merging non-empty blocks
5609 /// here. That is a PRE/scheduling problem and is best solved elsewhere. In
5610 /// practice, the optimizer produces empty landing pad blocks quite frequently
5611 /// when dealing with exception dense code. (see: instcombine, gvn, if-else
5612 /// sinking in this file)
5613 ///
5614 /// This is primarily a code size optimization. We need to avoid performing
5615 /// any transform which might inhibit optimization (such as our ability to
5616 /// specialize a particular handler via tail commoning). We do this by not
5617 /// merging any blocks which require us to introduce a phi. Since the same
5618 /// values are flowing through both blocks, we don't loose any ability to
5619 /// specialize. If anything, we make such specialization more likely.
5620 ///
5621 /// TODO - This transformation could remove entries from a phi in the target
5622 /// block when the inputs in the phi are the same for the two blocks being
5623 /// merged. In some cases, this could result in removal of the PHI entirely.
5625  BasicBlock *BB) {
5626  auto Succ = BB->getUniqueSuccessor();
5627  assert(Succ);
5628  // If there's a phi in the successor block, we'd likely have to introduce
5629  // a phi into the merged landing pad block.
5630  if (isa<PHINode>(*Succ->begin()))
5631  return false;
5632 
5633  for (BasicBlock *OtherPred : predecessors(Succ)) {
5634  if (BB == OtherPred)
5635  continue;
5636  BasicBlock::iterator I = OtherPred->begin();
5638  if (!LPad2 || !LPad2->isIdenticalTo(LPad))
5639  continue;
5640  for (++I; isa<DbgInfoIntrinsic>(I); ++I) {
5641  }
5642  BranchInst *BI2 = dyn_cast<BranchInst>(I);
5643  if (!BI2 || !BI2->isIdenticalTo(BI))
5644  continue;
5645 
5646  // We've found an identical block. Update our predecessors to take that
5647  // path instead and make ourselves dead.
5649  Preds.insert(pred_begin(BB), pred_end(BB));
5650  for (BasicBlock *Pred : Preds) {
5651  InvokeInst *II = cast<InvokeInst>(Pred->getTerminator());
5652  assert(II->getNormalDest() != BB && II->getUnwindDest() == BB &&
5653  "unexpected successor");
5654  II->setUnwindDest(OtherPred);
5655  }
5656 
5657  // The debug info in OtherPred doesn't cover the merged control flow that
5658  // used to go through BB. We need to delete it or update it.
5659  for (auto I = OtherPred->begin(), E = OtherPred->end(); I != E;) {
5660  Instruction &Inst = *I;
5661  I++;
5662  if (isa<DbgInfoIntrinsic>(Inst))
5663  Inst.eraseFromParent();
5664  }
5665 
5667  Succs.insert(succ_begin(BB), succ_end(BB));
5668  for (BasicBlock *Succ : Succs) {
5669  Succ->removePredecessor(BB);
5670  }
5671 
5672  IRBuilder<> Builder(BI);
5673  Builder.CreateUnreachable();
5674  BI->eraseFromParent();
5675  return true;
5676  }
5677  return false;
5678 }
5679 
5680 bool SimplifyCFGOpt::SimplifyUncondBranch(BranchInst *BI,
5681  IRBuilder<> &Builder) {
5682  BasicBlock *BB = BI->getParent();
5683 
5684  if (SinkCommon && SinkThenElseCodeToEnd(BI))
5685  return true;
5686 
5687  // If the Terminator is the only non-phi instruction, simplify the block.
5688  // if LoopHeader is provided, check if the block is a loop header
5689  // (This is for early invocations before loop simplify and vectorization
5690  // to keep canonical loop forms for nested loops.
5691  // These blocks can be eliminated when the pass is invoked later
5692  // in the back-end.)
5694  if (I->isTerminator() && BB != &BB->getParent()->getEntryBlock() &&
5695  (!LoopHeaders || !LoopHeaders->count(BB)) &&
5697  return true;
5698 
5699  // If the only instruction in the block is a seteq/setne comparison
5700  // against a constant, try to simplify the block.
5701  if (ICmpInst *ICI = dyn_cast<ICmpInst>(I))
5702  if (ICI->isEquality() && isa<ConstantInt>(ICI->getOperand(1))) {
5703  for (++I; isa<DbgInfoIntrinsic>(I); ++I)
5704  ;
5705  if (I->isTerminator() &&
5706  TryToSimplifyUncondBranchWithICmpInIt(ICI, Builder, DL, TTI,
5707  BonusInstThreshold, AC))
5708  return true;
5709  }
5710 
5711  // See if we can merge an empty landing pad block with another which is
5712  // equivalent.
5713  if (LandingPadInst *LPad = dyn_cast<LandingPadInst>(I)) {
5714  for (++I; isa<DbgInfoIntrinsic>(I); ++I) {
5715  }
5716  if (I->isTerminator() && TryToMergeLandingPad(LPad, BI, BB))
5717  return true;
5718  }
5719 
5720  // If this basic block is ONLY a compare and a branch, and if a predecessor
5721  // branches to us and our successor, fold the comparison into the
5722  // predecessor and use logical operations to update the incoming value
5723  // for PHI nodes in common successor.
5724  if (FoldBranchToCommonDest(BI, BonusInstThreshold))
5725  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5726  return false;
5727 }
5728 
5730  BasicBlock *PredPred = nullptr;
5731  for (auto *P : predecessors(BB)) {
5732  BasicBlock *PPred = P->getSinglePredecessor();
5733  if (!PPred || (PredPred && PredPred != PPred))
5734  return nullptr;
5735  PredPred = PPred;
5736  }
5737  return PredPred;
5738 }
5739 
5740 bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
5741  BasicBlock *BB = BI->getParent();
5742 
5743  // Conditional branch
5744  if (isValueEqualityComparison(BI)) {
5745  // If we only have one predecessor, and if it is a branch on this value,
5746  // see if that predecessor totally determines the outcome of this
5747  // switch.
5748  if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
5749  if (SimplifyEqualityComparisonWithOnlyPredecessor(BI, OnlyPred, Builder))
5750  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5751 
5752  // This block must be empty, except for the setcond inst, if it exists.
5753  // Ignore dbg intrinsics.
5754  BasicBlock::iterator I = BB->begin();
5755  // Ignore dbg intrinsics.
5756  while (isa<DbgInfoIntrinsic>(I))
5757  ++I;
5758  if (&*I == BI) {
5759  if (FoldValueComparisonIntoPredecessors(BI, Builder))
5760  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5761  } else if (&*I == cast<Instruction>(BI->getCondition())) {
5762  ++I;
5763  // Ignore dbg intrinsics.
5764  while (isa<DbgInfoIntrinsic>(I))
5765  ++I;
5766  if (&*I == BI && FoldValueComparisonIntoPredecessors(BI, Builder))
5767  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5768  }
5769  }
5770 
5771  // Try to turn "br (X == 0 | X == 1), T, F" into a switch instruction.
5772  if (SimplifyBranchOnICmpChain(BI, Builder, DL))
5773  return true;
5774 
5775  // If this basic block has a single dominating predecessor block and the
5776  // dominating block's condition implies BI's condition, we know the direction
5777  // of the BI branch.
5778  if (BasicBlock *Dom = BB->getSinglePredecessor()) {
5779  auto *PBI = dyn_cast_or_null<BranchInst>(Dom->getTerminator());
5780  if (PBI && PBI->isConditional() &&
5781  PBI->getSuccessor(0) != PBI->getSuccessor(1) &&
5782  (PBI->getSuccessor(0) == BB || PBI->getSuccessor(1) == BB)) {
5783  bool CondIsFalse = PBI->getSuccessor(1) == BB;
5784  Optional<bool> Implication = isImpliedCondition(
5785  PBI->getCondition(), BI->getCondition(), DL, CondIsFalse);
5786  if (Implication) {
5787  // Turn this into a branch on constant.
5788  auto *OldCond = BI->getCondition();
5789  ConstantInt *CI = *Implication
5792  BI->setCondition(CI);
5794  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5795  }
5796  }
5797  }
5798 
5799  // If this basic block is ONLY a compare and a branch, and if a predecessor
5800  // branches to us and one of our successors, fold the comparison into the
5801  // predecessor and use logical operations to pick the right destination.
5802  if (FoldBranchToCommonDest(BI, BonusInstThreshold))
5803  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5804 
5805  // We have a conditional branch to two blocks that are only reachable
5806  // from BI. We know that the condbr dominates the two blocks, so see if
5807  // there is any identical code in the "then" and "else" blocks. If so, we
5808  // can hoist it up to the branching block.
5809  if (BI->getSuccessor(0)->getSinglePredecessor()) {
5810  if (BI->getSuccessor(1)->getSinglePredecessor()) {
5811  if (HoistThenElseCodeToIf(BI, TTI))
5812  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5813  } else {
5814  // If Successor #1 has multiple preds, we may be able to conditionally
5815  // execute Successor #0 if it branches to Successor #1.
5816  TerminatorInst *Succ0TI = BI->getSuccessor(0)->getTerminator();
5817  if (Succ0TI->getNumSuccessors() == 1 &&
5818  Succ0TI->getSuccessor(0) == BI->getSuccessor(1))
5819  if (SpeculativelyExecuteBB(BI, BI->getSuccessor(0), TTI))
5820  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5821  }
5822  } else if (BI->getSuccessor(1)->getSinglePredecessor()) {
5823  // If Successor #0 has multiple preds, we may be able to conditionally
5824  // execute Successor #1 if it branches to Successor #0.
5825  TerminatorInst *Succ1TI = BI->getSuccessor(1)->getTerminator();
5826  if (Succ1TI->getNumSuccessors() == 1 &&
5827  Succ1TI->getSuccessor(0) == BI->getSuccessor(0))
5828  if (SpeculativelyExecuteBB(BI, BI->getSuccessor(1), TTI))
5829  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5830  }
5831 
5832  // If this is a branch on a phi node in the current block, thread control
5833  // through this block if any PHI node entries are constants.
5834  if (PHINode *PN = dyn_cast<PHINode>(BI->getCondition()))
5835  if (PN->getParent() == BI->getParent())
5836  if (FoldCondBranchOnPHI(BI, DL))
5837  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5838 
5839  // Scan predecessor blocks for conditional branches.
5840  for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
5841  if (BranchInst *PBI = dyn_cast<BranchInst>((*PI)->getTerminator()))
5842  if (PBI != BI && PBI->isConditional())
5843  if (SimplifyCondBranchToCondBranch(PBI, BI, DL))
5844  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5845 
5846  // Look for diamond patterns.
5847  if (MergeCondStores)
5849  if (BranchInst *PBI = dyn_cast<BranchInst>(PrevBB->getTerminator()))
5850  if (PBI != BI && PBI->isConditional())
5851  if (mergeConditionalStores(PBI, BI))
5852  return SimplifyCFG(BB, TTI, BonusInstThreshold, AC) | true;
5853 
5854  return false;
5855 }
5856 
5857 /// Check if passing a value to an instruction will cause undefined behavior.
5858 static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I) {
5859  Constant *C = dyn_cast<Constant>(V);
5860  if (!C)
5861  return false;
5862 
5863  if (I->use_empty())
5864  return false;
5865 
5866  if (C->isNullValue() || isa<UndefValue>(C)) {
5867  // Only look at the first use, avoid hurting compile time with long uselists
5868  User *Use = *I->user_begin();
5869 
5870  // Now make sure that there are no instructions in between that can alter
5871  // control flow (eg. calls)
5873  i = ++BasicBlock::iterator(I),
5874  UI = BasicBlock::iterator(dyn_cast<Instruction>(Use));
5875  i != UI; ++i)
5876  if (i == I->getParent()->end() || i->mayHaveSideEffects())
5877  return false;
5878 
5879  // Look through GEPs. A load from a GEP derived from NULL is still undefined
5880  if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Use))
5881  if (GEP->getPointerOperand() == I)
5883 
5884  // Look through bitcasts.
5885  if (BitCastInst *BC = dyn_cast<BitCastInst>(Use))
5886  return passingValueIsAlwaysUndefined(V, BC);
5887 
5888  // Load from null is undefined.
5889  if (LoadInst *LI = dyn_cast<LoadInst>(Use))
5890  if (!LI->isVolatile())
5891  return LI->getPointerAddressSpace() == 0;
5892 
5893  // Store to null is undefined.
5894  if (StoreInst *SI = dyn_cast<StoreInst>(Use))
5895  if (!SI->isVolatile())
5896  return SI->getPointerAddressSpace() == 0 &&
5897  SI->getPointerOperand() == I;
5898 
5899  // A call to null is undefined.
5900  if (auto CS = CallSite(Use))
5901  return CS.getCalledValue() == I;
5902  }
5903  return false;
5904 }
5905 
5906 /// If BB has an incoming value that will always trigger undefined behavior
5907 /// (eg. null pointer dereference), remove the branch leading here.
5909  for (BasicBlock::iterator i = BB->begin();
5910  PHINode *PHI = dyn_cast<PHINode>(i); ++i)
5911  for (unsigned i = 0, e = PHI->getNumIncomingValues(); i != e; ++i)
5912  if (passingValueIsAlwaysUndefined(PHI->getIncomingValue(i), PHI)) {
5914  IRBuilder<> Builder(T);
5915  if (BranchInst *BI = dyn_cast<BranchInst>(T)) {
5916  BB->removePredecessor(PHI->getIncomingBlock(i));
5917  // Turn uncoditional branches into unreachables and remove the dead
5918  // destination from conditional branches.
5919  if (BI->isUnconditional())
5920  Builder.CreateUnreachable();
5921  else
5922  Builder.CreateBr(BI->getSuccessor(0) == BB ? BI->getSuccessor(1)
5923  : BI->getSuccessor(0));
5924  BI->eraseFromParent();
5925  return true;
5926  }
5927  // TODO: SwitchInst.
5928  }
5929 
5930  return false;
5931 }
5932 
5933 bool SimplifyCFGOpt::run(BasicBlock *BB) {
5934  bool Changed = false;
5935 
5936  assert(BB && BB->getParent() && "Block not embedded in function!");
5937  assert(BB->getTerminator() && "Degenerate basic block encountered!");
5938 
5939  // Remove basic blocks that have no predecessors (except the entry block)...
5940  // or that just have themself as a predecessor. These are unreachable.
5941  if ((pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) ||
5942  BB->getSinglePredecessor() == BB) {
5943  DEBUG(dbgs() << "Removing BB: \n" << *BB);
5944  DeleteDeadBlock(BB);
5945  return true;
5946  }
5947 
5948  // Check to see if we can constant propagate this terminator instruction
5949  // away...
5950  Changed |= ConstantFoldTerminator(BB, true);
5951 
5952  // Check for and eliminate duplicate PHI nodes in this block.
5953  Changed |= EliminateDuplicatePHINodes(BB);
5954 
5955  // Check for and remove branches that will always cause undefined behavior.
5956  Changed |= removeUndefIntroducingPredecessor(BB);
5957 
5958  // Merge basic blocks into their predecessor if there is only one distinct
5959  // pred, and if there is only one distinct successor of the predecessor, and
5960  // if there are no PHI nodes.
5961  //
5962  if (MergeBlockIntoPredecessor(BB))
5963  return true;
5964 
5965  IRBuilder<> Builder(BB);
5966 
5967  // If there is a trivial two-entry PHI node in this basic block, and we can
5968  // eliminate it, do so now.
5969  if (PHINode *PN = dyn_cast<PHINode>(BB->begin()))
5970  if (PN->getNumIncomingValues() == 2)
5971  Changed |= FoldTwoEntryPHINode(PN, TTI, DL);
5972 
5973  Builder.SetInsertPoint(BB->getTerminator());
5974  if (BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator())) {
5975  if (BI->isUnconditional()) {
5976  if (SimplifyUncondBranch(BI, Builder))
5977  return true;
5978  } else {
5979  if (SimplifyCondBranch(BI, Builder))
5980  return true;
5981  }
5982  } else if (ReturnInst *RI = dyn_cast<ReturnInst>(BB->getTerminator())) {
5983  if (SimplifyReturn(RI, Builder))
5984  return true;
5985  } else if (ResumeInst *RI = dyn_cast<ResumeInst>(BB->getTerminator())) {
5986  if (SimplifyResume(RI, Builder))
5987  return true;
5988  } else if (CleanupReturnInst *RI =
5989  dyn_cast<CleanupReturnInst>(BB->getTerminator())) {
5990  if (SimplifyCleanupReturn(RI))
5991  return true;
5992  } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) {
5993  if (SimplifySwitch(SI, Builder))
5994  return true;
5995  } else if (UnreachableInst *UI =
5996  dyn_cast<UnreachableInst>(BB->getTerminator())) {
5997  if (SimplifyUnreachable(UI))
5998  return true;
5999  } else if (IndirectBrInst *IBI =
6000  dyn_cast<IndirectBrInst>(BB->getTerminator())) {
6001  if (SimplifyIndirectBr(IBI))
6002  return true;
6003  }
6004 
6005  return Changed;
6006 }
6007 
6008 /// This function is used to do simplification of a CFG.
6009 /// For example, it adjusts branches to branches to eliminate the extra hop,
6010 /// eliminates unreachable basic blocks, and does other "peephole" optimization
6011 /// of the CFG. It returns true if a modification was made.
6012 ///
6014  unsigned BonusInstThreshold, AssumptionCache *AC,
6015  SmallPtrSetImpl<BasicBlock *> *LoopHeaders) {
6016  return SimplifyCFGOpt(TTI, BB->getModule()->getDataLayout(),
6017  BonusInstThreshold, AC, LoopHeaders)
6018  .run(BB);
6019 }
bool shouldBuildLookupTablesForConstant(Constant *C) const
Return true if switches should be turned into lookup tables containing this constant value for the ta...
const Use & getOperandUse(unsigned i) const
Definition: User.h:158
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
Definition: PatternMatch.h:506
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::ZeroOrMore, cl::values(clEnumValN(DefaultIT,"arm-default-it","Generate IT block based on arch"), clEnumValN(RestrictedIT,"arm-restrict-it","Disallow deprecated IT based on ARMv8"), clEnumValN(NoRestrictedIT,"arm-no-restrict-it","Allow IT blocks based on ARMv7")))
Return a value (possibly void), from a function.
Value * getValueOperand()
Definition: Instructions.h:391
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:76
void computeKnownBits(const Value *V, APInt &KnownZero, APInt &KnownOne, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:102
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:513
BasicBlock * getUniqueSuccessor()
Return the successor of this block if it has a unique successor.
Definition: BasicBlock.cpp:261
IntegerType * getType() const
getType - Specialize the getType() method to always return an IntegerType, which reduces the amount o...
Definition: Constants.h:177
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Definition: IRBuilder.h:699
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:64
CaseIt case_end()
Returns a read/write iterator that points one past the last in the SwitchInst.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:870
iterator_range< use_iterator > uses()
Definition: Value.h:326
BasicBlock * getUniquePredecessor()
Return the predecessor of this block if it has a unique predecessor block.
Definition: BasicBlock.cpp:239
void removePredecessor(BasicBlock *Pred, bool DontDeleteUselessPHIs=false)
Notify the BasicBlock that the predecessor Pred is no longer able to reach it.
Definition: BasicBlock.cpp:281
static IntegerType * getInt1Ty(LLVMContext &C)
Definition: Type.cpp:166
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1032
iterator_range< CaseIt > cases()
Iteration adapter for range-for loops.
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1309
DiagnosticInfoOptimizationBase::Argument NV
uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B)
GreatestCommonDivisor64 - Return the greatest common divisor of the two values using Euclid's algorit...
Definition: MathExtras.h:538
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Definition: IRBuilder.h:1309
bool SimplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI, unsigned BonusInstThreshold, AssumptionCache *AC=nullptr, SmallPtrSetImpl< BasicBlock * > *LoopHeaders=nullptr)
This function is used to do simplification of a CFG.
STATISTIC(NumFunctions,"Total number of functions")
iterator erase(iterator where)
Definition: ilist.h:280
size_t i
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the specified block at the specified instruction - everything before SplitPt stays in Old and e...
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:162
void swapSuccessors()
Swap the successors of this branch instruction.
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1478
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:52
void dropUnknownNonDebugMetadata(ArrayRef< unsigned > KnownIDs)
Drop all unknown metadata except for debug locations.
Definition: Metadata.cpp:1156
static cl::opt< bool > MergeCondStoresAggressively("simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false), cl::desc("When merging conditional stores, do so even if the resultant ""basic blocks are unlikely to be if-converted as a result"))
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:51
unsigned getSuccessorIndex() const
Returns TerminatorInst's successor index for current case successor.
const T & front() const
front - Get the first element.
Definition: ArrayRef.h:144
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
static bool removeUndefIntroducingPredecessor(BasicBlock *BB)
If BB has an incoming value that will always trigger undefined behavior (eg.
unsigned getNumOperands() const
Definition: User.h:167
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1040
void DeleteDeadBlock(BasicBlock *BB)
Delete the specified block, which must have no predecessors.
void addCase(ConstantInt *OnVal, BasicBlock *Dest)
Add an entry to the switch instruction.
Value * getValue() const
Convenience accessor.
bool isSimple() const
Definition: Instructions.h:384
This class represents a function call, abstracting a target machine's calling convention.
This file contains the declarations for metadata subclasses.
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:380
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE, etc.
Definition: InstrTypes.h:984
static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2, Instruction *I1, Instruction *I2)
static cl::opt< unsigned > PHINodeFoldingThreshold("phi-node-folding-threshold", cl::Hidden, cl::init(2), cl::desc("Control the amount of phi node folding to perform (default = 2)"))
static bool mergeCleanupPad(CleanupReturnInst *RI)
CaseIt case_begin()
Returns a read/write iterator that points to the first case in the SwitchInst.
bool mayHaveSideEffects() const
Return true if the instruction may have side effects.
Definition: Instruction.h:450
A cache of .assume calls within a function.
Function Alias Analysis Results
static void GetBranchWeights(TerminatorInst *TI, SmallVectorImpl< uint64_t > &Weights)
Get Weights of a given TerminatorInst, the default weight is at the front of the vector.
static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI, const DataLayout &DL)
If we have a conditional branch as a predecessor of another block, this function tries to simplify it...
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
If this value is smaller than the specified limit, return it, otherwise return the limit value...
Definition: APInt.h:409
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:192
static bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:736
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:100
static bool GetCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest, BasicBlock **CommonDest, SmallVectorImpl< std::pair< PHINode *, Constant * >> &Res, const DataLayout &DL, const TargetTransformInfo &TTI)
Try to determine the resulting constant values in phi nodes at the common destination basic block...
static unsigned ComputeSpeculationCost(const Instruction *I, const TargetTransformInfo &TTI)
const Instruction & front() const
Definition: BasicBlock.h:240
Metadata node.
Definition: Metadata.h:830
static StoreInst * findUniqueStoreInBlocks(BasicBlock *BB1, BasicBlock *BB2)
An instruction for reading from memory.
Definition: Instructions.h:164
CleanupPadInst * getCleanupPad() const
Convenience accessor.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
Definition: DerivedTypes.h:65
Hexagon Common GEP
static Constant * getTrue(Type *Ty)
For a boolean type, or a vector of boolean type, return true, or a vector with every element true...
static ConstantRange makeAllowedICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other)
Produce the smallest range such that all values that may satisfy the given predicate with any value c...
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1...
Definition: MathExtras.h:180
static bool ValidLookupTableConstant(Constant *C, const TargetTransformInfo &TTI)
Return true if the backend will be able to handle initializing an array of constants like C...
op_iterator op_begin()
Definition: User.h:205
ConstantInt * findCaseDest(BasicBlock *BB)
Finds the unique case value for a given successor.
static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI, uint64_t &PredTrueWeight, uint64_t &PredFalseWeight, uint64_t &SuccTrueWeight, uint64_t &SuccFalseWeight)
Return true if either PBI or BI has branch weight available, and store the weights in {Pred|Succ}{Tru...
uint64_t getArrayNumElements() const
Definition: DerivedTypes.h:364
static ConstantInt * GetConstantInt(Value *V, const DataLayout &DL)
Extract ConstantInt from value, looking through IntToPtr and PointerNullValue.
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I)
Check if passing a value to an instruction will cause undefined behavior.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:345
Value * CreateNot(Value *V, const Twine &Name="")
Definition: IRBuilder.h:1066
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:228
bool isBundleOperand(Value::const_user_iterator UI) const
Determine whether the passed iterator points to a bundle operand.
Definition: CallSite.h:154
Instruction * getFirstNonPHIOrDbg()
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic...
Definition: BasicBlock.cpp:187
static bool isProfitableToFoldUnconditional(BranchInst *SI1, BranchInst *SI2, Instruction *Cond, SmallVectorImpl< PHINode * > &PhiNodes)
Return true if it is safe and profitable to merge these two terminator instructions together...
The address of a basic block.
Definition: Constants.h:822
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:41
LLVM_NODISCARD bool empty() const
Definition: SmallSet.h:55
bool isUnconditional() const
static Constant * getIntegerCast(Constant *C, Type *Ty, bool isSigned)
Create a ZExt, Bitcast or Trunc for integer -> integer casts.
Definition: Constants.cpp:1535
static bool SwitchToSelect(SwitchInst *SI, IRBuilder<> &Builder, AssumptionCache *AC, const DataLayout &DL, const TargetTransformInfo &TTI)
If the switch is only used to initialize one or more phi nodes in a common successor block with only ...
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
static bool SinkThenElseCodeToEnd(BranchInst *BI1)
Given an unconditional branch that goes to BBEnd, check whether BBEnd has only two predecessors and t...
This class represents the LLVM 'select' instruction.
static bool mergeConditionalStoreToAddress(BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB, BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond)
bool isIdenticalTo(const Instruction *I) const
Return true if the specified instruction is exactly identical to the current one. ...
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
#define Fail
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:143
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr it the function does no...
Definition: BasicBlock.cpp:116
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions=false, const TargetLibraryInfo *TLI=nullptr)
If a terminator instruction is predicated on a constant value, convert it into an unconditional branc...
Definition: Local.cpp:68
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
static bool SimplifyTerminatorOnSelect(TerminatorInst *OldTerm, Value *Cond, BasicBlock *TrueBB, BasicBlock *FalseBB, uint32_t TrueWeight, uint32_t FalseWeight)
static int ConstantIntSortPredicate(ConstantInt *const *P1, ConstantInt *const *P2)
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:180
ReturnInst * CreateRet(Value *V)
Create a 'ret <val>' instruction.
Definition: IRBuilder.h:674
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:588
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
Definition: BasicBlock.h:308
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1218
static bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI)
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:813
static const DILocation * getMergedLocation(const DILocation *LocA, const DILocation *LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:257
static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB)
Return true if we can thread a branch across this block.
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following: ...
static Value * isSafeToSpeculateStore(Instruction *I, BasicBlock *BrBB, BasicBlock *StoreBB, BasicBlock *EndBB)
Determine if we can hoist sink a sole store instruction out of a conditional block.
BasicBlock * getDestination(unsigned i)
Return the specified destination.
This file implements a class to represent arbitrary precision integral constant values and operations...
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
Definition: PatternMatch.h:434
This class represents a cast from a pointer to an integer.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:106
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:154
void assign(size_type NumElts, const T &Elt)
Definition: SmallVector.h:418
bool isDLLImportDependent() const
Return true if the value is dependent on a dllimport variable.
Definition: Constants.cpp:396
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:241
void setUnwindDest(BasicBlock *B)
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:873
static bool isSwitchDense(ArrayRef< int64_t > Values)
static bool SimplifyCondBranchToTwoReturns(BranchInst *BI, IRBuilder<> &Builder)
If we found a conditional branch that goes to two returning blocks, try to merge them together into o...
iterator begin() const
Definition: ArrayRef.h:324
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
bool isTypeLegal(Type *Ty) const
Return true if this type is legal.
#define T
static void MapCaseToResult(ConstantInt *CaseVal, SwitchCaseResultVectorTy &UniqueResults, Constant *Result)
Class to represent array types.
Definition: DerivedTypes.h:345
void andIRFlags(const Value *V)
Logical 'and' of any supported wrapping, exact, and fast-math flags of V and this instruction...
bool sgt(const APInt &RHS) const
Signed greather than comparison.
Definition: APInt.h:1101
BasicBlock * getSuccessor(unsigned i) const
This class represents a no-op cast from one type to another.
static cl::opt< bool > SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true), cl::desc("Sink common instructions down to the end block"))
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
static Value * ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector, Constant *DefaultResult, Value *Condition, IRBuilder<> &Builder)
void setSuccessor(unsigned idx, BasicBlock *B)
Update the specified successor to point at the provided block.
Definition: InstrTypes.h:84
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:835
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
static void EliminateBlockCases(BasicBlock *BB, std::vector< ValueEqualityComparisonCase > &Cases)
Given a vector of bb/value pairs, remove any entries in the list that match the specified block...
An instruction for storing to memory.
Definition: Instructions.h:300
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Definition: IRBuilder.h:151
uint64_t * Cases
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:401
static cl::opt< bool > MergeCondStores("simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true), cl::desc("Hoist conditional stores even if an unconditional store does not ""precede - hoist multiple conditional stores into a single ""predicated store"))
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1301
LLVM_NODISCARD char front() const
front - Get the first character in the string.
Definition: StringRef.h:139
bool EliminateDuplicatePHINodes(BasicBlock *BB)
Check for and eliminate duplicate PHI nodes in this block.
Definition: Local.cpp:916
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
Debug location.
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:263
Value * CreateInBoundsGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Definition: IRBuilder.h:1158
unsigned getMinSignedBits() const
Get the minimum bit size for this signed APInt.
Definition: APInt.h:1298
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:141
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Definition: IRBuilder.h:127
BasicBlock * getNormalDest() const
If this flag is set, the remapper knows that only local values within a function (such as an instruct...
Definition: ValueMapper.h:62
bool mayReadOrWriteMemory() const
Return true if this instruction may read or write memory.
Definition: Instruction.h:416
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.cpp:501
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
unsigned getNumIncomingValues() const
Return the number of incoming edges.
Interval::succ_iterator succ_end(Interval *I)
Definition: Interval.h:109
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition: User.cpp:24
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1003
static cl::opt< bool > HoistCondStores("simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true), cl::desc("Hoist conditional stores if an unconditional store precedes"))
static void reuseTableCompare(User *PhiUser, BasicBlock *PhiBlock, BranchInst *RangeCheckBranch, Constant *DefaultValue, const SmallVectorImpl< std::pair< ConstantInt *, Constant * >> &Values)
Try to reuse the switch table index compare.
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
Definition: InstrTypes.h:74
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
Definition: Instructions.h:830
LoadInst * CreateLoad(Value *Ptr, const char *Name)
Definition: IRBuilder.h:1082
succ_range successors()
Definition: InstrTypes.h:280
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
bool FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold=1)
If this basic block is ONLY a setcc and a branch, and if a predecessor branches to us and one of our ...
void array_pod_sort(IteratorTy Start, IteratorTy End)
array_pod_sort - This sorts an array with the specified start and end extent.
Definition: STLExtras.h:689
static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder, const DataLayout &DL, const TargetTransformInfo &TTI)
If the switch is only used to initialize one or more phi nodes in a common successor block with diffe...
The landingpad instruction holds all of the information necessary to generate correct exception handl...
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:52
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt...
Definition: PatternMatch.h:180
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:256
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:283
unsigned getCaseIndex() const
Returns number of current case.
static Value * ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB, Value *AlternativeV=nullptr)
bool isIdenticalToWhenDefined(const Instruction *I) const
This is like isIdenticalTo, except that it ignores the SubclassOptionalData flags, which may specify conditions under which the instruction's result is undefined.
void set(Value *Val)
Definition: Value.h:624
void set_intersect(S1Ty &S1, const S2Ty &S2)
set_intersect(A, B) - Compute A := A ^ B Identical to set_intersection, except that it works on set<&g...
Definition: SetOperations.h:40
static bool HasBranchWeights(const Instruction *I)
void setAAMetadata(const AAMDNodes &N)
Sets the metadata on this instruction from the AAMDNodes structure.
Definition: Metadata.cpp:1222
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
Definition: PatternMatch.h:512
BasicBlock * getSuccessor(unsigned idx) const
Return the specified successor.
Definition: InstrTypes.h:79
bool TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB)
BB is known to contain an unconditional branch, and contains no instructions other than PHI nodes...
Definition: Local.cpp:822
UnreachableInst * CreateUnreachable()
Definition: IRBuilder.h:783
Conditional or Unconditional Branch instruction.
static bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder)
Turn a switch with two reachable destinations into an integer range comparison and branch...
This function has undefined behavior.
This is an important base class in LLVM.
Definition: Constant.h:42
const Value * getCondition() const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
Resume the propagation of an exception.
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1321
static bool CasesAreContiguous(SmallVectorImpl< ConstantInt * > &Cases)
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:36
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.h:1609
Indirect Branch Instruction.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1947
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:116
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:259
bool extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) const
Retrieve the raw weight values of a conditional branch or select.
Definition: Metadata.cpp:1272
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
constexpr bool isPowerOf2_64(uint64_t Value)
isPowerOf2_64 - This function returns true if the argument is a power of two 0 (64 bit edition...
Definition: MathExtras.h:405
op_iterator op_end()
Definition: User.h:207
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
void splice(iterator where, iplist_impl &L2)
Definition: ilist.h:342
bool any_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:743
uint32_t Offset
const InstListType & getInstList() const
Return the underlying instruction list container.
Definition: BasicBlock.h:249
static bool canSinkInstructions(ArrayRef< Instruction * > Insts, DenseMap< Instruction *, SmallVector< Value *, 4 >> &PHIOperands)
This instruction compares its operands according to the predicate given to the constructor.
static const unsigned End
User * getUser() const
Returns the User that contains this Use.
Definition: Use.cpp:41
Value * getOperand(unsigned i) const
Definition: User.h:145
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:119
static bool EliminateDeadSwitchCases(SwitchInst *SI, AssumptionCache *AC, const DataLayout &DL)
Compute masked bits for the condition of a switch and use it to remove dead cases.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:93
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1466
self_iterator getIterator()
Definition: ilist_node.h:81
unsigned getIntegerBitWidth() const
Definition: DerivedTypes.h:96
Class to represent integer types.
Definition: DerivedTypes.h:39
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:80
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:960
bool pred_empty(const BasicBlock *BB)
Definition: IR/CFG.h:107
static bool checkCSEInPredecessor(Instruction *Inst, BasicBlock *PB)
Return true if the given instruction is available in its predecessor block.
static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI, const DataLayout &DL)
Given a BB that starts with the specified two-entry PHI node, see if we can eliminate it...
Optional< bool > isImpliedCondition(const Value *LHS, const Value *RHS, const DataLayout &DL, bool InvertAPred=false, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Return true if RHS is known to be implied true by LHS.
bool isUnordered() const
Definition: Instructions.h:385
LLVM_NODISCARD bool empty() const
Definition: SmallPtrSet.h:98
static cl::opt< bool > SpeculateOneExpensiveInst("speculate-one-expensive-inst", cl::Hidden, cl::init(true), cl::desc("Allow exactly one expensive instruction to be speculatively ""executed"))
bool isEmptySet() const
Return true if this set contains no members.
static Constant * ConstantFold(Instruction *I, const DataLayout &DL, const SmallDenseMap< Value *, Constant * > &ConstantPool)
Try to fold instruction I into a constant.
BlockMass operator*(BlockMass L, BranchProbability R)
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:213
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1337
uint64_t NextPowerOf2(uint64_t A)
NextPowerOf2 - Returns the next power of two (in 64-bits) that is strictly greater than A...
Definition: MathExtras.h:619
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr)
If the specified value is a trivially dead instruction, delete it.
Definition: Local.cpp:355
iterator erase(const_iterator CI)
Definition: SmallVector.h:431
Value(Type *Ty, unsigned scid)
Definition: Value.cpp:48
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:654
static bool sinkLastInstruction(ArrayRef< BasicBlock * > Blocks)
const Value * getTrueValue() const
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldCompareInstOperands - Attempt to constant fold a compare instruction (icmp/fcmp) with the...
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:391
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1183
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:857
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1298
const T & back() const
back - Get the last element.
Definition: ArrayRef.h:150
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
hexagon gen pred
bool isTerminator() const
Definition: Instruction.h:114
BasicBlock * getUnwindDest() const
bool isConditional() const
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1083
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
Definition: DataLayout.cpp:709
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1034
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:292
uint64_t * Vals
Iterator for intrusive lists based on ilist_node.
static cl::opt< unsigned > MaxSpeculationDepth("max-speculation-depth", cl::Hidden, cl::init(10), cl::desc("Limit maximum recursion depth when calculating costs of ""speculatively executed instructions"))
BasicBlock * getUnwindDest() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
unsigned countPopulation(T Value)
Count the number of set bits in a value.
Definition: MathExtras.h:494
bool fitsInLegalInteger(unsigned Width) const
Returns true if the specified type fits in a native integer type supported by the CPU...
Definition: DataLayout.h:307
This is the shared class of boolean and integer constants.
Definition: Constants.h:88
unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Return the number of times the sign bit of the register is replicated into the other bits...
bool slt(const APInt &RHS) const
Signed less than comparison.
Definition: APInt.cpp:533
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
bool cannotDuplicate() const
Determine if the call cannot be duplicated.
iterator end()
Definition: BasicBlock.h:230
unsigned getNumSuccessors() const
APInt getSetSize() const
Return the number of elements in this set.
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:1425
static bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder, const DataLayout &DL)
The specified branch is a conditional branch.
ConstantRange inverse() const
Return a new range that is the logical not of the current set.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:59
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:175
This class represents a range of values.
Definition: ConstantRange.h:45
This is the common base class for debug info intrinsics.
Definition: IntrinsicInst.h:67
Value * GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue, BasicBlock *&IfFalse)
Check whether BB is the merge point of a if-region.
TerminatorInst * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
const APInt & getLower() const
Return the lower value for this range.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:625
INITIALIZE_PASS(HexagonGenMux,"hexagon-mux","Hexagon generate mux instructions", false, false) void HexagonGenMux I isValid()
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:382
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:307
BasicBlockTy * getCaseSuccessor()
Resolves successor for current case.
BasicBlock * getBasicBlock() const
Definition: Constants.h:851
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:558
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
pred_range predecessors(BasicBlock *BB)
Definition: IR/CFG.h:110
const BasicBlock & getEntryBlock() const
Definition: Function.h:519
bool isConvergent() const
Determine if the call is convergent.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:90
CaseIt findCaseValue(const ConstantInt *C)
Search all of the case values for the specified constant.
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:506
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
void setPredicate(Predicate P)
Set the predicate for this instruction to the specified value.
Definition: InstrTypes.h:965
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
static void EraseTerminatorInstAndDCECond(TerminatorInst *TI)
void setOperand(unsigned i, Value *Val)
Definition: User.h:150
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:122
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
Definition: Constants.cpp:105
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
Class for arbitrary precision integers.
Definition: APInt.h:77
static Constant * getFalse(Type *Ty)
For a boolean type, or a vector of boolean type, return false, or a vector with every element false...
static bool HoistThenElseCodeToIf(BranchInst *BI, const TargetTransformInfo &TTI)
Given a conditional branch that goes to BB1 and BB2, hoist any common code in the two blocks up into ...
BasicBlock * getSingleSuccessor()
Return the successor of this block if it has a single successor.
Definition: BasicBlock.cpp:253
static cl::opt< bool > DupRet("simplifycfg-dup-ret", cl::Hidden, cl::init(false), cl::desc("Duplicate return instructions into unconditional branches"))
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:195
iterator_range< user_iterator > users()
Definition: Value.h:370
BasicBlock * getSinglePredecessor()
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:226
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM...
Definition: ValueMapper.h:243
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:932
void FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceResults *MemDep=nullptr)
We know that BB has one predecessor.
Value * getCondition() const
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:464
void removeUnwindEdge(BasicBlock *BB)
Replace 'BB's terminator with one that does not have an unwind successor block.
Definition: Local.cpp:1611
If this flag is set, the remapper ignores missing function-local entries (Argument, Instruction, BasicBlock) that are not in the value map.
Definition: ValueMapper.h:80
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1942
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Definition: Operator.h:49
SwitchInst * CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases=10, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a switch instruction with the specified value, default dest, and with a hint for the number of...
Definition: IRBuilder.h:722
static Constant * getNeg(Constant *C, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2113
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
Definition: IRBuilder.h:669
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:384
BasicBlock * getSuccessor(unsigned idx) const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
Value * getCondition() const
iterator insert(iterator where, pointer New)
Definition: ilist.h:241
bool shouldBuildLookupTables() const
Return true if switches should be turned into lookup tables for the target.
BasicBlock * getDefaultDest() const
bool MergeBlockIntoPredecessor(BasicBlock *BB, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemoryDependenceResults *MemDep=nullptr)
Attempts to merge a block into its predecessor, if possible.
iterator begin()
Definition: DenseMap.h:65
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
ImmutableCallSite - establish a view to a call site for examination.
Definition: CallSite.h:665
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
Definition: BasicBlock.cpp:97
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:383
static bool removeEmptyCleanup(CleanupReturnInst *RI)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
ReturnInst * FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB, BasicBlock *Pred)
This method duplicates the specified return instruction into a predecessor which ends in an unconditi...
static bool ValuesOverlap(std::vector< ValueEqualityComparisonCase > &C1, std::vector< ValueEqualityComparisonCase > &C2)
Return true if there are any keys in C1 that exist in C2 as well.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:374
void removeCase(CaseIt i)
This method removes the specified case and its successor from the switch instruction.
void combineMetadataForCSE(Instruction *K, const Instruction *J)
Combine the metadata of two instructions so that K can replace J.
Definition: Local.cpp:1747
size_t size() const
Definition: BasicBlock.h:238
CaseIt case_default()
Returns an iterator that points to the default case.
static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI, Value *SelectValue, IRBuilder<> &Builder)
static bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB, const TargetTransformInfo &TTI)
Speculate a conditional basic block flattening the CFG.
static bool InitializeUniqueCases(SwitchInst *SI, PHINode *&PHI, BasicBlock *&CommonDest, SwitchCaseResultVectorTy &UniqueResults, Constant *&DefaultResult, const DataLayout &DL, const TargetTransformInfo &TTI)
static bool ShouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize, const TargetTransformInfo &TTI, const DataLayout &DL, const SmallDenseMap< PHINode *, Type * > &ResultTypes)
Determine whether a lookup table should be built for this switch, based on the number of cases...
unsigned getNumCases() const
Return the number of 'cases' in this switch instruction, excluding the default case.
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1354
BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
void setCondition(Value *V)
static BasicBlock * allPredecessorsComeFromSameSource(BasicBlock *BB)
static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout &DL)
If we have a conditional branch on a PHI node value that is defined in the same block as the branch a...
const unsigned Kind
Multiway switch.
static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder, const DataLayout &DL, const TargetTransformInfo &TTI)
bool use_empty() const
Definition: Value.h:299
const APInt & getUpper() const
Return the upper value for this range.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static Constant * LookupConstant(Value *V, const SmallDenseMap< Value *, Constant * > &ConstantPool)
If V is a Constant, return it.
user_iterator user_begin()
Definition: Value.h:346
The cost of a typical 'add' instruction.
Constant * ConstantFoldInstOperands(Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands...
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:33
static bool canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx)
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:326
unsigned getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:108
static bool ForwardSwitchConditionToPHI(SwitchInst *SI)
Try to forward the condition of a switch instruction to a phi node dominated by the switch...
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:537
LLVM Value Representation.
Definition: Value.h:71
succ_range successors(BasicBlock *BB)
Definition: IR/CFG.h:143
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:111
void setDefaultDest(BasicBlock *DefaultCase)
BranchInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
Definition: IRBuilder.h:693
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:81
static const Function * getParent(const Value *V)
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
Definition: Instruction.cpp:95
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:951
Invoke instruction.
#define DEBUG(X)
Definition: Debug.h:100
iterator end() const
Definition: ArrayRef.h:325
static PHINode * FindPHIForConditionForwarding(ConstantInt *CaseValue, BasicBlock *BB, int *PhiIndex)
If BB would be eligible for simplification by TryToSimplifyUncondBranchFromEmptyBlock (i...
const Value * getFalseValue() const
A single uniqued string.
Definition: Metadata.h:586
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:980
Value * SimplifyInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
See if we can compute a simplified version of this instruction.
This pass exposes codegen information to IR-level passes.
bool operator==(uint64_t V1, const APInt &V2)
Definition: APInt.h:1722
static void FitWeights(MutableArrayRef< uint64_t > Weights)
Keep halving the weights until all can fit in uint32_t.
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:209
int * Ptr
unsigned getNumUses() const
This method computes the number of uses of this Value.
Definition: Value.cpp:137
void setIncomingValue(unsigned i, Value *V)
int getUserCost(const User *U) const
Estimate the cost of a given IR user when lowered.
static bool SafeToMergeTerminators(TerminatorInst *SI1, TerminatorInst *SI2, SmallSetVector< BasicBlock *, 4 > *FailBlocks=nullptr)
Return true if it is safe to merge these two terminator instructions together.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI)
static bool TryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI, IRBuilder<> &Builder, const DataLayout &DL, const TargetTransformInfo &TTI, unsigned BonusInstThreshold, AssumptionCache *AC)
This is called when we find an icmp instruction (a seteq/setne with a constant) as the only instructi...
bool isThreadDependent() const
Return true if the value can vary between threads.
Definition: Constants.cpp:389
Value * getPointerOperand()
Definition: Instructions.h:394
static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred, BasicBlock *ExistPred)
Update PHI nodes in Succ to indicate that there will now be entries in it from the 'NewPred' block...
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
void combineMetadata(Instruction *K, const Instruction *J, ArrayRef< unsigned > KnownIDs)
Combine the metadata of two instructions so that K can replace J.
Definition: Local.cpp:1682
unsigned getNumDestinations() const
return the number of possible destinations in this indirectbr instruction.
void removeDestination(unsigned i)
This method removes the specified successor from the indirectbr instruction.
unsigned Log2_64(uint64_t Value)
Log2_64 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:519
const BasicBlock * getParent() const
Definition: Instruction.h:62
bool isProfitableToHoist(Instruction *I) const
Return true if it is profitable to hoist instruction in the then/else to before if.
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:83
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:206
static bool DominatesMergePoint(Value *V, BasicBlock *BB, SmallPtrSetImpl< Instruction * > *AggressiveInsts, unsigned &CostRemaining, const TargetTransformInfo &TTI, unsigned Depth=0)
If we have a merge point of an "if condition" as accepted above, return true if the specified value d...
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:222
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
bool isMetadataTy() const
Return true if this is 'metadata'.
Definition: Type.h:189
gep_type_iterator gep_type_begin(const User *GEP)
static bool TryToMergeLandingPad(LandingPadInst *LPad, BranchInst *BI, BasicBlock *BB)
Given an block with only a single landing pad and a unconditional branch try to find another basic bl...