LLVM  3.7.0
TailRecursionElimination.cpp
Go to the documentation of this file.
1 //===- TailRecursionElimination.cpp - Eliminate Tail Calls ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file transforms calls of the current function (self recursion) followed
11 // by a return instruction with a branch to the entry of the function, creating
12 // a loop. This pass also implements the following extensions to the basic
13 // algorithm:
14 //
15 // 1. Trivial instructions between the call and return do not prevent the
16 // transformation from taking place, though currently the analysis cannot
17 // support moving any really useful instructions (only dead ones).
18 // 2. This pass transforms functions that are prevented from being tail
19 // recursive by an associative and commutative expression to use an
20 // accumulator variable, thus compiling the typical naive factorial or
21 // 'fib' implementation into efficient code.
22 // 3. TRE is performed if the function returns void, if the return
23 // returns the result returned by the call, or if the function returns a
24 // run-time constant on all exits from the function. It is possible, though
25 // unlikely, that the return returns something else (like constant 0), and
26 // can still be TRE'd. It can be TRE'd if ALL OTHER return instructions in
27 // the function return the exact same value.
28 // 4. If it can prove that callees do not access their caller stack frame,
29 // they are marked as eligible for tail call elimination (by the code
30 // generator).
31 //
32 // There are several improvements that could be made:
33 //
34 // 1. If the function has any alloca instructions, these instructions will be
35 // moved out of the entry block of the function, causing them to be
36 // evaluated each time through the tail recursion. Safely keeping allocas
37 // in the entry block requires analysis to proves that the tail-called
38 // function does not read or write the stack object.
39 // 2. Tail recursion is only performed if the call immediately precedes the
40 // return instruction. It's possible that there could be a jump between
41 // the call and the return.
42 // 3. There can be intervening operations between the call and the return that
43 // prevent the TRE from occurring. For example, there could be GEP's and
44 // stores to memory that will not be read or written by the call. This
45 // requires some substantial analysis (such as with DSA) to prove safe to
46 // move ahead of the call, but doing so could allow many more TREs to be
47 // performed, for example in TreeAdd/TreeAlloc from the treeadd benchmark.
48 // 4. The algorithm we use to detect if callees access their caller stack
49 // frames is very primitive.
50 //
51 //===----------------------------------------------------------------------===//
52 
53 #include "llvm/Transforms/Scalar.h"
54 #include "llvm/ADT/STLExtras.h"
55 #include "llvm/ADT/SmallPtrSet.h"
56 #include "llvm/ADT/Statistic.h"
57 #include "llvm/Analysis/CFG.h"
61 #include "llvm/Analysis/Loads.h"
63 #include "llvm/IR/CFG.h"
64 #include "llvm/IR/CallSite.h"
65 #include "llvm/IR/Constants.h"
66 #include "llvm/IR/DataLayout.h"
67 #include "llvm/IR/DerivedTypes.h"
68 #include "llvm/IR/DiagnosticInfo.h"
69 #include "llvm/IR/Function.h"
70 #include "llvm/IR/Instructions.h"
71 #include "llvm/IR/IntrinsicInst.h"
72 #include "llvm/IR/Module.h"
73 #include "llvm/IR/ValueHandle.h"
74 #include "llvm/Pass.h"
75 #include "llvm/Support/Debug.h"
79 using namespace llvm;
80 
81 #define DEBUG_TYPE "tailcallelim"
82 
83 STATISTIC(NumEliminated, "Number of tail calls removed");
84 STATISTIC(NumRetDuped, "Number of return duplicated");
85 STATISTIC(NumAccumAdded, "Number of accumulators introduced");
86 
87 namespace {
88  struct TailCallElim : public FunctionPass {
89  const TargetTransformInfo *TTI;
90 
91  static char ID; // Pass identification, replacement for typeid
92  TailCallElim() : FunctionPass(ID) {
94  }
95 
96  void getAnalysisUsage(AnalysisUsage &AU) const override;
97 
98  bool runOnFunction(Function &F) override;
99 
100  private:
101  bool runTRE(Function &F);
102  bool markTails(Function &F, bool &AllCallsAreTailCalls);
103 
104  CallInst *FindTRECandidate(Instruction *I,
105  bool CannotTailCallElimCallsMarkedTail);
106  bool EliminateRecursiveTailCall(CallInst *CI, ReturnInst *Ret,
107  BasicBlock *&OldEntry,
108  bool &TailCallsAreMarkedTail,
109  SmallVectorImpl<PHINode *> &ArgumentPHIs,
110  bool CannotTailCallElimCallsMarkedTail);
111  bool FoldReturnAndProcessPred(BasicBlock *BB,
112  ReturnInst *Ret, BasicBlock *&OldEntry,
113  bool &TailCallsAreMarkedTail,
114  SmallVectorImpl<PHINode *> &ArgumentPHIs,
115  bool CannotTailCallElimCallsMarkedTail);
116  bool ProcessReturningBlock(ReturnInst *RI, BasicBlock *&OldEntry,
117  bool &TailCallsAreMarkedTail,
118  SmallVectorImpl<PHINode *> &ArgumentPHIs,
119  bool CannotTailCallElimCallsMarkedTail);
120  bool CanMoveAboveCall(Instruction *I, CallInst *CI);
121  Value *CanTransformAccumulatorRecursion(Instruction *I, CallInst *CI);
122  };
123 }
124 
125 char TailCallElim::ID = 0;
126 INITIALIZE_PASS_BEGIN(TailCallElim, "tailcallelim",
127  "Tail Call Elimination", false, false)
130  "Tail Call Elimination", false, false)
131 
132 // Public interface to the TailCallElimination pass
134  return new TailCallElim();
135 }
136 
137 void TailCallElim::getAnalysisUsage(AnalysisUsage &AU) const {
139 }
140 
141 /// \brief Scan the specified function for alloca instructions.
142 /// If it contains any dynamic allocas, returns false.
143 static bool CanTRE(Function &F) {
144  // Because of PR962, we don't TRE dynamic allocas.
145  for (auto &BB : F) {
146  for (auto &I : BB) {
147  if (AllocaInst *AI = dyn_cast<AllocaInst>(&I)) {
148  if (!AI->isStaticAlloca())
149  return false;
150  }
151  }
152  }
153 
154  return true;
155 }
156 
157 bool TailCallElim::runOnFunction(Function &F) {
158  if (skipOptnoneFunction(F))
159  return false;
160 
161  if (F.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
162  return false;
163 
164  bool AllCallsAreTailCalls = false;
165  bool Modified = markTails(F, AllCallsAreTailCalls);
166  if (AllCallsAreTailCalls)
167  Modified |= runTRE(F);
168  return Modified;
169 }
170 
171 namespace {
172 struct AllocaDerivedValueTracker {
173  // Start at a root value and walk its use-def chain to mark calls that use the
174  // value or a derived value in AllocaUsers, and places where it may escape in
175  // EscapePoints.
176  void walk(Value *Root) {
177  SmallVector<Use *, 32> Worklist;
178  SmallPtrSet<Use *, 32> Visited;
179 
180  auto AddUsesToWorklist = [&](Value *V) {
181  for (auto &U : V->uses()) {
182  if (!Visited.insert(&U).second)
183  continue;
184  Worklist.push_back(&U);
185  }
186  };
187 
188  AddUsesToWorklist(Root);
189 
190  while (!Worklist.empty()) {
191  Use *U = Worklist.pop_back_val();
192  Instruction *I = cast<Instruction>(U->getUser());
193 
194  switch (I->getOpcode()) {
195  case Instruction::Call:
196  case Instruction::Invoke: {
197  CallSite CS(I);
198  bool IsNocapture = !CS.isCallee(U) &&
199  CS.doesNotCapture(CS.getArgumentNo(U));
200  callUsesLocalStack(CS, IsNocapture);
201  if (IsNocapture) {
202  // If the alloca-derived argument is passed in as nocapture, then it
203  // can't propagate to the call's return. That would be capturing.
204  continue;
205  }
206  break;
207  }
208  case Instruction::Load: {
209  // The result of a load is not alloca-derived (unless an alloca has
210  // otherwise escaped, but this is a local analysis).
211  continue;
212  }
213  case Instruction::Store: {
214  if (U->getOperandNo() == 0)
215  EscapePoints.insert(I);
216  continue; // Stores have no users to analyze.
217  }
218  case Instruction::BitCast:
219  case Instruction::GetElementPtr:
220  case Instruction::PHI:
221  case Instruction::Select:
222  case Instruction::AddrSpaceCast:
223  break;
224  default:
225  EscapePoints.insert(I);
226  break;
227  }
228 
229  AddUsesToWorklist(I);
230  }
231  }
232 
233  void callUsesLocalStack(CallSite CS, bool IsNocapture) {
234  // Add it to the list of alloca users.
235  AllocaUsers.insert(CS.getInstruction());
236 
237  // If it's nocapture then it can't capture this alloca.
238  if (IsNocapture)
239  return;
240 
241  // If it can write to memory, it can leak the alloca value.
242  if (!CS.onlyReadsMemory())
243  EscapePoints.insert(CS.getInstruction());
244  }
245 
246  SmallPtrSet<Instruction *, 32> AllocaUsers;
247  SmallPtrSet<Instruction *, 32> EscapePoints;
248 };
249 }
250 
251 bool TailCallElim::markTails(Function &F, bool &AllCallsAreTailCalls) {
253  return false;
254  AllCallsAreTailCalls = true;
255 
256  // The local stack holds all alloca instructions and all byval arguments.
257  AllocaDerivedValueTracker Tracker;
258  for (Argument &Arg : F.args()) {
259  if (Arg.hasByValAttr())
260  Tracker.walk(&Arg);
261  }
262  for (auto &BB : F) {
263  for (auto &I : BB)
264  if (AllocaInst *AI = dyn_cast<AllocaInst>(&I))
265  Tracker.walk(AI);
266  }
267 
268  bool Modified = false;
269 
270  // Track whether a block is reachable after an alloca has escaped. Blocks that
271  // contain the escaping instruction will be marked as being visited without an
272  // escaped alloca, since that is how the block began.
273  enum VisitType {
274  UNVISITED,
275  UNESCAPED,
276  ESCAPED
277  };
279 
280  // We propagate the fact that an alloca has escaped from block to successor.
281  // Visit the blocks that are propagating the escapedness first. To do this, we
282  // maintain two worklists.
283  SmallVector<BasicBlock *, 32> WorklistUnescaped, WorklistEscaped;
284 
285  // We may enter a block and visit it thinking that no alloca has escaped yet,
286  // then see an escape point and go back around a loop edge and come back to
287  // the same block twice. Because of this, we defer setting tail on calls when
288  // we first encounter them in a block. Every entry in this list does not
289  // statically use an alloca via use-def chain analysis, but may find an alloca
290  // through other means if the block turns out to be reachable after an escape
291  // point.
292  SmallVector<CallInst *, 32> DeferredTails;
293 
294  BasicBlock *BB = &F.getEntryBlock();
295  VisitType Escaped = UNESCAPED;
296  do {
297  for (auto &I : *BB) {
298  if (Tracker.EscapePoints.count(&I))
299  Escaped = ESCAPED;
300 
301  CallInst *CI = dyn_cast<CallInst>(&I);
302  if (!CI || CI->isTailCall())
303  continue;
304 
305  if (CI->doesNotAccessMemory()) {
306  // A call to a readnone function whose arguments are all things computed
307  // outside this function can be marked tail. Even if you stored the
308  // alloca address into a global, a readnone function can't load the
309  // global anyhow.
310  //
311  // Note that this runs whether we know an alloca has escaped or not. If
312  // it has, then we can't trust Tracker.AllocaUsers to be accurate.
313  bool SafeToTail = true;
314  for (auto &Arg : CI->arg_operands()) {
315  if (isa<Constant>(Arg.getUser()))
316  continue;
317  if (Argument *A = dyn_cast<Argument>(Arg.getUser()))
318  if (!A->hasByValAttr())
319  continue;
320  SafeToTail = false;
321  break;
322  }
323  if (SafeToTail) {
325  F.getContext(), "tailcallelim", F, CI->getDebugLoc(),
326  "marked this readnone call a tail call candidate");
327  CI->setTailCall();
328  Modified = true;
329  continue;
330  }
331  }
332 
333  if (Escaped == UNESCAPED && !Tracker.AllocaUsers.count(CI)) {
334  DeferredTails.push_back(CI);
335  } else {
336  AllCallsAreTailCalls = false;
337  }
338  }
339 
340  for (auto *SuccBB : make_range(succ_begin(BB), succ_end(BB))) {
341  auto &State = Visited[SuccBB];
342  if (State < Escaped) {
343  State = Escaped;
344  if (State == ESCAPED)
345  WorklistEscaped.push_back(SuccBB);
346  else
347  WorklistUnescaped.push_back(SuccBB);
348  }
349  }
350 
351  if (!WorklistEscaped.empty()) {
352  BB = WorklistEscaped.pop_back_val();
353  Escaped = ESCAPED;
354  } else {
355  BB = nullptr;
356  while (!WorklistUnescaped.empty()) {
357  auto *NextBB = WorklistUnescaped.pop_back_val();
358  if (Visited[NextBB] == UNESCAPED) {
359  BB = NextBB;
360  Escaped = UNESCAPED;
361  break;
362  }
363  }
364  }
365  } while (BB);
366 
367  for (CallInst *CI : DeferredTails) {
368  if (Visited[CI->getParent()] != ESCAPED) {
369  // If the escape point was part way through the block, calls after the
370  // escape point wouldn't have been put into DeferredTails.
371  emitOptimizationRemark(F.getContext(), "tailcallelim", F,
372  CI->getDebugLoc(),
373  "marked this call a tail call candidate");
374  CI->setTailCall();
375  Modified = true;
376  } else {
377  AllCallsAreTailCalls = false;
378  }
379  }
380 
381  return Modified;
382 }
383 
384 bool TailCallElim::runTRE(Function &F) {
385  // If this function is a varargs function, we won't be able to PHI the args
386  // right, so don't even try to convert it...
387  if (F.getFunctionType()->isVarArg()) return false;
388 
389  TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
390  BasicBlock *OldEntry = nullptr;
391  bool TailCallsAreMarkedTail = false;
392  SmallVector<PHINode*, 8> ArgumentPHIs;
393  bool MadeChange = false;
394 
395  // If false, we cannot perform TRE on tail calls marked with the 'tail'
396  // attribute, because doing so would cause the stack size to increase (real
397  // TRE would deallocate variable sized allocas, TRE doesn't).
398  bool CanTRETailMarkedCall = CanTRE(F);
399 
400  // Change any tail recursive calls to loops.
401  //
402  // FIXME: The code generator produces really bad code when an 'escaping
403  // alloca' is changed from being a static alloca to being a dynamic alloca.
404  // Until this is resolved, disable this transformation if that would ever
405  // happen. This bug is PR962.
406  for (Function::iterator BBI = F.begin(), E = F.end(); BBI != E; /*in loop*/) {
407  BasicBlock *BB = BBI++; // FoldReturnAndProcessPred may delete BB.
408  if (ReturnInst *Ret = dyn_cast<ReturnInst>(BB->getTerminator())) {
409  bool Change = ProcessReturningBlock(Ret, OldEntry, TailCallsAreMarkedTail,
410  ArgumentPHIs, !CanTRETailMarkedCall);
411  if (!Change && BB->getFirstNonPHIOrDbg() == Ret)
412  Change = FoldReturnAndProcessPred(BB, Ret, OldEntry,
413  TailCallsAreMarkedTail, ArgumentPHIs,
414  !CanTRETailMarkedCall);
415  MadeChange |= Change;
416  }
417  }
418 
419  // If we eliminated any tail recursions, it's possible that we inserted some
420  // silly PHI nodes which just merge an initial value (the incoming operand)
421  // with themselves. Check to see if we did and clean up our mess if so. This
422  // occurs when a function passes an argument straight through to its tail
423  // call.
424  for (unsigned i = 0, e = ArgumentPHIs.size(); i != e; ++i) {
425  PHINode *PN = ArgumentPHIs[i];
426 
427  // If the PHI Node is a dynamic constant, replace it with the value it is.
428  if (Value *PNV = SimplifyInstruction(PN, F.getParent()->getDataLayout())) {
429  PN->replaceAllUsesWith(PNV);
430  PN->eraseFromParent();
431  }
432  }
433 
434  return MadeChange;
435 }
436 
437 
438 /// Return true if it is safe to move the specified
439 /// instruction from after the call to before the call, assuming that all
440 /// instructions between the call and this instruction are movable.
441 ///
442 bool TailCallElim::CanMoveAboveCall(Instruction *I, CallInst *CI) {
443  // FIXME: We can move load/store/call/free instructions above the call if the
444  // call does not mod/ref the memory location being processed.
445  if (I->mayHaveSideEffects()) // This also handles volatile loads.
446  return false;
447 
448  if (LoadInst *L = dyn_cast<LoadInst>(I)) {
449  // Loads may always be moved above calls without side effects.
450  if (CI->mayHaveSideEffects()) {
451  // Non-volatile loads may be moved above a call with side effects if it
452  // does not write to memory and the load provably won't trap.
453  // FIXME: Writes to memory only matter if they may alias the pointer
454  // being loaded from.
455  if (CI->mayWriteToMemory() ||
456  !isSafeToLoadUnconditionally(L->getPointerOperand(), L,
457  L->getAlignment()))
458  return false;
459  }
460  }
461 
462  // Otherwise, if this is a side-effect free instruction, check to make sure
463  // that it does not use the return value of the call. If it doesn't use the
464  // return value of the call, it must only use things that are defined before
465  // the call, or movable instructions between the call and the instruction
466  // itself.
467  for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i)
468  if (I->getOperand(i) == CI)
469  return false;
470  return true;
471 }
472 
473 /// Return true if the specified value is the same when the return would exit
474 /// as it was when the initial iteration of the recursive function was executed.
475 ///
476 /// We currently handle static constants and arguments that are not modified as
477 /// part of the recursion.
478 static bool isDynamicConstant(Value *V, CallInst *CI, ReturnInst *RI) {
479  if (isa<Constant>(V)) return true; // Static constants are always dyn consts
480 
481  // Check to see if this is an immutable argument, if so, the value
482  // will be available to initialize the accumulator.
483  if (Argument *Arg = dyn_cast<Argument>(V)) {
484  // Figure out which argument number this is...
485  unsigned ArgNo = 0;
486  Function *F = CI->getParent()->getParent();
487  for (Function::arg_iterator AI = F->arg_begin(); &*AI != Arg; ++AI)
488  ++ArgNo;
489 
490  // If we are passing this argument into call as the corresponding
491  // argument operand, then the argument is dynamically constant.
492  // Otherwise, we cannot transform this function safely.
493  if (CI->getArgOperand(ArgNo) == Arg)
494  return true;
495  }
496 
497  // Switch cases are always constant integers. If the value is being switched
498  // on and the return is only reachable from one of its cases, it's
499  // effectively constant.
500  if (BasicBlock *UniquePred = RI->getParent()->getUniquePredecessor())
501  if (SwitchInst *SI = dyn_cast<SwitchInst>(UniquePred->getTerminator()))
502  if (SI->getCondition() == V)
503  return SI->getDefaultDest() != RI->getParent();
504 
505  // Not a constant or immutable argument, we can't safely transform.
506  return false;
507 }
508 
509 /// Check to see if the function containing the specified tail call consistently
510 /// returns the same runtime-constant value at all exit points except for
511 /// IgnoreRI. If so, return the returned value.
513  Function *F = CI->getParent()->getParent();
514  Value *ReturnedValue = nullptr;
515 
516  for (Function::iterator BBI = F->begin(), E = F->end(); BBI != E; ++BBI) {
517  ReturnInst *RI = dyn_cast<ReturnInst>(BBI->getTerminator());
518  if (RI == nullptr || RI == IgnoreRI) continue;
519 
520  // We can only perform this transformation if the value returned is
521  // evaluatable at the start of the initial invocation of the function,
522  // instead of at the end of the evaluation.
523  //
524  Value *RetOp = RI->getOperand(0);
525  if (!isDynamicConstant(RetOp, CI, RI))
526  return nullptr;
527 
528  if (ReturnedValue && RetOp != ReturnedValue)
529  return nullptr; // Cannot transform if differing values are returned.
530  ReturnedValue = RetOp;
531  }
532  return ReturnedValue;
533 }
534 
535 /// If the specified instruction can be transformed using accumulator recursion
536 /// elimination, return the constant which is the start of the accumulator
537 /// value. Otherwise return null.
538 Value *TailCallElim::CanTransformAccumulatorRecursion(Instruction *I,
539  CallInst *CI) {
540  if (!I->isAssociative() || !I->isCommutative()) return nullptr;
541  assert(I->getNumOperands() == 2 &&
542  "Associative/commutative operations should have 2 args!");
543 
544  // Exactly one operand should be the result of the call instruction.
545  if ((I->getOperand(0) == CI && I->getOperand(1) == CI) ||
546  (I->getOperand(0) != CI && I->getOperand(1) != CI))
547  return nullptr;
548 
549  // The only user of this instruction we allow is a single return instruction.
550  if (!I->hasOneUse() || !isa<ReturnInst>(I->user_back()))
551  return nullptr;
552 
553  // Ok, now we have to check all of the other return instructions in this
554  // function. If they return non-constants or differing values, then we cannot
555  // transform the function safely.
556  return getCommonReturnValue(cast<ReturnInst>(I->user_back()), CI);
557 }
558 
560  while (isa<DbgInfoIntrinsic>(I))
561  ++I;
562  return &*I;
563 }
564 
565 CallInst*
566 TailCallElim::FindTRECandidate(Instruction *TI,
567  bool CannotTailCallElimCallsMarkedTail) {
568  BasicBlock *BB = TI->getParent();
569  Function *F = BB->getParent();
570 
571  if (&BB->front() == TI) // Make sure there is something before the terminator.
572  return nullptr;
573 
574  // Scan backwards from the return, checking to see if there is a tail call in
575  // this block. If so, set CI to it.
576  CallInst *CI = nullptr;
577  BasicBlock::iterator BBI = TI;
578  while (true) {
579  CI = dyn_cast<CallInst>(BBI);
580  if (CI && CI->getCalledFunction() == F)
581  break;
582 
583  if (BBI == BB->begin())
584  return nullptr; // Didn't find a potential tail call.
585  --BBI;
586  }
587 
588  // If this call is marked as a tail call, and if there are dynamic allocas in
589  // the function, we cannot perform this optimization.
590  if (CI->isTailCall() && CannotTailCallElimCallsMarkedTail)
591  return nullptr;
592 
593  // As a special case, detect code like this:
594  // double fabs(double f) { return __builtin_fabs(f); } // a 'fabs' call
595  // and disable this xform in this case, because the code generator will
596  // lower the call to fabs into inline code.
597  if (BB == &F->getEntryBlock() &&
598  FirstNonDbg(BB->front()) == CI &&
599  FirstNonDbg(std::next(BB->begin())) == TI &&
600  CI->getCalledFunction() &&
601  !TTI->isLoweredToCall(CI->getCalledFunction())) {
602  // A single-block function with just a call and a return. Check that
603  // the arguments match.
605  E = CallSite(CI).arg_end();
607  FE = F->arg_end();
608  for (; I != E && FI != FE; ++I, ++FI)
609  if (*I != &*FI) break;
610  if (I == E && FI == FE)
611  return nullptr;
612  }
613 
614  return CI;
615 }
616 
617 bool TailCallElim::EliminateRecursiveTailCall(CallInst *CI, ReturnInst *Ret,
618  BasicBlock *&OldEntry,
619  bool &TailCallsAreMarkedTail,
620  SmallVectorImpl<PHINode *> &ArgumentPHIs,
621  bool CannotTailCallElimCallsMarkedTail) {
622  // If we are introducing accumulator recursion to eliminate operations after
623  // the call instruction that are both associative and commutative, the initial
624  // value for the accumulator is placed in this variable. If this value is set
625  // then we actually perform accumulator recursion elimination instead of
626  // simple tail recursion elimination. If the operation is an LLVM instruction
627  // (eg: "add") then it is recorded in AccumulatorRecursionInstr. If not, then
628  // we are handling the case when the return instruction returns a constant C
629  // which is different to the constant returned by other return instructions
630  // (which is recorded in AccumulatorRecursionEliminationInitVal). This is a
631  // special case of accumulator recursion, the operation being "return C".
632  Value *AccumulatorRecursionEliminationInitVal = nullptr;
633  Instruction *AccumulatorRecursionInstr = nullptr;
634 
635  // Ok, we found a potential tail call. We can currently only transform the
636  // tail call if all of the instructions between the call and the return are
637  // movable to above the call itself, leaving the call next to the return.
638  // Check that this is the case now.
639  BasicBlock::iterator BBI = CI;
640  for (++BBI; &*BBI != Ret; ++BBI) {
641  if (CanMoveAboveCall(BBI, CI)) continue;
642 
643  // If we can't move the instruction above the call, it might be because it
644  // is an associative and commutative operation that could be transformed
645  // using accumulator recursion elimination. Check to see if this is the
646  // case, and if so, remember the initial accumulator value for later.
647  if ((AccumulatorRecursionEliminationInitVal =
648  CanTransformAccumulatorRecursion(BBI, CI))) {
649  // Yes, this is accumulator recursion. Remember which instruction
650  // accumulates.
651  AccumulatorRecursionInstr = BBI;
652  } else {
653  return false; // Otherwise, we cannot eliminate the tail recursion!
654  }
655  }
656 
657  // We can only transform call/return pairs that either ignore the return value
658  // of the call and return void, ignore the value of the call and return a
659  // constant, return the value returned by the tail call, or that are being
660  // accumulator recursion variable eliminated.
661  if (Ret->getNumOperands() == 1 && Ret->getReturnValue() != CI &&
662  !isa<UndefValue>(Ret->getReturnValue()) &&
663  AccumulatorRecursionEliminationInitVal == nullptr &&
664  !getCommonReturnValue(nullptr, CI)) {
665  // One case remains that we are able to handle: the current return
666  // instruction returns a constant, and all other return instructions
667  // return a different constant.
668  if (!isDynamicConstant(Ret->getReturnValue(), CI, Ret))
669  return false; // Current return instruction does not return a constant.
670  // Check that all other return instructions return a common constant. If
671  // so, record it in AccumulatorRecursionEliminationInitVal.
672  AccumulatorRecursionEliminationInitVal = getCommonReturnValue(Ret, CI);
673  if (!AccumulatorRecursionEliminationInitVal)
674  return false;
675  }
676 
677  BasicBlock *BB = Ret->getParent();
678  Function *F = BB->getParent();
679 
680  emitOptimizationRemark(F->getContext(), "tailcallelim", *F, CI->getDebugLoc(),
681  "transforming tail recursion to loop");
682 
683  // OK! We can transform this tail call. If this is the first one found,
684  // create the new entry block, allowing us to branch back to the old entry.
685  if (!OldEntry) {
686  OldEntry = &F->getEntryBlock();
687  BasicBlock *NewEntry = BasicBlock::Create(F->getContext(), "", F, OldEntry);
688  NewEntry->takeName(OldEntry);
689  OldEntry->setName("tailrecurse");
690  BranchInst::Create(OldEntry, NewEntry);
691 
692  // If this tail call is marked 'tail' and if there are any allocas in the
693  // entry block, move them up to the new entry block.
694  TailCallsAreMarkedTail = CI->isTailCall();
695  if (TailCallsAreMarkedTail)
696  // Move all fixed sized allocas from OldEntry to NewEntry.
697  for (BasicBlock::iterator OEBI = OldEntry->begin(), E = OldEntry->end(),
698  NEBI = NewEntry->begin(); OEBI != E; )
699  if (AllocaInst *AI = dyn_cast<AllocaInst>(OEBI++))
700  if (isa<ConstantInt>(AI->getArraySize()))
701  AI->moveBefore(NEBI);
702 
703  // Now that we have created a new block, which jumps to the entry
704  // block, insert a PHI node for each argument of the function.
705  // For now, we initialize each PHI to only have the real arguments
706  // which are passed in.
707  Instruction *InsertPos = OldEntry->begin();
708  for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end();
709  I != E; ++I) {
710  PHINode *PN = PHINode::Create(I->getType(), 2,
711  I->getName() + ".tr", InsertPos);
712  I->replaceAllUsesWith(PN); // Everyone use the PHI node now!
713  PN->addIncoming(I, NewEntry);
714  ArgumentPHIs.push_back(PN);
715  }
716  }
717 
718  // If this function has self recursive calls in the tail position where some
719  // are marked tail and some are not, only transform one flavor or another. We
720  // have to choose whether we move allocas in the entry block to the new entry
721  // block or not, so we can't make a good choice for both. NOTE: We could do
722  // slightly better here in the case that the function has no entry block
723  // allocas.
724  if (TailCallsAreMarkedTail && !CI->isTailCall())
725  return false;
726 
727  // Ok, now that we know we have a pseudo-entry block WITH all of the
728  // required PHI nodes, add entries into the PHI node for the actual
729  // parameters passed into the tail-recursive call.
730  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i)
731  ArgumentPHIs[i]->addIncoming(CI->getArgOperand(i), BB);
732 
733  // If we are introducing an accumulator variable to eliminate the recursion,
734  // do so now. Note that we _know_ that no subsequent tail recursion
735  // eliminations will happen on this function because of the way the
736  // accumulator recursion predicate is set up.
737  //
738  if (AccumulatorRecursionEliminationInitVal) {
739  Instruction *AccRecInstr = AccumulatorRecursionInstr;
740  // Start by inserting a new PHI node for the accumulator.
741  pred_iterator PB = pred_begin(OldEntry), PE = pred_end(OldEntry);
742  PHINode *AccPN =
743  PHINode::Create(AccumulatorRecursionEliminationInitVal->getType(),
744  std::distance(PB, PE) + 1,
745  "accumulator.tr", OldEntry->begin());
746 
747  // Loop over all of the predecessors of the tail recursion block. For the
748  // real entry into the function we seed the PHI with the initial value,
749  // computed earlier. For any other existing branches to this block (due to
750  // other tail recursions eliminated) the accumulator is not modified.
751  // Because we haven't added the branch in the current block to OldEntry yet,
752  // it will not show up as a predecessor.
753  for (pred_iterator PI = PB; PI != PE; ++PI) {
754  BasicBlock *P = *PI;
755  if (P == &F->getEntryBlock())
756  AccPN->addIncoming(AccumulatorRecursionEliminationInitVal, P);
757  else
758  AccPN->addIncoming(AccPN, P);
759  }
760 
761  if (AccRecInstr) {
762  // Add an incoming argument for the current block, which is computed by
763  // our associative and commutative accumulator instruction.
764  AccPN->addIncoming(AccRecInstr, BB);
765 
766  // Next, rewrite the accumulator recursion instruction so that it does not
767  // use the result of the call anymore, instead, use the PHI node we just
768  // inserted.
769  AccRecInstr->setOperand(AccRecInstr->getOperand(0) != CI, AccPN);
770  } else {
771  // Add an incoming argument for the current block, which is just the
772  // constant returned by the current return instruction.
773  AccPN->addIncoming(Ret->getReturnValue(), BB);
774  }
775 
776  // Finally, rewrite any return instructions in the program to return the PHI
777  // node instead of the "initval" that they do currently. This loop will
778  // actually rewrite the return value we are destroying, but that's ok.
779  for (Function::iterator BBI = F->begin(), E = F->end(); BBI != E; ++BBI)
780  if (ReturnInst *RI = dyn_cast<ReturnInst>(BBI->getTerminator()))
781  RI->setOperand(0, AccPN);
782  ++NumAccumAdded;
783  }
784 
785  // Now that all of the PHI nodes are in place, remove the call and
786  // ret instructions, replacing them with an unconditional branch.
787  BranchInst *NewBI = BranchInst::Create(OldEntry, Ret);
788  NewBI->setDebugLoc(CI->getDebugLoc());
789 
790  BB->getInstList().erase(Ret); // Remove return.
791  BB->getInstList().erase(CI); // Remove call.
792  ++NumEliminated;
793  return true;
794 }
795 
796 bool TailCallElim::FoldReturnAndProcessPred(BasicBlock *BB,
797  ReturnInst *Ret, BasicBlock *&OldEntry,
798  bool &TailCallsAreMarkedTail,
799  SmallVectorImpl<PHINode *> &ArgumentPHIs,
800  bool CannotTailCallElimCallsMarkedTail) {
801  bool Change = false;
802 
803  // If the return block contains nothing but the return and PHI's,
804  // there might be an opportunity to duplicate the return in its
805  // predecessors and perform TRC there. Look for predecessors that end
806  // in unconditional branch and recursive call(s).
807  SmallVector<BranchInst*, 8> UncondBranchPreds;
808  for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
809  BasicBlock *Pred = *PI;
810  TerminatorInst *PTI = Pred->getTerminator();
811  if (BranchInst *BI = dyn_cast<BranchInst>(PTI))
812  if (BI->isUnconditional())
813  UncondBranchPreds.push_back(BI);
814  }
815 
816  while (!UncondBranchPreds.empty()) {
817  BranchInst *BI = UncondBranchPreds.pop_back_val();
818  BasicBlock *Pred = BI->getParent();
819  if (CallInst *CI = FindTRECandidate(BI, CannotTailCallElimCallsMarkedTail)){
820  DEBUG(dbgs() << "FOLDING: " << *BB
821  << "INTO UNCOND BRANCH PRED: " << *Pred);
822  ReturnInst *RI = FoldReturnIntoUncondBranch(Ret, BB, Pred);
823 
824  // Cleanup: if all predecessors of BB have been eliminated by
825  // FoldReturnIntoUncondBranch, delete it. It is important to empty it,
826  // because the ret instruction in there is still using a value which
827  // EliminateRecursiveTailCall will attempt to remove.
828  if (!BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB))
829  BB->eraseFromParent();
830 
831  EliminateRecursiveTailCall(CI, RI, OldEntry, TailCallsAreMarkedTail,
832  ArgumentPHIs,
833  CannotTailCallElimCallsMarkedTail);
834  ++NumRetDuped;
835  Change = true;
836  }
837  }
838 
839  return Change;
840 }
841 
842 bool
843 TailCallElim::ProcessReturningBlock(ReturnInst *Ret, BasicBlock *&OldEntry,
844  bool &TailCallsAreMarkedTail,
845  SmallVectorImpl<PHINode *> &ArgumentPHIs,
846  bool CannotTailCallElimCallsMarkedTail) {
847  CallInst *CI = FindTRECandidate(Ret, CannotTailCallElimCallsMarkedTail);
848  if (!CI)
849  return false;
850 
851  return EliminateRecursiveTailCall(CI, Ret, OldEntry, TailCallsAreMarkedTail,
852  ArgumentPHIs,
853  CannotTailCallElimCallsMarkedTail);
854 }
ReturnInst - Return a value (possibly void), from a function.
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
Definition: Instruction.cpp:70
BasicBlock * getUniquePredecessor()
Return the predecessor of this block if it has a unique predecessor block.
Definition: BasicBlock.cpp:224
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:223
LLVM Argument representation.
Definition: Argument.h:35
STATISTIC(NumFunctions,"Total number of functions")
iterator end()
Definition: Function.h:459
InstrTy * getInstruction() const
Definition: CallSite.h:82
unsigned getNumOperands() const
Definition: User.h:138
CallInst - This class represents a function call, abstracting a target machine's calling convention...
bool mayHaveSideEffects() const
mayHaveSideEffects - Return true if the instruction may have side effects.
Definition: Instruction.h:387
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:111
arg_iterator arg_end()
Definition: Function.h:480
const Instruction & front() const
Definition: BasicBlock.h:243
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:225
F(f)
LoadInst - an instruction for reading from memory.
Definition: Instructions.h:177
User::op_iterator arg_iterator
arg_iterator - The type of iterator to use when looping over actual arguments at this call site...
Definition: CallSite.h:147
void emitOptimizationRemark(LLVMContext &Ctx, const char *PassName, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg)
Emit an optimization-applied message.
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:231
Instruction * getFirstNonPHIOrDbg()
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic...
Definition: BasicBlock.cpp:172
IterTy arg_end() const
Definition: CallSite.h:157
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:70
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
Definition: SmallVector.h:406
A Use represents the edge between a Value definition and its users.
Definition: Use.h:69
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:75
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
unsigned getNumArgOperands() const
getNumArgOperands - Return the number of call arguments.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
Definition: BasicBlock.h:306
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:250
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:104
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
bool isAssociative() const
isAssociative - Return true if the instruction is associative:
static bool CanTRE(Function &F)
Scan the specified function for alloca instructions.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:351
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:256
iterator begin()
Definition: Function.h:457
Interval::succ_iterator succ_end(Interval *I)
Definition: Interval.h:107
#define P(N)
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:35
Wrapper pass for TargetTransformInfo.
void setDebugLoc(DebugLoc Loc)
setDebugLoc - Set the debug location information for this instruction.
Definition: Instruction.h:227
LLVM Basic Block Representation.
Definition: BasicBlock.h:65
BranchInst - Conditional or Unconditional Branch instruction.
FunctionPass * createTailCallEliminationPass()
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool isDynamicConstant(Value *V, CallInst *CI, ReturnInst *RI)
Return true if the specified value is the same when the return would exit as it was when the initial ...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:114
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:230
Represent the analysis usage information of a pass.
const InstListType & getInstList() const
Return the underlying instruction list container.
Definition: BasicBlock.h:252
bool doesNotAccessMemory() const
Determine if the call does not access memory.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:294
Value * getOperand(unsigned i) const
Definition: User.h:118
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:117
bool isCommutative() const
isCommutative - Return true if the instruction is commutative:
Definition: Instruction.h:327
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:103
arg_iterator arg_begin()
Definition: Function.h:472
Tail Call Elimination
void setTailCall(bool isTC=true)
iterator erase(iterator where)
Definition: ilist.h:465
static Value * getCommonReturnValue(ReturnInst *IgnoreRI, CallInst *CI)
Check to see if the function containing the specified tail call consistently returns the same runtime...
bool mayWriteToMemory() const
mayWriteToMemory - Return true if this instruction may modify memory.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
INITIALIZE_PASS_BEGIN(TailCallElim,"tailcallelim","Tail Call Elimination", false, false) INITIALIZE_PASS_END(TailCallElim
iterator end()
Definition: BasicBlock.h:233
static Instruction * FirstNonDbg(BasicBlock::iterator I)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
Instruction * user_back()
user_back - Specialize the methods defined in Value, as we know that an instruction can only be used ...
Definition: Instruction.h:69
Function * getCalledFunction() const
getCalledFunction - Return the function called, or null if this is an indirect function invocation...
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
const BasicBlock & getEntryBlock() const
Definition: Function.h:442
void setOperand(unsigned i, Value *Val)
Definition: User.h:122
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
void initializeTailCallElimPass(PassRegistry &)
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:372
iplist< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
Definition: BasicBlock.cpp:97
#define I(x, y, z)
Definition: MD5.cpp:54
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
FunctionType * getFunctionType() const
Definition: Function.cpp:227
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:311
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
Definition: Function.cpp:916
bool isTailCall() const
ReturnInst * FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB, BasicBlock *Pred)
FoldReturnIntoUncondBranch - This method duplicates the specified return instruction into a predecess...
Tail Call false
bool isVarArg() const
Definition: DerivedTypes.h:120
SwitchInst - Multiway switch.
iterator_range< op_iterator > arg_operands()
arg_operands - iteration adapter for range-for loops.
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:140
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:365
LLVM Value Representation.
Definition: Value.h:69
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:112
#define DEBUG(X)
Definition: Debug.h:92
bool isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom, unsigned Align)
isSafeToLoadUnconditionally - Return true if we know that executing a load from this value cannot tra...
Definition: Loads.cpp:65
IterTy arg_begin() const
arg_begin/arg_end - Return iterators corresponding to the actual argument list for a call site...
Definition: CallSite.h:151
Value * SimplifyInstruction(Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
SimplifyInstruction - See if we can compute a simplified version of this instruction.
This pass exposes codegen information to IR-level passes.
const BasicBlock * getParent() const
Definition: Instruction.h:72
bool onlyReadsMemory() const
Determine if the call does not access or only reads memory.
Definition: CallSite.h:286
iterator_range< arg_iterator > args()
Definition: Function.h:489
AllocaInst - an instruction to allocate memory on the stack.
Definition: Instructions.h:76