LLVM  6.0.0svn
TailRecursionElimination.cpp
Go to the documentation of this file.
1 //===- TailRecursionElimination.cpp - Eliminate Tail Calls ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file transforms calls of the current function (self recursion) followed
11 // by a return instruction with a branch to the entry of the function, creating
12 // a loop. This pass also implements the following extensions to the basic
13 // algorithm:
14 //
15 // 1. Trivial instructions between the call and return do not prevent the
16 // transformation from taking place, though currently the analysis cannot
17 // support moving any really useful instructions (only dead ones).
18 // 2. This pass transforms functions that are prevented from being tail
19 // recursive by an associative and commutative expression to use an
20 // accumulator variable, thus compiling the typical naive factorial or
21 // 'fib' implementation into efficient code.
22 // 3. TRE is performed if the function returns void, if the return
23 // returns the result returned by the call, or if the function returns a
24 // run-time constant on all exits from the function. It is possible, though
25 // unlikely, that the return returns something else (like constant 0), and
26 // can still be TRE'd. It can be TRE'd if ALL OTHER return instructions in
27 // the function return the exact same value.
28 // 4. If it can prove that callees do not access their caller stack frame,
29 // they are marked as eligible for tail call elimination (by the code
30 // generator).
31 //
32 // There are several improvements that could be made:
33 //
34 // 1. If the function has any alloca instructions, these instructions will be
35 // moved out of the entry block of the function, causing them to be
36 // evaluated each time through the tail recursion. Safely keeping allocas
37 // in the entry block requires analysis to proves that the tail-called
38 // function does not read or write the stack object.
39 // 2. Tail recursion is only performed if the call immediately precedes the
40 // return instruction. It's possible that there could be a jump between
41 // the call and the return.
42 // 3. There can be intervening operations between the call and the return that
43 // prevent the TRE from occurring. For example, there could be GEP's and
44 // stores to memory that will not be read or written by the call. This
45 // requires some substantial analysis (such as with DSA) to prove safe to
46 // move ahead of the call, but doing so could allow many more TREs to be
47 // performed, for example in TreeAdd/TreeAlloc from the treeadd benchmark.
48 // 4. The algorithm we use to detect if callees access their caller stack
49 // frames is very primitive.
50 //
51 //===----------------------------------------------------------------------===//
52 
54 #include "llvm/ADT/STLExtras.h"
55 #include "llvm/ADT/SmallPtrSet.h"
56 #include "llvm/ADT/Statistic.h"
57 #include "llvm/Analysis/CFG.h"
62 #include "llvm/Analysis/Loads.h"
65 #include "llvm/IR/CFG.h"
66 #include "llvm/IR/CallSite.h"
67 #include "llvm/IR/Constants.h"
68 #include "llvm/IR/DataLayout.h"
69 #include "llvm/IR/DerivedTypes.h"
70 #include "llvm/IR/DiagnosticInfo.h"
71 #include "llvm/IR/Function.h"
72 #include "llvm/IR/InstIterator.h"
73 #include "llvm/IR/Instructions.h"
74 #include "llvm/IR/IntrinsicInst.h"
75 #include "llvm/IR/Module.h"
76 #include "llvm/IR/ValueHandle.h"
77 #include "llvm/Pass.h"
78 #include "llvm/Support/Debug.h"
80 #include "llvm/Transforms/Scalar.h"
83 using namespace llvm;
84 
85 #define DEBUG_TYPE "tailcallelim"
86 
87 STATISTIC(NumEliminated, "Number of tail calls removed");
88 STATISTIC(NumRetDuped, "Number of return duplicated");
89 STATISTIC(NumAccumAdded, "Number of accumulators introduced");
90 
91 /// \brief Scan the specified function for alloca instructions.
92 /// If it contains any dynamic allocas, returns false.
93 static bool canTRE(Function &F) {
94  // Because of PR962, we don't TRE dynamic allocas.
95  return llvm::all_of(instructions(F), [](Instruction &I) {
96  auto *AI = dyn_cast<AllocaInst>(&I);
97  return !AI || AI->isStaticAlloca();
98  });
99 }
100 
101 namespace {
102 struct AllocaDerivedValueTracker {
103  // Start at a root value and walk its use-def chain to mark calls that use the
104  // value or a derived value in AllocaUsers, and places where it may escape in
105  // EscapePoints.
106  void walk(Value *Root) {
107  SmallVector<Use *, 32> Worklist;
108  SmallPtrSet<Use *, 32> Visited;
109 
110  auto AddUsesToWorklist = [&](Value *V) {
111  for (auto &U : V->uses()) {
112  if (!Visited.insert(&U).second)
113  continue;
114  Worklist.push_back(&U);
115  }
116  };
117 
118  AddUsesToWorklist(Root);
119 
120  while (!Worklist.empty()) {
121  Use *U = Worklist.pop_back_val();
122  Instruction *I = cast<Instruction>(U->getUser());
123 
124  switch (I->getOpcode()) {
125  case Instruction::Call:
126  case Instruction::Invoke: {
127  CallSite CS(I);
128  bool IsNocapture =
129  CS.isDataOperand(U) && CS.doesNotCapture(CS.getDataOperandNo(U));
130  callUsesLocalStack(CS, IsNocapture);
131  if (IsNocapture) {
132  // If the alloca-derived argument is passed in as nocapture, then it
133  // can't propagate to the call's return. That would be capturing.
134  continue;
135  }
136  break;
137  }
138  case Instruction::Load: {
139  // The result of a load is not alloca-derived (unless an alloca has
140  // otherwise escaped, but this is a local analysis).
141  continue;
142  }
143  case Instruction::Store: {
144  if (U->getOperandNo() == 0)
145  EscapePoints.insert(I);
146  continue; // Stores have no users to analyze.
147  }
148  case Instruction::BitCast:
149  case Instruction::GetElementPtr:
150  case Instruction::PHI:
151  case Instruction::Select:
152  case Instruction::AddrSpaceCast:
153  break;
154  default:
155  EscapePoints.insert(I);
156  break;
157  }
158 
159  AddUsesToWorklist(I);
160  }
161  }
162 
163  void callUsesLocalStack(CallSite CS, bool IsNocapture) {
164  // Add it to the list of alloca users.
165  AllocaUsers.insert(CS.getInstruction());
166 
167  // If it's nocapture then it can't capture this alloca.
168  if (IsNocapture)
169  return;
170 
171  // If it can write to memory, it can leak the alloca value.
172  if (!CS.onlyReadsMemory())
173  EscapePoints.insert(CS.getInstruction());
174  }
175 
176  SmallPtrSet<Instruction *, 32> AllocaUsers;
177  SmallPtrSet<Instruction *, 32> EscapePoints;
178 };
179 }
180 
181 static bool markTails(Function &F, bool &AllCallsAreTailCalls,
184  return false;
185  AllCallsAreTailCalls = true;
186 
187  // The local stack holds all alloca instructions and all byval arguments.
188  AllocaDerivedValueTracker Tracker;
189  for (Argument &Arg : F.args()) {
190  if (Arg.hasByValAttr())
191  Tracker.walk(&Arg);
192  }
193  for (auto &BB : F) {
194  for (auto &I : BB)
195  if (AllocaInst *AI = dyn_cast<AllocaInst>(&I))
196  Tracker.walk(AI);
197  }
198 
199  bool Modified = false;
200 
201  // Track whether a block is reachable after an alloca has escaped. Blocks that
202  // contain the escaping instruction will be marked as being visited without an
203  // escaped alloca, since that is how the block began.
204  enum VisitType {
205  UNVISITED,
206  UNESCAPED,
207  ESCAPED
208  };
210 
211  // We propagate the fact that an alloca has escaped from block to successor.
212  // Visit the blocks that are propagating the escapedness first. To do this, we
213  // maintain two worklists.
214  SmallVector<BasicBlock *, 32> WorklistUnescaped, WorklistEscaped;
215 
216  // We may enter a block and visit it thinking that no alloca has escaped yet,
217  // then see an escape point and go back around a loop edge and come back to
218  // the same block twice. Because of this, we defer setting tail on calls when
219  // we first encounter them in a block. Every entry in this list does not
220  // statically use an alloca via use-def chain analysis, but may find an alloca
221  // through other means if the block turns out to be reachable after an escape
222  // point.
223  SmallVector<CallInst *, 32> DeferredTails;
224 
225  BasicBlock *BB = &F.getEntryBlock();
226  VisitType Escaped = UNESCAPED;
227  do {
228  for (auto &I : *BB) {
229  if (Tracker.EscapePoints.count(&I))
230  Escaped = ESCAPED;
231 
232  CallInst *CI = dyn_cast<CallInst>(&I);
233  if (!CI || CI->isTailCall())
234  continue;
235 
236  bool IsNoTail = CI->isNoTailCall() || CI->hasOperandBundles();
237 
238  if (!IsNoTail && CI->doesNotAccessMemory()) {
239  // A call to a readnone function whose arguments are all things computed
240  // outside this function can be marked tail. Even if you stored the
241  // alloca address into a global, a readnone function can't load the
242  // global anyhow.
243  //
244  // Note that this runs whether we know an alloca has escaped or not. If
245  // it has, then we can't trust Tracker.AllocaUsers to be accurate.
246  bool SafeToTail = true;
247  for (auto &Arg : CI->arg_operands()) {
248  if (isa<Constant>(Arg.getUser()))
249  continue;
250  if (Argument *A = dyn_cast<Argument>(Arg.getUser()))
251  if (!A->hasByValAttr())
252  continue;
253  SafeToTail = false;
254  break;
255  }
256  if (SafeToTail) {
257  using namespace ore;
258  ORE->emit(OptimizationRemark(DEBUG_TYPE, "tailcall-readnone", CI)
259  << "marked as tail call candidate (readnone)");
260  CI->setTailCall();
261  Modified = true;
262  continue;
263  }
264  }
265 
266  if (!IsNoTail && Escaped == UNESCAPED && !Tracker.AllocaUsers.count(CI)) {
267  DeferredTails.push_back(CI);
268  } else {
269  AllCallsAreTailCalls = false;
270  }
271  }
272 
273  for (auto *SuccBB : make_range(succ_begin(BB), succ_end(BB))) {
274  auto &State = Visited[SuccBB];
275  if (State < Escaped) {
276  State = Escaped;
277  if (State == ESCAPED)
278  WorklistEscaped.push_back(SuccBB);
279  else
280  WorklistUnescaped.push_back(SuccBB);
281  }
282  }
283 
284  if (!WorklistEscaped.empty()) {
285  BB = WorklistEscaped.pop_back_val();
286  Escaped = ESCAPED;
287  } else {
288  BB = nullptr;
289  while (!WorklistUnescaped.empty()) {
290  auto *NextBB = WorklistUnescaped.pop_back_val();
291  if (Visited[NextBB] == UNESCAPED) {
292  BB = NextBB;
293  Escaped = UNESCAPED;
294  break;
295  }
296  }
297  }
298  } while (BB);
299 
300  for (CallInst *CI : DeferredTails) {
301  if (Visited[CI->getParent()] != ESCAPED) {
302  // If the escape point was part way through the block, calls after the
303  // escape point wouldn't have been put into DeferredTails.
304  ORE->emit(OptimizationRemark(DEBUG_TYPE, "tailcall", CI)
305  << "marked as tail call candidate");
306  CI->setTailCall();
307  Modified = true;
308  } else {
309  AllCallsAreTailCalls = false;
310  }
311  }
312 
313  return Modified;
314 }
315 
316 /// Return true if it is safe to move the specified
317 /// instruction from after the call to before the call, assuming that all
318 /// instructions between the call and this instruction are movable.
319 ///
321  // FIXME: We can move load/store/call/free instructions above the call if the
322  // call does not mod/ref the memory location being processed.
323  if (I->mayHaveSideEffects()) // This also handles volatile loads.
324  return false;
325 
326  if (LoadInst *L = dyn_cast<LoadInst>(I)) {
327  // Loads may always be moved above calls without side effects.
328  if (CI->mayHaveSideEffects()) {
329  // Non-volatile loads may be moved above a call with side effects if it
330  // does not write to memory and the load provably won't trap.
331  // Writes to memory only matter if they may alias the pointer
332  // being loaded from.
333  const DataLayout &DL = L->getModule()->getDataLayout();
334  if ((AA->getModRefInfo(CI, MemoryLocation::get(L)) & MRI_Mod) ||
335  !isSafeToLoadUnconditionally(L->getPointerOperand(),
336  L->getAlignment(), DL, L))
337  return false;
338  }
339  }
340 
341  // Otherwise, if this is a side-effect free instruction, check to make sure
342  // that it does not use the return value of the call. If it doesn't use the
343  // return value of the call, it must only use things that are defined before
344  // the call, or movable instructions between the call and the instruction
345  // itself.
346  return !is_contained(I->operands(), CI);
347 }
348 
349 /// Return true if the specified value is the same when the return would exit
350 /// as it was when the initial iteration of the recursive function was executed.
351 ///
352 /// We currently handle static constants and arguments that are not modified as
353 /// part of the recursion.
354 static bool isDynamicConstant(Value *V, CallInst *CI, ReturnInst *RI) {
355  if (isa<Constant>(V)) return true; // Static constants are always dyn consts
356 
357  // Check to see if this is an immutable argument, if so, the value
358  // will be available to initialize the accumulator.
359  if (Argument *Arg = dyn_cast<Argument>(V)) {
360  // Figure out which argument number this is...
361  unsigned ArgNo = 0;
362  Function *F = CI->getParent()->getParent();
363  for (Function::arg_iterator AI = F->arg_begin(); &*AI != Arg; ++AI)
364  ++ArgNo;
365 
366  // If we are passing this argument into call as the corresponding
367  // argument operand, then the argument is dynamically constant.
368  // Otherwise, we cannot transform this function safely.
369  if (CI->getArgOperand(ArgNo) == Arg)
370  return true;
371  }
372 
373  // Switch cases are always constant integers. If the value is being switched
374  // on and the return is only reachable from one of its cases, it's
375  // effectively constant.
376  if (BasicBlock *UniquePred = RI->getParent()->getUniquePredecessor())
377  if (SwitchInst *SI = dyn_cast<SwitchInst>(UniquePred->getTerminator()))
378  if (SI->getCondition() == V)
379  return SI->getDefaultDest() != RI->getParent();
380 
381  // Not a constant or immutable argument, we can't safely transform.
382  return false;
383 }
384 
385 /// Check to see if the function containing the specified tail call consistently
386 /// returns the same runtime-constant value at all exit points except for
387 /// IgnoreRI. If so, return the returned value.
389  Function *F = CI->getParent()->getParent();
390  Value *ReturnedValue = nullptr;
391 
392  for (BasicBlock &BBI : *F) {
393  ReturnInst *RI = dyn_cast<ReturnInst>(BBI.getTerminator());
394  if (RI == nullptr || RI == IgnoreRI) continue;
395 
396  // We can only perform this transformation if the value returned is
397  // evaluatable at the start of the initial invocation of the function,
398  // instead of at the end of the evaluation.
399  //
400  Value *RetOp = RI->getOperand(0);
401  if (!isDynamicConstant(RetOp, CI, RI))
402  return nullptr;
403 
404  if (ReturnedValue && RetOp != ReturnedValue)
405  return nullptr; // Cannot transform if differing values are returned.
406  ReturnedValue = RetOp;
407  }
408  return ReturnedValue;
409 }
410 
411 /// If the specified instruction can be transformed using accumulator recursion
412 /// elimination, return the constant which is the start of the accumulator
413 /// value. Otherwise return null.
415  if (!I->isAssociative() || !I->isCommutative()) return nullptr;
416  assert(I->getNumOperands() == 2 &&
417  "Associative/commutative operations should have 2 args!");
418 
419  // Exactly one operand should be the result of the call instruction.
420  if ((I->getOperand(0) == CI && I->getOperand(1) == CI) ||
421  (I->getOperand(0) != CI && I->getOperand(1) != CI))
422  return nullptr;
423 
424  // The only user of this instruction we allow is a single return instruction.
425  if (!I->hasOneUse() || !isa<ReturnInst>(I->user_back()))
426  return nullptr;
427 
428  // Ok, now we have to check all of the other return instructions in this
429  // function. If they return non-constants or differing values, then we cannot
430  // transform the function safely.
431  return getCommonReturnValue(cast<ReturnInst>(I->user_back()), CI);
432 }
433 
435  while (isa<DbgInfoIntrinsic>(I))
436  ++I;
437  return &*I;
438 }
439 
441  bool CannotTailCallElimCallsMarkedTail,
442  const TargetTransformInfo *TTI) {
443  BasicBlock *BB = TI->getParent();
444  Function *F = BB->getParent();
445 
446  if (&BB->front() == TI) // Make sure there is something before the terminator.
447  return nullptr;
448 
449  // Scan backwards from the return, checking to see if there is a tail call in
450  // this block. If so, set CI to it.
451  CallInst *CI = nullptr;
452  BasicBlock::iterator BBI(TI);
453  while (true) {
454  CI = dyn_cast<CallInst>(BBI);
455  if (CI && CI->getCalledFunction() == F)
456  break;
457 
458  if (BBI == BB->begin())
459  return nullptr; // Didn't find a potential tail call.
460  --BBI;
461  }
462 
463  // If this call is marked as a tail call, and if there are dynamic allocas in
464  // the function, we cannot perform this optimization.
465  if (CI->isTailCall() && CannotTailCallElimCallsMarkedTail)
466  return nullptr;
467 
468  // As a special case, detect code like this:
469  // double fabs(double f) { return __builtin_fabs(f); } // a 'fabs' call
470  // and disable this xform in this case, because the code generator will
471  // lower the call to fabs into inline code.
472  if (BB == &F->getEntryBlock() &&
473  firstNonDbg(BB->front().getIterator()) == CI &&
474  firstNonDbg(std::next(BB->begin())) == TI && CI->getCalledFunction() &&
475  !TTI->isLoweredToCall(CI->getCalledFunction())) {
476  // A single-block function with just a call and a return. Check that
477  // the arguments match.
479  E = CallSite(CI).arg_end();
481  FE = F->arg_end();
482  for (; I != E && FI != FE; ++I, ++FI)
483  if (*I != &*FI) break;
484  if (I == E && FI == FE)
485  return nullptr;
486  }
487 
488  return CI;
489 }
490 
492  BasicBlock *&OldEntry,
493  bool &TailCallsAreMarkedTail,
494  SmallVectorImpl<PHINode *> &ArgumentPHIs,
495  AliasAnalysis *AA,
497  // If we are introducing accumulator recursion to eliminate operations after
498  // the call instruction that are both associative and commutative, the initial
499  // value for the accumulator is placed in this variable. If this value is set
500  // then we actually perform accumulator recursion elimination instead of
501  // simple tail recursion elimination. If the operation is an LLVM instruction
502  // (eg: "add") then it is recorded in AccumulatorRecursionInstr. If not, then
503  // we are handling the case when the return instruction returns a constant C
504  // which is different to the constant returned by other return instructions
505  // (which is recorded in AccumulatorRecursionEliminationInitVal). This is a
506  // special case of accumulator recursion, the operation being "return C".
507  Value *AccumulatorRecursionEliminationInitVal = nullptr;
508  Instruction *AccumulatorRecursionInstr = nullptr;
509 
510  // Ok, we found a potential tail call. We can currently only transform the
511  // tail call if all of the instructions between the call and the return are
512  // movable to above the call itself, leaving the call next to the return.
513  // Check that this is the case now.
514  BasicBlock::iterator BBI(CI);
515  for (++BBI; &*BBI != Ret; ++BBI) {
516  if (canMoveAboveCall(&*BBI, CI, AA))
517  continue;
518 
519  // If we can't move the instruction above the call, it might be because it
520  // is an associative and commutative operation that could be transformed
521  // using accumulator recursion elimination. Check to see if this is the
522  // case, and if so, remember the initial accumulator value for later.
523  if ((AccumulatorRecursionEliminationInitVal =
524  canTransformAccumulatorRecursion(&*BBI, CI))) {
525  // Yes, this is accumulator recursion. Remember which instruction
526  // accumulates.
527  AccumulatorRecursionInstr = &*BBI;
528  } else {
529  return false; // Otherwise, we cannot eliminate the tail recursion!
530  }
531  }
532 
533  // We can only transform call/return pairs that either ignore the return value
534  // of the call and return void, ignore the value of the call and return a
535  // constant, return the value returned by the tail call, or that are being
536  // accumulator recursion variable eliminated.
537  if (Ret->getNumOperands() == 1 && Ret->getReturnValue() != CI &&
538  !isa<UndefValue>(Ret->getReturnValue()) &&
539  AccumulatorRecursionEliminationInitVal == nullptr &&
540  !getCommonReturnValue(nullptr, CI)) {
541  // One case remains that we are able to handle: the current return
542  // instruction returns a constant, and all other return instructions
543  // return a different constant.
544  if (!isDynamicConstant(Ret->getReturnValue(), CI, Ret))
545  return false; // Current return instruction does not return a constant.
546  // Check that all other return instructions return a common constant. If
547  // so, record it in AccumulatorRecursionEliminationInitVal.
548  AccumulatorRecursionEliminationInitVal = getCommonReturnValue(Ret, CI);
549  if (!AccumulatorRecursionEliminationInitVal)
550  return false;
551  }
552 
553  BasicBlock *BB = Ret->getParent();
554  Function *F = BB->getParent();
555 
556  using namespace ore;
557  ORE->emit(OptimizationRemark(DEBUG_TYPE, "tailcall-recursion", CI)
558  << "transforming tail recursion into loop");
559 
560  // OK! We can transform this tail call. If this is the first one found,
561  // create the new entry block, allowing us to branch back to the old entry.
562  if (!OldEntry) {
563  OldEntry = &F->getEntryBlock();
564  BasicBlock *NewEntry = BasicBlock::Create(F->getContext(), "", F, OldEntry);
565  NewEntry->takeName(OldEntry);
566  OldEntry->setName("tailrecurse");
567  BranchInst::Create(OldEntry, NewEntry);
568 
569  // If this tail call is marked 'tail' and if there are any allocas in the
570  // entry block, move them up to the new entry block.
571  TailCallsAreMarkedTail = CI->isTailCall();
572  if (TailCallsAreMarkedTail)
573  // Move all fixed sized allocas from OldEntry to NewEntry.
574  for (BasicBlock::iterator OEBI = OldEntry->begin(), E = OldEntry->end(),
575  NEBI = NewEntry->begin(); OEBI != E; )
576  if (AllocaInst *AI = dyn_cast<AllocaInst>(OEBI++))
577  if (isa<ConstantInt>(AI->getArraySize()))
578  AI->moveBefore(&*NEBI);
579 
580  // Now that we have created a new block, which jumps to the entry
581  // block, insert a PHI node for each argument of the function.
582  // For now, we initialize each PHI to only have the real arguments
583  // which are passed in.
584  Instruction *InsertPos = &OldEntry->front();
585  for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end();
586  I != E; ++I) {
587  PHINode *PN = PHINode::Create(I->getType(), 2,
588  I->getName() + ".tr", InsertPos);
589  I->replaceAllUsesWith(PN); // Everyone use the PHI node now!
590  PN->addIncoming(&*I, NewEntry);
591  ArgumentPHIs.push_back(PN);
592  }
593  }
594 
595  // If this function has self recursive calls in the tail position where some
596  // are marked tail and some are not, only transform one flavor or another. We
597  // have to choose whether we move allocas in the entry block to the new entry
598  // block or not, so we can't make a good choice for both. NOTE: We could do
599  // slightly better here in the case that the function has no entry block
600  // allocas.
601  if (TailCallsAreMarkedTail && !CI->isTailCall())
602  return false;
603 
604  // Ok, now that we know we have a pseudo-entry block WITH all of the
605  // required PHI nodes, add entries into the PHI node for the actual
606  // parameters passed into the tail-recursive call.
607  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i)
608  ArgumentPHIs[i]->addIncoming(CI->getArgOperand(i), BB);
609 
610  // If we are introducing an accumulator variable to eliminate the recursion,
611  // do so now. Note that we _know_ that no subsequent tail recursion
612  // eliminations will happen on this function because of the way the
613  // accumulator recursion predicate is set up.
614  //
615  if (AccumulatorRecursionEliminationInitVal) {
616  Instruction *AccRecInstr = AccumulatorRecursionInstr;
617  // Start by inserting a new PHI node for the accumulator.
618  pred_iterator PB = pred_begin(OldEntry), PE = pred_end(OldEntry);
619  PHINode *AccPN = PHINode::Create(
620  AccumulatorRecursionEliminationInitVal->getType(),
621  std::distance(PB, PE) + 1, "accumulator.tr", &OldEntry->front());
622 
623  // Loop over all of the predecessors of the tail recursion block. For the
624  // real entry into the function we seed the PHI with the initial value,
625  // computed earlier. For any other existing branches to this block (due to
626  // other tail recursions eliminated) the accumulator is not modified.
627  // Because we haven't added the branch in the current block to OldEntry yet,
628  // it will not show up as a predecessor.
629  for (pred_iterator PI = PB; PI != PE; ++PI) {
630  BasicBlock *P = *PI;
631  if (P == &F->getEntryBlock())
632  AccPN->addIncoming(AccumulatorRecursionEliminationInitVal, P);
633  else
634  AccPN->addIncoming(AccPN, P);
635  }
636 
637  if (AccRecInstr) {
638  // Add an incoming argument for the current block, which is computed by
639  // our associative and commutative accumulator instruction.
640  AccPN->addIncoming(AccRecInstr, BB);
641 
642  // Next, rewrite the accumulator recursion instruction so that it does not
643  // use the result of the call anymore, instead, use the PHI node we just
644  // inserted.
645  AccRecInstr->setOperand(AccRecInstr->getOperand(0) != CI, AccPN);
646  } else {
647  // Add an incoming argument for the current block, which is just the
648  // constant returned by the current return instruction.
649  AccPN->addIncoming(Ret->getReturnValue(), BB);
650  }
651 
652  // Finally, rewrite any return instructions in the program to return the PHI
653  // node instead of the "initval" that they do currently. This loop will
654  // actually rewrite the return value we are destroying, but that's ok.
655  for (BasicBlock &BBI : *F)
656  if (ReturnInst *RI = dyn_cast<ReturnInst>(BBI.getTerminator()))
657  RI->setOperand(0, AccPN);
658  ++NumAccumAdded;
659  }
660 
661  // Now that all of the PHI nodes are in place, remove the call and
662  // ret instructions, replacing them with an unconditional branch.
663  BranchInst *NewBI = BranchInst::Create(OldEntry, Ret);
664  NewBI->setDebugLoc(CI->getDebugLoc());
665 
666  BB->getInstList().erase(Ret); // Remove return.
667  BB->getInstList().erase(CI); // Remove call.
668  ++NumEliminated;
669  return true;
670 }
671 
673  BasicBlock *BB, ReturnInst *Ret, BasicBlock *&OldEntry,
674  bool &TailCallsAreMarkedTail, SmallVectorImpl<PHINode *> &ArgumentPHIs,
675  bool CannotTailCallElimCallsMarkedTail, const TargetTransformInfo *TTI,
677  bool Change = false;
678 
679  // Make sure this block is a trivial return block.
680  assert(BB->getFirstNonPHIOrDbg() == Ret &&
681  "Trying to fold non-trivial return block");
682 
683  // If the return block contains nothing but the return and PHI's,
684  // there might be an opportunity to duplicate the return in its
685  // predecessors and perform TRE there. Look for predecessors that end
686  // in unconditional branch and recursive call(s).
687  SmallVector<BranchInst*, 8> UncondBranchPreds;
688  for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
689  BasicBlock *Pred = *PI;
690  TerminatorInst *PTI = Pred->getTerminator();
691  if (BranchInst *BI = dyn_cast<BranchInst>(PTI))
692  if (BI->isUnconditional())
693  UncondBranchPreds.push_back(BI);
694  }
695 
696  while (!UncondBranchPreds.empty()) {
697  BranchInst *BI = UncondBranchPreds.pop_back_val();
698  BasicBlock *Pred = BI->getParent();
699  if (CallInst *CI = findTRECandidate(BI, CannotTailCallElimCallsMarkedTail, TTI)){
700  DEBUG(dbgs() << "FOLDING: " << *BB
701  << "INTO UNCOND BRANCH PRED: " << *Pred);
702  ReturnInst *RI = FoldReturnIntoUncondBranch(Ret, BB, Pred);
703 
704  // Cleanup: if all predecessors of BB have been eliminated by
705  // FoldReturnIntoUncondBranch, delete it. It is important to empty it,
706  // because the ret instruction in there is still using a value which
707  // eliminateRecursiveTailCall will attempt to remove.
708  if (!BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB))
709  BB->eraseFromParent();
710 
711  eliminateRecursiveTailCall(CI, RI, OldEntry, TailCallsAreMarkedTail,
712  ArgumentPHIs, AA, ORE);
713  ++NumRetDuped;
714  Change = true;
715  }
716  }
717 
718  return Change;
719 }
720 
722  bool &TailCallsAreMarkedTail,
723  SmallVectorImpl<PHINode *> &ArgumentPHIs,
724  bool CannotTailCallElimCallsMarkedTail,
725  const TargetTransformInfo *TTI,
726  AliasAnalysis *AA,
728  CallInst *CI = findTRECandidate(Ret, CannotTailCallElimCallsMarkedTail, TTI);
729  if (!CI)
730  return false;
731 
732  return eliminateRecursiveTailCall(CI, Ret, OldEntry, TailCallsAreMarkedTail,
733  ArgumentPHIs, AA, ORE);
734 }
735 
737  AliasAnalysis *AA,
739  if (F.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
740  return false;
741 
742  bool MadeChange = false;
743  bool AllCallsAreTailCalls = false;
744  MadeChange |= markTails(F, AllCallsAreTailCalls, ORE);
745  if (!AllCallsAreTailCalls)
746  return MadeChange;
747 
748  // If this function is a varargs function, we won't be able to PHI the args
749  // right, so don't even try to convert it...
750  if (F.getFunctionType()->isVarArg())
751  return false;
752 
753  BasicBlock *OldEntry = nullptr;
754  bool TailCallsAreMarkedTail = false;
755  SmallVector<PHINode*, 8> ArgumentPHIs;
756 
757  // If false, we cannot perform TRE on tail calls marked with the 'tail'
758  // attribute, because doing so would cause the stack size to increase (real
759  // TRE would deallocate variable sized allocas, TRE doesn't).
760  bool CanTRETailMarkedCall = canTRE(F);
761 
762  // Change any tail recursive calls to loops.
763  //
764  // FIXME: The code generator produces really bad code when an 'escaping
765  // alloca' is changed from being a static alloca to being a dynamic alloca.
766  // Until this is resolved, disable this transformation if that would ever
767  // happen. This bug is PR962.
768  for (Function::iterator BBI = F.begin(), E = F.end(); BBI != E; /*in loop*/) {
769  BasicBlock *BB = &*BBI++; // foldReturnAndProcessPred may delete BB.
770  if (ReturnInst *Ret = dyn_cast<ReturnInst>(BB->getTerminator())) {
771  bool Change = processReturningBlock(Ret, OldEntry, TailCallsAreMarkedTail,
772  ArgumentPHIs, !CanTRETailMarkedCall,
773  TTI, AA, ORE);
774  if (!Change && BB->getFirstNonPHIOrDbg() == Ret)
775  Change = foldReturnAndProcessPred(BB, Ret, OldEntry,
776  TailCallsAreMarkedTail, ArgumentPHIs,
777  !CanTRETailMarkedCall, TTI, AA, ORE);
778  MadeChange |= Change;
779  }
780  }
781 
782  // If we eliminated any tail recursions, it's possible that we inserted some
783  // silly PHI nodes which just merge an initial value (the incoming operand)
784  // with themselves. Check to see if we did and clean up our mess if so. This
785  // occurs when a function passes an argument straight through to its tail
786  // call.
787  for (PHINode *PN : ArgumentPHIs) {
788  // If the PHI Node is a dynamic constant, replace it with the value it is.
789  if (Value *PNV = SimplifyInstruction(PN, F.getParent()->getDataLayout())) {
790  PN->replaceAllUsesWith(PNV);
791  PN->eraseFromParent();
792  }
793  }
794 
795  return MadeChange;
796 }
797 
798 namespace {
799 struct TailCallElim : public FunctionPass {
800  static char ID; // Pass identification, replacement for typeid
801  TailCallElim() : FunctionPass(ID) {
803  }
804 
805  void getAnalysisUsage(AnalysisUsage &AU) const override {
810  }
811 
812  bool runOnFunction(Function &F) override {
813  if (skipFunction(F))
814  return false;
815 
816  return eliminateTailRecursion(
817  F, &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F),
818  &getAnalysis<AAResultsWrapperPass>().getAAResults(),
819  &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE());
820  }
821 };
822 }
823 
824 char TailCallElim::ID = 0;
825 INITIALIZE_PASS_BEGIN(TailCallElim, "tailcallelim", "Tail Call Elimination",
826  false, false)
829 INITIALIZE_PASS_END(TailCallElim, "tailcallelim", "Tail Call Elimination",
830  false, false)
831 
832 // Public interface to the TailCallElimination pass
834  return new TailCallElim();
835 }
836 
839 
841  AliasAnalysis &AA = AM.getResult<AAManager>(F);
843 
844  bool Changed = eliminateTailRecursion(F, &TTI, &AA, &ORE);
845 
846  if (!Changed)
847  return PreservedAnalyses::all();
849  PA.preserve<GlobalsAA>();
850  return PA;
851 }
Legacy wrapper pass to provide the GlobalsAAResult object.
bool hasOperandBundles() const
Return true if this User has any operand bundles.
Definition: InstrTypes.h:1305
Return a value (possibly void), from a function.
User::op_iterator arg_iterator
The type of iterator to use when looping over actual arguments at this call site. ...
Definition: CallSite.h:210
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:109
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
This class represents an incoming formal argument to a Function.
Definition: Argument.h:30
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:687
iterator erase(iterator where)
Definition: ilist.h:280
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
This is the interface for a simple mod/ref and alias analysis over globals.
iterator end()
Definition: Function.h:582
This class represents a function call, abstracting a target machine&#39;s calling convention.
Analysis pass providing the TargetTransformInfo.
static bool eliminateRecursiveTailCall(CallInst *CI, ReturnInst *Ret, BasicBlock *&OldEntry, bool &TailCallsAreMarkedTail, SmallVectorImpl< PHINode *> &ArgumentPHIs, AliasAnalysis *AA, OptimizationRemarkEmitter *ORE)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:818
arg_iterator arg_end()
Definition: Function.h:604
STATISTIC(NumFunctions, "Total number of functions")
An instruction for reading from memory.
Definition: Instructions.h:164
bool isSafeToLoadUnconditionally(Value *V, unsigned Align, const DataLayout &DL, Instruction *ScanFrom=nullptr, const DominatorTree *DT=nullptr)
Return true if we know that executing a load from this value cannot trap.
Definition: Loads.cpp:201
The access modifies the value stored in memory.
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:252
bool onlyReadsMemory() const
Determine if the call does not access or only reads memory.
Definition: CallSite.h:446
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:51
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:361
unsigned getNumArgOperands() const
Return the number of call arguments.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
IterTy arg_end() const
Definition: CallSite.h:549
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
InstrTy * getInstruction() const
Definition: CallSite.h:89
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:284
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:106
#define F(x, y, z)
Definition: MD5.cpp:55
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
Definition: BasicBlock.cpp:230
bool isVarArg() const
Definition: DerivedTypes.h:123
unsigned getDataOperandNo(Value::const_user_iterator UI) const
Given a value use iterator, return the data operand corresponding to it.
Definition: CallSite.h:220
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:121
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:290
iterator begin()
Definition: Function.h:580
Value * getOperand(unsigned i) const
Definition: User.h:154
Interval::succ_iterator succ_end(Interval *I)
Definition: Interval.h:109
const BasicBlock & getEntryBlock() const
Definition: Function.h:564
#define P(N)
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:54
Wrapper pass for TargetTransformInfo.
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:153
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:277
LLVM Basic Block Representation.
Definition: BasicBlock.h:59
Conditional or Unconditional Branch instruction.
FunctionPass * createTailCallEliminationPass()
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const Instruction & front() const
Definition: BasicBlock.h:264
#define DEBUG_TYPE
static bool isDynamicConstant(Value *V, CallInst *CI, ReturnInst *RI)
Return true if the specified value is the same when the return would exit as it was when the initial ...
static bool processReturningBlock(ReturnInst *Ret, BasicBlock *&OldEntry, bool &TailCallsAreMarkedTail, SmallVectorImpl< PHINode *> &ArgumentPHIs, bool CannotTailCallElimCallsMarkedTail, const TargetTransformInfo *TTI, AliasAnalysis *AA, OptimizationRemarkEmitter *ORE)
A manager for alias analyses.
#define A
Definition: LargeTest.cpp:12
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:372
bool mayHaveSideEffects() const
Return true if the instruction may have side effects.
Definition: Instruction.h:500
Diagnostic information for applied optimization remarks.
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:116
bool isAssociative() const LLVM_READONLY
Return true if the instruction is associative:
Represent the analysis usage information of a pass.
Analysis pass providing a never-invalidated alias analysis result.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:119
op_range operands()
Definition: User.h:222
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:101
arg_iterator arg_begin()
Definition: Function.h:595
self_iterator getIterator()
Definition: ilist_node.h:82
bool isDataOperand(Value::const_user_iterator UI) const
Determine whether the passed iterator points to a data operand.
Definition: CallSite.h:174
Tail Call Elimination
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:194
bool doesNotCapture(unsigned OpNo) const
Determine whether this data operand is not captured.
Definition: CallSite.h:567
void setTailCall(bool isTC=true)
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:159
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
static Instruction * firstNonDbg(BasicBlock::iterator I)
static Value * getCommonReturnValue(ReturnInst *IgnoreRI, CallInst *CI)
Check to see if the function containing the specified tail call consistently returns the same runtime...
static CallInst * findTRECandidate(Instruction *TI, bool CannotTailCallElimCallsMarkedTail, const TargetTransformInfo *TTI)
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
Definition: BasicBlock.h:376
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
const InstListType & getInstList() const
Return the underlying instruction list container.
Definition: BasicBlock.h:317
Iterator for intrusive lists based on ilist_node.
unsigned getNumOperands() const
Definition: User.h:176
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:423
#define E
Definition: LargeTest.cpp:27
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file. ...
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
iterator end()
Definition: BasicBlock.h:254
IterTy arg_begin() const
Definition: CallSite.h:545
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:864
Module.h This file contains the declarations for the Module class.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:63
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:385
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
bool isCommutative() const
Return true if the instruction is commutative:
Definition: Instruction.h:416
void setOperand(unsigned i, Value *Val)
Definition: User.h:159
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
static bool foldReturnAndProcessPred(BasicBlock *BB, ReturnInst *Ret, BasicBlock *&OldEntry, bool &TailCallsAreMarkedTail, SmallVectorImpl< PHINode *> &ArgumentPHIs, bool CannotTailCallElimCallsMarkedTail, const TargetTransformInfo *TTI, AliasAnalysis *AA, OptimizationRemarkEmitter *ORE)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:137
void initializeTailCallElimPass(PassRegistry &)
ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc)
getModRefInfo (for call sites) - Return information about whether a particular call site modifies or ...
static bool canTRE(Function &F)
Scan the specified function for alloca instructions.
bool isTailCall() const
INITIALIZE_PASS_BEGIN(TailCallElim, "tailcallelim", "Tail Call Elimination", false, false) INITIALIZE_PASS_END(TailCallElim
static bool canMoveAboveCall(Instruction *I, CallInst *CI, AliasAnalysis *AA)
Return true if it is safe to move the specified instruction from after the call to before the call...
Function * getCalledFunction() const
Return the function called, or null if this is an indirect function invocation.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:280
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:61
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:195
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:108
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink &#39;this&#39; from the containing function and delete it.
Definition: BasicBlock.cpp:97
#define I(x, y, z)
Definition: MD5.cpp:58
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
ReturnInst * FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB, BasicBlock *Pred)
This method duplicates the specified return instruction into a predecessor which ends in an unconditi...
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:174
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
Definition: Function.cpp:1244
Multiway switch.
iterator_range< op_iterator > arg_operands()
Iteration adapter for range-for loops.
static Value * canTransformAccumulatorRecursion(Instruction *I, CallInst *CI)
If the specified instruction can be transformed using accumulator recursion elimination, return the constant which is the start of the accumulator value.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:545
LLVM Value Representation.
Definition: Value.h:73
bool isLoweredToCall(const Function *F) const
Test whether calls to a function lower to actual program function calls.
OptimizationRemarkEmitter legacy analysis pass.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:262
bool doesNotAccessMemory() const
Determine if the call does not access memory.
#define DEBUG(X)
Definition: Debug.h:118
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:408
bool isNoTailCall() const
static bool eliminateTailRecursion(Function &F, const TargetTransformInfo *TTI, AliasAnalysis *AA, OptimizationRemarkEmitter *ORE)
inst_range instructions(Function *F)
Definition: InstIterator.h:134
A container for analyses that lazily runs them and caches their results.
const Instruction * getFirstNonPHIOrDbg() const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic...
Definition: BasicBlock.cpp:178
This pass exposes codegen information to IR-level passes.
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size...
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
const TerminatorInst * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:120
static bool markTails(Function &F, bool &AllCallsAreTailCalls, OptimizationRemarkEmitter *ORE)
Value * SimplifyInstruction(Instruction *I, const SimplifyQuery &Q, OptimizationRemarkEmitter *ORE=nullptr)
See if we can compute a simplified version of this instruction.
The optimization diagnostic interface.
iterator_range< arg_iterator > args()
Definition: Function.h:613
const BasicBlock * getParent() const
Definition: Instruction.h:66
an instruction to allocate memory on the stack
Definition: Instructions.h:60
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:872