LLVM  7.0.0svn
InlineFunction.cpp
Go to the documentation of this file.
1 //===- InlineFunction.cpp - Code to perform function inlining -------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements inlining of a function into a call site, resolving
11 // parameters and the return value as appropriate.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/ADT/DenseMap.h"
16 #include "llvm/ADT/None.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SetVector.h"
20 #include "llvm/ADT/SmallPtrSet.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/StringExtras.h"
34 #include "llvm/IR/Argument.h"
35 #include "llvm/IR/BasicBlock.h"
36 #include "llvm/IR/CFG.h"
37 #include "llvm/IR/CallSite.h"
38 #include "llvm/IR/Constant.h"
39 #include "llvm/IR/Constants.h"
40 #include "llvm/IR/DIBuilder.h"
41 #include "llvm/IR/DataLayout.h"
43 #include "llvm/IR/DebugLoc.h"
44 #include "llvm/IR/DerivedTypes.h"
45 #include "llvm/IR/Dominators.h"
46 #include "llvm/IR/Function.h"
47 #include "llvm/IR/IRBuilder.h"
48 #include "llvm/IR/InstrTypes.h"
49 #include "llvm/IR/Instruction.h"
50 #include "llvm/IR/Instructions.h"
51 #include "llvm/IR/IntrinsicInst.h"
52 #include "llvm/IR/Intrinsics.h"
53 #include "llvm/IR/LLVMContext.h"
54 #include "llvm/IR/MDBuilder.h"
55 #include "llvm/IR/Metadata.h"
56 #include "llvm/IR/Module.h"
57 #include "llvm/IR/Type.h"
58 #include "llvm/IR/User.h"
59 #include "llvm/IR/Value.h"
60 #include "llvm/Support/Casting.h"
65 #include <algorithm>
66 #include <cassert>
67 #include <cstdint>
68 #include <iterator>
69 #include <limits>
70 #include <string>
71 #include <utility>
72 #include <vector>
73 
74 using namespace llvm;
76 
77 static cl::opt<bool>
78 EnableNoAliasConversion("enable-noalias-to-md-conversion", cl::init(true),
79  cl::Hidden,
80  cl::desc("Convert noalias attributes to metadata during inlining."));
81 
82 static cl::opt<bool>
83 PreserveAlignmentAssumptions("preserve-alignment-assumptions-during-inlining",
84  cl::init(true), cl::Hidden,
85  cl::desc("Convert align attributes to assumptions during inlining."));
86 
88  AAResults *CalleeAAR, bool InsertLifetime) {
89  return InlineFunction(CallSite(CI), IFI, CalleeAAR, InsertLifetime);
90 }
91 
93  AAResults *CalleeAAR, bool InsertLifetime) {
94  return InlineFunction(CallSite(II), IFI, CalleeAAR, InsertLifetime);
95 }
96 
97 namespace {
98 
99  /// A class for recording information about inlining a landing pad.
100  class LandingPadInliningInfo {
101  /// Destination of the invoke's unwind.
102  BasicBlock *OuterResumeDest;
103 
104  /// Destination for the callee's resume.
105  BasicBlock *InnerResumeDest = nullptr;
106 
107  /// LandingPadInst associated with the invoke.
108  LandingPadInst *CallerLPad = nullptr;
109 
110  /// PHI for EH values from landingpad insts.
111  PHINode *InnerEHValuesPHI = nullptr;
112 
113  SmallVector<Value*, 8> UnwindDestPHIValues;
114 
115  public:
116  LandingPadInliningInfo(InvokeInst *II)
117  : OuterResumeDest(II->getUnwindDest()) {
118  // If there are PHI nodes in the unwind destination block, we need to keep
119  // track of which values came into them from the invoke before removing
120  // the edge from this block.
121  BasicBlock *InvokeBB = II->getParent();
122  BasicBlock::iterator I = OuterResumeDest->begin();
123  for (; isa<PHINode>(I); ++I) {
124  // Save the value to use for this edge.
125  PHINode *PHI = cast<PHINode>(I);
126  UnwindDestPHIValues.push_back(PHI->getIncomingValueForBlock(InvokeBB));
127  }
128 
129  CallerLPad = cast<LandingPadInst>(I);
130  }
131 
132  /// The outer unwind destination is the target of
133  /// unwind edges introduced for calls within the inlined function.
134  BasicBlock *getOuterResumeDest() const {
135  return OuterResumeDest;
136  }
137 
138  BasicBlock *getInnerResumeDest();
139 
140  LandingPadInst *getLandingPadInst() const { return CallerLPad; }
141 
142  /// Forward the 'resume' instruction to the caller's landing pad block.
143  /// When the landing pad block has only one predecessor, this is
144  /// a simple branch. When there is more than one predecessor, we need to
145  /// split the landing pad block after the landingpad instruction and jump
146  /// to there.
147  void forwardResume(ResumeInst *RI,
148  SmallPtrSetImpl<LandingPadInst*> &InlinedLPads);
149 
150  /// Add incoming-PHI values to the unwind destination block for the given
151  /// basic block, using the values for the original invoke's source block.
152  void addIncomingPHIValuesFor(BasicBlock *BB) const {
153  addIncomingPHIValuesForInto(BB, OuterResumeDest);
154  }
155 
156  void addIncomingPHIValuesForInto(BasicBlock *src, BasicBlock *dest) const {
157  BasicBlock::iterator I = dest->begin();
158  for (unsigned i = 0, e = UnwindDestPHIValues.size(); i != e; ++i, ++I) {
159  PHINode *phi = cast<PHINode>(I);
160  phi->addIncoming(UnwindDestPHIValues[i], src);
161  }
162  }
163  };
164 
165 } // end anonymous namespace
166 
167 /// Get or create a target for the branch from ResumeInsts.
168 BasicBlock *LandingPadInliningInfo::getInnerResumeDest() {
169  if (InnerResumeDest) return InnerResumeDest;
170 
171  // Split the landing pad.
172  BasicBlock::iterator SplitPoint = ++CallerLPad->getIterator();
173  InnerResumeDest =
174  OuterResumeDest->splitBasicBlock(SplitPoint,
175  OuterResumeDest->getName() + ".body");
176 
177  // The number of incoming edges we expect to the inner landing pad.
178  const unsigned PHICapacity = 2;
179 
180  // Create corresponding new PHIs for all the PHIs in the outer landing pad.
181  Instruction *InsertPoint = &InnerResumeDest->front();
182  BasicBlock::iterator I = OuterResumeDest->begin();
183  for (unsigned i = 0, e = UnwindDestPHIValues.size(); i != e; ++i, ++I) {
184  PHINode *OuterPHI = cast<PHINode>(I);
185  PHINode *InnerPHI = PHINode::Create(OuterPHI->getType(), PHICapacity,
186  OuterPHI->getName() + ".lpad-body",
187  InsertPoint);
188  OuterPHI->replaceAllUsesWith(InnerPHI);
189  InnerPHI->addIncoming(OuterPHI, OuterResumeDest);
190  }
191 
192  // Create a PHI for the exception values.
193  InnerEHValuesPHI = PHINode::Create(CallerLPad->getType(), PHICapacity,
194  "eh.lpad-body", InsertPoint);
195  CallerLPad->replaceAllUsesWith(InnerEHValuesPHI);
196  InnerEHValuesPHI->addIncoming(CallerLPad, OuterResumeDest);
197 
198  // All done.
199  return InnerResumeDest;
200 }
201 
202 /// Forward the 'resume' instruction to the caller's landing pad block.
203 /// When the landing pad block has only one predecessor, this is a simple
204 /// branch. When there is more than one predecessor, we need to split the
205 /// landing pad block after the landingpad instruction and jump to there.
206 void LandingPadInliningInfo::forwardResume(
207  ResumeInst *RI, SmallPtrSetImpl<LandingPadInst *> &InlinedLPads) {
208  BasicBlock *Dest = getInnerResumeDest();
209  BasicBlock *Src = RI->getParent();
210 
211  BranchInst::Create(Dest, Src);
212 
213  // Update the PHIs in the destination. They were inserted in an order which
214  // makes this work.
215  addIncomingPHIValuesForInto(Src, Dest);
216 
217  InnerEHValuesPHI->addIncoming(RI->getOperand(0), Src);
218  RI->eraseFromParent();
219 }
220 
221 /// Helper for getUnwindDestToken/getUnwindDestTokenHelper.
222 static Value *getParentPad(Value *EHPad) {
223  if (auto *FPI = dyn_cast<FuncletPadInst>(EHPad))
224  return FPI->getParentPad();
225  return cast<CatchSwitchInst>(EHPad)->getParentPad();
226 }
227 
229 
230 /// Helper for getUnwindDestToken that does the descendant-ward part of
231 /// the search.
233  UnwindDestMemoTy &MemoMap) {
234  SmallVector<Instruction *, 8> Worklist(1, EHPad);
235 
236  while (!Worklist.empty()) {
237  Instruction *CurrentPad = Worklist.pop_back_val();
238  // We only put pads on the worklist that aren't in the MemoMap. When
239  // we find an unwind dest for a pad we may update its ancestors, but
240  // the queue only ever contains uncles/great-uncles/etc. of CurrentPad,
241  // so they should never get updated while queued on the worklist.
242  assert(!MemoMap.count(CurrentPad));
243  Value *UnwindDestToken = nullptr;
244  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(CurrentPad)) {
245  if (CatchSwitch->hasUnwindDest()) {
246  UnwindDestToken = CatchSwitch->getUnwindDest()->getFirstNonPHI();
247  } else {
248  // Catchswitch doesn't have a 'nounwind' variant, and one might be
249  // annotated as "unwinds to caller" when really it's nounwind (see
250  // e.g. SimplifyCFGOpt::SimplifyUnreachable), so we can't infer the
251  // parent's unwind dest from this. We can check its catchpads'
252  // descendants, since they might include a cleanuppad with an
253  // "unwinds to caller" cleanupret, which can be trusted.
254  for (auto HI = CatchSwitch->handler_begin(),
255  HE = CatchSwitch->handler_end();
256  HI != HE && !UnwindDestToken; ++HI) {
257  BasicBlock *HandlerBlock = *HI;
258  auto *CatchPad = cast<CatchPadInst>(HandlerBlock->getFirstNonPHI());
259  for (User *Child : CatchPad->users()) {
260  // Intentionally ignore invokes here -- since the catchswitch is
261  // marked "unwind to caller", it would be a verifier error if it
262  // contained an invoke which unwinds out of it, so any invoke we'd
263  // encounter must unwind to some child of the catch.
264  if (!isa<CleanupPadInst>(Child) && !isa<CatchSwitchInst>(Child))
265  continue;
266 
267  Instruction *ChildPad = cast<Instruction>(Child);
268  auto Memo = MemoMap.find(ChildPad);
269  if (Memo == MemoMap.end()) {
270  // Haven't figured out this child pad yet; queue it.
271  Worklist.push_back(ChildPad);
272  continue;
273  }
274  // We've already checked this child, but might have found that
275  // it offers no proof either way.
276  Value *ChildUnwindDestToken = Memo->second;
277  if (!ChildUnwindDestToken)
278  continue;
279  // We already know the child's unwind dest, which can either
280  // be ConstantTokenNone to indicate unwind to caller, or can
281  // be another child of the catchpad. Only the former indicates
282  // the unwind dest of the catchswitch.
283  if (isa<ConstantTokenNone>(ChildUnwindDestToken)) {
284  UnwindDestToken = ChildUnwindDestToken;
285  break;
286  }
287  assert(getParentPad(ChildUnwindDestToken) == CatchPad);
288  }
289  }
290  }
291  } else {
292  auto *CleanupPad = cast<CleanupPadInst>(CurrentPad);
293  for (User *U : CleanupPad->users()) {
294  if (auto *CleanupRet = dyn_cast<CleanupReturnInst>(U)) {
295  if (BasicBlock *RetUnwindDest = CleanupRet->getUnwindDest())
296  UnwindDestToken = RetUnwindDest->getFirstNonPHI();
297  else
298  UnwindDestToken = ConstantTokenNone::get(CleanupPad->getContext());
299  break;
300  }
301  Value *ChildUnwindDestToken;
302  if (auto *Invoke = dyn_cast<InvokeInst>(U)) {
303  ChildUnwindDestToken = Invoke->getUnwindDest()->getFirstNonPHI();
304  } else if (isa<CleanupPadInst>(U) || isa<CatchSwitchInst>(U)) {
305  Instruction *ChildPad = cast<Instruction>(U);
306  auto Memo = MemoMap.find(ChildPad);
307  if (Memo == MemoMap.end()) {
308  // Haven't resolved this child yet; queue it and keep searching.
309  Worklist.push_back(ChildPad);
310  continue;
311  }
312  // We've checked this child, but still need to ignore it if it
313  // had no proof either way.
314  ChildUnwindDestToken = Memo->second;
315  if (!ChildUnwindDestToken)
316  continue;
317  } else {
318  // Not a relevant user of the cleanuppad
319  continue;
320  }
321  // In a well-formed program, the child/invoke must either unwind to
322  // an(other) child of the cleanup, or exit the cleanup. In the
323  // first case, continue searching.
324  if (isa<Instruction>(ChildUnwindDestToken) &&
325  getParentPad(ChildUnwindDestToken) == CleanupPad)
326  continue;
327  UnwindDestToken = ChildUnwindDestToken;
328  break;
329  }
330  }
331  // If we haven't found an unwind dest for CurrentPad, we may have queued its
332  // children, so move on to the next in the worklist.
333  if (!UnwindDestToken)
334  continue;
335 
336  // Now we know that CurrentPad unwinds to UnwindDestToken. It also exits
337  // any ancestors of CurrentPad up to but not including UnwindDestToken's
338  // parent pad. Record this in the memo map, and check to see if the
339  // original EHPad being queried is one of the ones exited.
340  Value *UnwindParent;
341  if (auto *UnwindPad = dyn_cast<Instruction>(UnwindDestToken))
342  UnwindParent = getParentPad(UnwindPad);
343  else
344  UnwindParent = nullptr;
345  bool ExitedOriginalPad = false;
346  for (Instruction *ExitedPad = CurrentPad;
347  ExitedPad && ExitedPad != UnwindParent;
348  ExitedPad = dyn_cast<Instruction>(getParentPad(ExitedPad))) {
349  // Skip over catchpads since they just follow their catchswitches.
350  if (isa<CatchPadInst>(ExitedPad))
351  continue;
352  MemoMap[ExitedPad] = UnwindDestToken;
353  ExitedOriginalPad |= (ExitedPad == EHPad);
354  }
355 
356  if (ExitedOriginalPad)
357  return UnwindDestToken;
358 
359  // Continue the search.
360  }
361 
362  // No definitive information is contained within this funclet.
363  return nullptr;
364 }
365 
366 /// Given an EH pad, find where it unwinds. If it unwinds to an EH pad,
367 /// return that pad instruction. If it unwinds to caller, return
368 /// ConstantTokenNone. If it does not have a definitive unwind destination,
369 /// return nullptr.
370 ///
371 /// This routine gets invoked for calls in funclets in inlinees when inlining
372 /// an invoke. Since many funclets don't have calls inside them, it's queried
373 /// on-demand rather than building a map of pads to unwind dests up front.
374 /// Determining a funclet's unwind dest may require recursively searching its
375 /// descendants, and also ancestors and cousins if the descendants don't provide
376 /// an answer. Since most funclets will have their unwind dest immediately
377 /// available as the unwind dest of a catchswitch or cleanupret, this routine
378 /// searches top-down from the given pad and then up. To avoid worst-case
379 /// quadratic run-time given that approach, it uses a memo map to avoid
380 /// re-processing funclet trees. The callers that rewrite the IR as they go
381 /// take advantage of this, for correctness, by checking/forcing rewritten
382 /// pads' entries to match the original callee view.
384  UnwindDestMemoTy &MemoMap) {
385  // Catchpads unwind to the same place as their catchswitch;
386  // redirct any queries on catchpads so the code below can
387  // deal with just catchswitches and cleanuppads.
388  if (auto *CPI = dyn_cast<CatchPadInst>(EHPad))
389  EHPad = CPI->getCatchSwitch();
390 
391  // Check if we've already determined the unwind dest for this pad.
392  auto Memo = MemoMap.find(EHPad);
393  if (Memo != MemoMap.end())
394  return Memo->second;
395 
396  // Search EHPad and, if necessary, its descendants.
397  Value *UnwindDestToken = getUnwindDestTokenHelper(EHPad, MemoMap);
398  assert((UnwindDestToken == nullptr) != (MemoMap.count(EHPad) != 0));
399  if (UnwindDestToken)
400  return UnwindDestToken;
401 
402  // No information is available for this EHPad from itself or any of its
403  // descendants. An unwind all the way out to a pad in the caller would
404  // need also to agree with the unwind dest of the parent funclet, so
405  // search up the chain to try to find a funclet with information. Put
406  // null entries in the memo map to avoid re-processing as we go up.
407  MemoMap[EHPad] = nullptr;
408 #ifndef NDEBUG
410  TempMemos.insert(EHPad);
411 #endif
412  Instruction *LastUselessPad = EHPad;
413  Value *AncestorToken;
414  for (AncestorToken = getParentPad(EHPad);
415  auto *AncestorPad = dyn_cast<Instruction>(AncestorToken);
416  AncestorToken = getParentPad(AncestorToken)) {
417  // Skip over catchpads since they just follow their catchswitches.
418  if (isa<CatchPadInst>(AncestorPad))
419  continue;
420  // If the MemoMap had an entry mapping AncestorPad to nullptr, since we
421  // haven't yet called getUnwindDestTokenHelper for AncestorPad in this
422  // call to getUnwindDestToken, that would mean that AncestorPad had no
423  // information in itself, its descendants, or its ancestors. If that
424  // were the case, then we should also have recorded the lack of information
425  // for the descendant that we're coming from. So assert that we don't
426  // find a null entry in the MemoMap for AncestorPad.
427  assert(!MemoMap.count(AncestorPad) || MemoMap[AncestorPad]);
428  auto AncestorMemo = MemoMap.find(AncestorPad);
429  if (AncestorMemo == MemoMap.end()) {
430  UnwindDestToken = getUnwindDestTokenHelper(AncestorPad, MemoMap);
431  } else {
432  UnwindDestToken = AncestorMemo->second;
433  }
434  if (UnwindDestToken)
435  break;
436  LastUselessPad = AncestorPad;
437  MemoMap[LastUselessPad] = nullptr;
438 #ifndef NDEBUG
439  TempMemos.insert(LastUselessPad);
440 #endif
441  }
442 
443  // We know that getUnwindDestTokenHelper was called on LastUselessPad and
444  // returned nullptr (and likewise for EHPad and any of its ancestors up to
445  // LastUselessPad), so LastUselessPad has no information from below. Since
446  // getUnwindDestTokenHelper must investigate all downward paths through
447  // no-information nodes to prove that a node has no information like this,
448  // and since any time it finds information it records it in the MemoMap for
449  // not just the immediately-containing funclet but also any ancestors also
450  // exited, it must be the case that, walking downward from LastUselessPad,
451  // visiting just those nodes which have not been mapped to an unwind dest
452  // by getUnwindDestTokenHelper (the nullptr TempMemos notwithstanding, since
453  // they are just used to keep getUnwindDestTokenHelper from repeating work),
454  // any node visited must have been exhaustively searched with no information
455  // for it found.
456  SmallVector<Instruction *, 8> Worklist(1, LastUselessPad);
457  while (!Worklist.empty()) {
458  Instruction *UselessPad = Worklist.pop_back_val();
459  auto Memo = MemoMap.find(UselessPad);
460  if (Memo != MemoMap.end() && Memo->second) {
461  // Here the name 'UselessPad' is a bit of a misnomer, because we've found
462  // that it is a funclet that does have information about unwinding to
463  // a particular destination; its parent was a useless pad.
464  // Since its parent has no information, the unwind edge must not escape
465  // the parent, and must target a sibling of this pad. This local unwind
466  // gives us no information about EHPad. Leave it and the subtree rooted
467  // at it alone.
468  assert(getParentPad(Memo->second) == getParentPad(UselessPad));
469  continue;
470  }
471  // We know we don't have information for UselesPad. If it has an entry in
472  // the MemoMap (mapping it to nullptr), it must be one of the TempMemos
473  // added on this invocation of getUnwindDestToken; if a previous invocation
474  // recorded nullptr, it would have had to prove that the ancestors of
475  // UselessPad, which include LastUselessPad, had no information, and that
476  // in turn would have required proving that the descendants of
477  // LastUselesPad, which include EHPad, have no information about
478  // LastUselessPad, which would imply that EHPad was mapped to nullptr in
479  // the MemoMap on that invocation, which isn't the case if we got here.
480  assert(!MemoMap.count(UselessPad) || TempMemos.count(UselessPad));
481  // Assert as we enumerate users that 'UselessPad' doesn't have any unwind
482  // information that we'd be contradicting by making a map entry for it
483  // (which is something that getUnwindDestTokenHelper must have proved for
484  // us to get here). Just assert on is direct users here; the checks in
485  // this downward walk at its descendants will verify that they don't have
486  // any unwind edges that exit 'UselessPad' either (i.e. they either have no
487  // unwind edges or unwind to a sibling).
488  MemoMap[UselessPad] = UnwindDestToken;
489  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(UselessPad)) {
490  assert(CatchSwitch->getUnwindDest() == nullptr && "Expected useless pad");
491  for (BasicBlock *HandlerBlock : CatchSwitch->handlers()) {
492  auto *CatchPad = HandlerBlock->getFirstNonPHI();
493  for (User *U : CatchPad->users()) {
494  assert(
495  (!isa<InvokeInst>(U) ||
496  (getParentPad(
497  cast<InvokeInst>(U)->getUnwindDest()->getFirstNonPHI()) ==
498  CatchPad)) &&
499  "Expected useless pad");
500  if (isa<CatchSwitchInst>(U) || isa<CleanupPadInst>(U))
501  Worklist.push_back(cast<Instruction>(U));
502  }
503  }
504  } else {
505  assert(isa<CleanupPadInst>(UselessPad));
506  for (User *U : UselessPad->users()) {
507  assert(!isa<CleanupReturnInst>(U) && "Expected useless pad");
508  assert((!isa<InvokeInst>(U) ||
509  (getParentPad(
510  cast<InvokeInst>(U)->getUnwindDest()->getFirstNonPHI()) ==
511  UselessPad)) &&
512  "Expected useless pad");
513  if (isa<CatchSwitchInst>(U) || isa<CleanupPadInst>(U))
514  Worklist.push_back(cast<Instruction>(U));
515  }
516  }
517  }
518 
519  return UnwindDestToken;
520 }
521 
522 /// When we inline a basic block into an invoke,
523 /// we have to turn all of the calls that can throw into invokes.
524 /// This function analyze BB to see if there are any calls, and if so,
525 /// it rewrites them to be invokes that jump to InvokeDest and fills in the PHI
526 /// nodes in that block with the values specified in InvokeDestPHIValues.
528  BasicBlock *BB, BasicBlock *UnwindEdge,
529  UnwindDestMemoTy *FuncletUnwindMap = nullptr) {
530  for (BasicBlock::iterator BBI = BB->begin(), E = BB->end(); BBI != E; ) {
531  Instruction *I = &*BBI++;
532 
533  // We only need to check for function calls: inlined invoke
534  // instructions require no special handling.
535  CallInst *CI = dyn_cast<CallInst>(I);
536 
537  if (!CI || CI->doesNotThrow() || isa<InlineAsm>(CI->getCalledValue()))
538  continue;
539 
540  // We do not need to (and in fact, cannot) convert possibly throwing calls
541  // to @llvm.experimental_deoptimize (resp. @llvm.experimental.guard) into
542  // invokes. The caller's "segment" of the deoptimization continuation
543  // attached to the newly inlined @llvm.experimental_deoptimize
544  // (resp. @llvm.experimental.guard) call should contain the exception
545  // handling logic, if any.
546  if (auto *F = CI->getCalledFunction())
547  if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize ||
548  F->getIntrinsicID() == Intrinsic::experimental_guard)
549  continue;
550 
551  if (auto FuncletBundle = CI->getOperandBundle(LLVMContext::OB_funclet)) {
552  // This call is nested inside a funclet. If that funclet has an unwind
553  // destination within the inlinee, then unwinding out of this call would
554  // be UB. Rewriting this call to an invoke which targets the inlined
555  // invoke's unwind dest would give the call's parent funclet multiple
556  // unwind destinations, which is something that subsequent EH table
557  // generation can't handle and that the veirifer rejects. So when we
558  // see such a call, leave it as a call.
559  auto *FuncletPad = cast<Instruction>(FuncletBundle->Inputs[0]);
560  Value *UnwindDestToken =
561  getUnwindDestToken(FuncletPad, *FuncletUnwindMap);
562  if (UnwindDestToken && !isa<ConstantTokenNone>(UnwindDestToken))
563  continue;
564 #ifndef NDEBUG
565  Instruction *MemoKey;
566  if (auto *CatchPad = dyn_cast<CatchPadInst>(FuncletPad))
567  MemoKey = CatchPad->getCatchSwitch();
568  else
569  MemoKey = FuncletPad;
570  assert(FuncletUnwindMap->count(MemoKey) &&
571  (*FuncletUnwindMap)[MemoKey] == UnwindDestToken &&
572  "must get memoized to avoid confusing later searches");
573 #endif // NDEBUG
574  }
575 
576  changeToInvokeAndSplitBasicBlock(CI, UnwindEdge);
577  return BB;
578  }
579  return nullptr;
580 }
581 
582 /// If we inlined an invoke site, we need to convert calls
583 /// in the body of the inlined function into invokes.
584 ///
585 /// II is the invoke instruction being inlined. FirstNewBlock is the first
586 /// block of the inlined code (the last block is the end of the function),
587 /// and InlineCodeInfo is information about the code that got inlined.
588 static void HandleInlinedLandingPad(InvokeInst *II, BasicBlock *FirstNewBlock,
589  ClonedCodeInfo &InlinedCodeInfo) {
590  BasicBlock *InvokeDest = II->getUnwindDest();
591 
592  Function *Caller = FirstNewBlock->getParent();
593 
594  // The inlined code is currently at the end of the function, scan from the
595  // start of the inlined code to its end, checking for stuff we need to
596  // rewrite.
597  LandingPadInliningInfo Invoke(II);
598 
599  // Get all of the inlined landing pad instructions.
601  for (Function::iterator I = FirstNewBlock->getIterator(), E = Caller->end();
602  I != E; ++I)
603  if (InvokeInst *II = dyn_cast<InvokeInst>(I->getTerminator()))
604  InlinedLPads.insert(II->getLandingPadInst());
605 
606  // Append the clauses from the outer landing pad instruction into the inlined
607  // landing pad instructions.
608  LandingPadInst *OuterLPad = Invoke.getLandingPadInst();
609  for (LandingPadInst *InlinedLPad : InlinedLPads) {
610  unsigned OuterNum = OuterLPad->getNumClauses();
611  InlinedLPad->reserveClauses(OuterNum);
612  for (unsigned OuterIdx = 0; OuterIdx != OuterNum; ++OuterIdx)
613  InlinedLPad->addClause(OuterLPad->getClause(OuterIdx));
614  if (OuterLPad->isCleanup())
615  InlinedLPad->setCleanup(true);
616  }
617 
618  for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end();
619  BB != E; ++BB) {
620  if (InlinedCodeInfo.ContainsCalls)
622  &*BB, Invoke.getOuterResumeDest()))
623  // Update any PHI nodes in the exceptional block to indicate that there
624  // is now a new entry in them.
625  Invoke.addIncomingPHIValuesFor(NewBB);
626 
627  // Forward any resumes that are remaining here.
628  if (ResumeInst *RI = dyn_cast<ResumeInst>(BB->getTerminator()))
629  Invoke.forwardResume(RI, InlinedLPads);
630  }
631 
632  // Now that everything is happy, we have one final detail. The PHI nodes in
633  // the exception destination block still have entries due to the original
634  // invoke instruction. Eliminate these entries (which might even delete the
635  // PHI node) now.
636  InvokeDest->removePredecessor(II->getParent());
637 }
638 
639 /// If we inlined an invoke site, we need to convert calls
640 /// in the body of the inlined function into invokes.
641 ///
642 /// II is the invoke instruction being inlined. FirstNewBlock is the first
643 /// block of the inlined code (the last block is the end of the function),
644 /// and InlineCodeInfo is information about the code that got inlined.
645 static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock,
646  ClonedCodeInfo &InlinedCodeInfo) {
647  BasicBlock *UnwindDest = II->getUnwindDest();
648  Function *Caller = FirstNewBlock->getParent();
649 
650  assert(UnwindDest->getFirstNonPHI()->isEHPad() && "unexpected BasicBlock!");
651 
652  // If there are PHI nodes in the unwind destination block, we need to keep
653  // track of which values came into them from the invoke before removing the
654  // edge from this block.
655  SmallVector<Value *, 8> UnwindDestPHIValues;
656  BasicBlock *InvokeBB = II->getParent();
657  for (Instruction &I : *UnwindDest) {
658  // Save the value to use for this edge.
659  PHINode *PHI = dyn_cast<PHINode>(&I);
660  if (!PHI)
661  break;
662  UnwindDestPHIValues.push_back(PHI->getIncomingValueForBlock(InvokeBB));
663  }
664 
665  // Add incoming-PHI values to the unwind destination block for the given basic
666  // block, using the values for the original invoke's source block.
667  auto UpdatePHINodes = [&](BasicBlock *Src) {
668  BasicBlock::iterator I = UnwindDest->begin();
669  for (Value *V : UnwindDestPHIValues) {
670  PHINode *PHI = cast<PHINode>(I);
671  PHI->addIncoming(V, Src);
672  ++I;
673  }
674  };
675 
676  // This connects all the instructions which 'unwind to caller' to the invoke
677  // destination.
678  UnwindDestMemoTy FuncletUnwindMap;
679  for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end();
680  BB != E; ++BB) {
681  if (auto *CRI = dyn_cast<CleanupReturnInst>(BB->getTerminator())) {
682  if (CRI->unwindsToCaller()) {
683  auto *CleanupPad = CRI->getCleanupPad();
684  CleanupReturnInst::Create(CleanupPad, UnwindDest, CRI);
685  CRI->eraseFromParent();
686  UpdatePHINodes(&*BB);
687  // Finding a cleanupret with an unwind destination would confuse
688  // subsequent calls to getUnwindDestToken, so map the cleanuppad
689  // to short-circuit any such calls and recognize this as an "unwind
690  // to caller" cleanup.
691  assert(!FuncletUnwindMap.count(CleanupPad) ||
692  isa<ConstantTokenNone>(FuncletUnwindMap[CleanupPad]));
693  FuncletUnwindMap[CleanupPad] =
695  }
696  }
697 
698  Instruction *I = BB->getFirstNonPHI();
699  if (!I->isEHPad())
700  continue;
701 
702  Instruction *Replacement = nullptr;
703  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I)) {
704  if (CatchSwitch->unwindsToCaller()) {
705  Value *UnwindDestToken;
706  if (auto *ParentPad =
707  dyn_cast<Instruction>(CatchSwitch->getParentPad())) {
708  // This catchswitch is nested inside another funclet. If that
709  // funclet has an unwind destination within the inlinee, then
710  // unwinding out of this catchswitch would be UB. Rewriting this
711  // catchswitch to unwind to the inlined invoke's unwind dest would
712  // give the parent funclet multiple unwind destinations, which is
713  // something that subsequent EH table generation can't handle and
714  // that the veirifer rejects. So when we see such a call, leave it
715  // as "unwind to caller".
716  UnwindDestToken = getUnwindDestToken(ParentPad, FuncletUnwindMap);
717  if (UnwindDestToken && !isa<ConstantTokenNone>(UnwindDestToken))
718  continue;
719  } else {
720  // This catchswitch has no parent to inherit constraints from, and
721  // none of its descendants can have an unwind edge that exits it and
722  // targets another funclet in the inlinee. It may or may not have a
723  // descendant that definitively has an unwind to caller. In either
724  // case, we'll have to assume that any unwinds out of it may need to
725  // be routed to the caller, so treat it as though it has a definitive
726  // unwind to caller.
727  UnwindDestToken = ConstantTokenNone::get(Caller->getContext());
728  }
729  auto *NewCatchSwitch = CatchSwitchInst::Create(
730  CatchSwitch->getParentPad(), UnwindDest,
731  CatchSwitch->getNumHandlers(), CatchSwitch->getName(),
732  CatchSwitch);
733  for (BasicBlock *PadBB : CatchSwitch->handlers())
734  NewCatchSwitch->addHandler(PadBB);
735  // Propagate info for the old catchswitch over to the new one in
736  // the unwind map. This also serves to short-circuit any subsequent
737  // checks for the unwind dest of this catchswitch, which would get
738  // confused if they found the outer handler in the callee.
739  FuncletUnwindMap[NewCatchSwitch] = UnwindDestToken;
740  Replacement = NewCatchSwitch;
741  }
742  } else if (!isa<FuncletPadInst>(I)) {
743  llvm_unreachable("unexpected EHPad!");
744  }
745 
746  if (Replacement) {
747  Replacement->takeName(I);
748  I->replaceAllUsesWith(Replacement);
749  I->eraseFromParent();
750  UpdatePHINodes(&*BB);
751  }
752  }
753 
754  if (InlinedCodeInfo.ContainsCalls)
755  for (Function::iterator BB = FirstNewBlock->getIterator(),
756  E = Caller->end();
757  BB != E; ++BB)
759  &*BB, UnwindDest, &FuncletUnwindMap))
760  // Update any PHI nodes in the exceptional block to indicate that there
761  // is now a new entry in them.
762  UpdatePHINodes(NewBB);
763 
764  // Now that everything is happy, we have one final detail. The PHI nodes in
765  // the exception destination block still have entries due to the original
766  // invoke instruction. Eliminate these entries (which might even delete the
767  // PHI node) now.
768  UnwindDest->removePredecessor(InvokeBB);
769 }
770 
771 /// When inlining a call site that has !llvm.mem.parallel_loop_access metadata,
772 /// that metadata should be propagated to all memory-accessing cloned
773 /// instructions.
775  ValueToValueMapTy &VMap) {
776  MDNode *M =
778  if (!M)
779  return;
780 
781  for (ValueToValueMapTy::iterator VMI = VMap.begin(), VMIE = VMap.end();
782  VMI != VMIE; ++VMI) {
783  if (!VMI->second)
784  continue;
785 
786  Instruction *NI = dyn_cast<Instruction>(VMI->second);
787  if (!NI)
788  continue;
789 
791  M = MDNode::concatenate(PM, M);
793  } else if (NI->mayReadOrWriteMemory()) {
795  }
796  }
797 }
798 
799 /// When inlining a function that contains noalias scope metadata,
800 /// this metadata needs to be cloned so that the inlined blocks
801 /// have different "unique scopes" at every call site. Were this not done, then
802 /// aliasing scopes from a function inlined into a caller multiple times could
803 /// not be differentiated (and this would lead to miscompiles because the
804 /// non-aliasing property communicated by the metadata could have
805 /// call-site-specific control dependencies).
807  const Function *CalledFunc = CS.getCalledFunction();
809 
810  // Note: We could only clone the metadata if it is already used in the
811  // caller. I'm omitting that check here because it might confuse
812  // inter-procedural alias analysis passes. We can revisit this if it becomes
813  // an efficiency or overhead problem.
814 
815  for (const BasicBlock &I : *CalledFunc)
816  for (const Instruction &J : I) {
817  if (const MDNode *M = J.getMetadata(LLVMContext::MD_alias_scope))
818  MD.insert(M);
819  if (const MDNode *M = J.getMetadata(LLVMContext::MD_noalias))
820  MD.insert(M);
821  }
822 
823  if (MD.empty())
824  return;
825 
826  // Walk the existing metadata, adding the complete (perhaps cyclic) chain to
827  // the set.
828  SmallVector<const Metadata *, 16> Queue(MD.begin(), MD.end());
829  while (!Queue.empty()) {
830  const MDNode *M = cast<MDNode>(Queue.pop_back_val());
831  for (unsigned i = 0, ie = M->getNumOperands(); i != ie; ++i)
832  if (const MDNode *M1 = dyn_cast<MDNode>(M->getOperand(i)))
833  if (MD.insert(M1))
834  Queue.push_back(M1);
835  }
836 
837  // Now we have a complete set of all metadata in the chains used to specify
838  // the noalias scopes and the lists of those scopes.
839  SmallVector<TempMDTuple, 16> DummyNodes;
841  for (const MDNode *I : MD) {
842  DummyNodes.push_back(MDTuple::getTemporary(CalledFunc->getContext(), None));
843  MDMap[I].reset(DummyNodes.back().get());
844  }
845 
846  // Create new metadata nodes to replace the dummy nodes, replacing old
847  // metadata references with either a dummy node or an already-created new
848  // node.
849  for (const MDNode *I : MD) {
851  for (unsigned i = 0, ie = I->getNumOperands(); i != ie; ++i) {
852  const Metadata *V = I->getOperand(i);
853  if (const MDNode *M = dyn_cast<MDNode>(V))
854  NewOps.push_back(MDMap[M]);
855  else
856  NewOps.push_back(const_cast<Metadata *>(V));
857  }
858 
859  MDNode *NewM = MDNode::get(CalledFunc->getContext(), NewOps);
860  MDTuple *TempM = cast<MDTuple>(MDMap[I]);
861  assert(TempM->isTemporary() && "Expected temporary node");
862 
863  TempM->replaceAllUsesWith(NewM);
864  }
865 
866  // Now replace the metadata in the new inlined instructions with the
867  // repacements from the map.
868  for (ValueToValueMapTy::iterator VMI = VMap.begin(), VMIE = VMap.end();
869  VMI != VMIE; ++VMI) {
870  if (!VMI->second)
871  continue;
872 
873  Instruction *NI = dyn_cast<Instruction>(VMI->second);
874  if (!NI)
875  continue;
876 
878  MDNode *NewMD = MDMap[M];
879  // If the call site also had alias scope metadata (a list of scopes to
880  // which instructions inside it might belong), propagate those scopes to
881  // the inlined instructions.
882  if (MDNode *CSM =
884  NewMD = MDNode::concatenate(NewMD, CSM);
886  } else if (NI->mayReadOrWriteMemory()) {
887  if (MDNode *M =
890  }
891 
893  MDNode *NewMD = MDMap[M];
894  // If the call site also had noalias metadata (a list of scopes with
895  // which instructions inside it don't alias), propagate those scopes to
896  // the inlined instructions.
897  if (MDNode *CSM =
899  NewMD = MDNode::concatenate(NewMD, CSM);
901  } else if (NI->mayReadOrWriteMemory()) {
904  }
905  }
906 }
907 
908 /// If the inlined function has noalias arguments,
909 /// then add new alias scopes for each noalias argument, tag the mapped noalias
910 /// parameters with noalias metadata specifying the new scope, and tag all
911 /// non-derived loads, stores and memory intrinsics with the new alias scopes.
913  const DataLayout &DL, AAResults *CalleeAAR) {
915  return;
916 
917  const Function *CalledFunc = CS.getCalledFunction();
919 
920  for (const Argument &Arg : CalledFunc->args())
921  if (Arg.hasNoAliasAttr() && !Arg.use_empty())
922  NoAliasArgs.push_back(&Arg);
923 
924  if (NoAliasArgs.empty())
925  return;
926 
927  // To do a good job, if a noalias variable is captured, we need to know if
928  // the capture point dominates the particular use we're considering.
929  DominatorTree DT;
930  DT.recalculate(const_cast<Function&>(*CalledFunc));
931 
932  // noalias indicates that pointer values based on the argument do not alias
933  // pointer values which are not based on it. So we add a new "scope" for each
934  // noalias function argument. Accesses using pointers based on that argument
935  // become part of that alias scope, accesses using pointers not based on that
936  // argument are tagged as noalias with that scope.
937 
939  MDBuilder MDB(CalledFunc->getContext());
940 
941  // Create a new scope domain for this function.
942  MDNode *NewDomain =
943  MDB.createAnonymousAliasScopeDomain(CalledFunc->getName());
944  for (unsigned i = 0, e = NoAliasArgs.size(); i != e; ++i) {
945  const Argument *A = NoAliasArgs[i];
946 
947  std::string Name = CalledFunc->getName();
948  if (A->hasName()) {
949  Name += ": %";
950  Name += A->getName();
951  } else {
952  Name += ": argument ";
953  Name += utostr(i);
954  }
955 
956  // Note: We always create a new anonymous root here. This is true regardless
957  // of the linkage of the callee because the aliasing "scope" is not just a
958  // property of the callee, but also all control dependencies in the caller.
959  MDNode *NewScope = MDB.createAnonymousAliasScope(NewDomain, Name);
960  NewScopes.insert(std::make_pair(A, NewScope));
961  }
962 
963  // Iterate over all new instructions in the map; for all memory-access
964  // instructions, add the alias scope metadata.
965  for (ValueToValueMapTy::iterator VMI = VMap.begin(), VMIE = VMap.end();
966  VMI != VMIE; ++VMI) {
967  if (const Instruction *I = dyn_cast<Instruction>(VMI->first)) {
968  if (!VMI->second)
969  continue;
970 
971  Instruction *NI = dyn_cast<Instruction>(VMI->second);
972  if (!NI)
973  continue;
974 
975  bool IsArgMemOnlyCall = false, IsFuncCall = false;
977 
978  if (const LoadInst *LI = dyn_cast<LoadInst>(I))
979  PtrArgs.push_back(LI->getPointerOperand());
980  else if (const StoreInst *SI = dyn_cast<StoreInst>(I))
981  PtrArgs.push_back(SI->getPointerOperand());
982  else if (const VAArgInst *VAAI = dyn_cast<VAArgInst>(I))
983  PtrArgs.push_back(VAAI->getPointerOperand());
984  else if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(I))
985  PtrArgs.push_back(CXI->getPointerOperand());
986  else if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I))
987  PtrArgs.push_back(RMWI->getPointerOperand());
988  else if (ImmutableCallSite ICS = ImmutableCallSite(I)) {
989  // If we know that the call does not access memory, then we'll still
990  // know that about the inlined clone of this call site, and we don't
991  // need to add metadata.
992  if (ICS.doesNotAccessMemory())
993  continue;
994 
995  IsFuncCall = true;
996  if (CalleeAAR) {
997  FunctionModRefBehavior MRB = CalleeAAR->getModRefBehavior(ICS);
1000  IsArgMemOnlyCall = true;
1001  }
1002 
1003  for (Value *Arg : ICS.args()) {
1004  // We need to check the underlying objects of all arguments, not just
1005  // the pointer arguments, because we might be passing pointers as
1006  // integers, etc.
1007  // However, if we know that the call only accesses pointer arguments,
1008  // then we only need to check the pointer arguments.
1009  if (IsArgMemOnlyCall && !Arg->getType()->isPointerTy())
1010  continue;
1011 
1012  PtrArgs.push_back(Arg);
1013  }
1014  }
1015 
1016  // If we found no pointers, then this instruction is not suitable for
1017  // pairing with an instruction to receive aliasing metadata.
1018  // However, if this is a call, this we might just alias with none of the
1019  // noalias arguments.
1020  if (PtrArgs.empty() && !IsFuncCall)
1021  continue;
1022 
1023  // It is possible that there is only one underlying object, but you
1024  // need to go through several PHIs to see it, and thus could be
1025  // repeated in the Objects list.
1028 
1030  for (const Value *V : PtrArgs) {
1031  SmallVector<Value *, 4> Objects;
1032  GetUnderlyingObjects(const_cast<Value*>(V),
1033  Objects, DL, /* LI = */ nullptr);
1034 
1035  for (Value *O : Objects)
1036  ObjSet.insert(O);
1037  }
1038 
1039  // Figure out if we're derived from anything that is not a noalias
1040  // argument.
1041  bool CanDeriveViaCapture = false, UsesAliasingPtr = false;
1042  for (const Value *V : ObjSet) {
1043  // Is this value a constant that cannot be derived from any pointer
1044  // value (we need to exclude constant expressions, for example, that
1045  // are formed from arithmetic on global symbols).
1046  bool IsNonPtrConst = isa<ConstantInt>(V) || isa<ConstantFP>(V) ||
1047  isa<ConstantPointerNull>(V) ||
1048  isa<ConstantDataVector>(V) || isa<UndefValue>(V);
1049  if (IsNonPtrConst)
1050  continue;
1051 
1052  // If this is anything other than a noalias argument, then we cannot
1053  // completely describe the aliasing properties using alias.scope
1054  // metadata (and, thus, won't add any).
1055  if (const Argument *A = dyn_cast<Argument>(V)) {
1056  if (!A->hasNoAliasAttr())
1057  UsesAliasingPtr = true;
1058  } else {
1059  UsesAliasingPtr = true;
1060  }
1061 
1062  // If this is not some identified function-local object (which cannot
1063  // directly alias a noalias argument), or some other argument (which,
1064  // by definition, also cannot alias a noalias argument), then we could
1065  // alias a noalias argument that has been captured).
1066  if (!isa<Argument>(V) &&
1067  !isIdentifiedFunctionLocal(const_cast<Value*>(V)))
1068  CanDeriveViaCapture = true;
1069  }
1070 
1071  // A function call can always get captured noalias pointers (via other
1072  // parameters, globals, etc.).
1073  if (IsFuncCall && !IsArgMemOnlyCall)
1074  CanDeriveViaCapture = true;
1075 
1076  // First, we want to figure out all of the sets with which we definitely
1077  // don't alias. Iterate over all noalias set, and add those for which:
1078  // 1. The noalias argument is not in the set of objects from which we
1079  // definitely derive.
1080  // 2. The noalias argument has not yet been captured.
1081  // An arbitrary function that might load pointers could see captured
1082  // noalias arguments via other noalias arguments or globals, and so we
1083  // must always check for prior capture.
1084  for (const Argument *A : NoAliasArgs) {
1085  if (!ObjSet.count(A) && (!CanDeriveViaCapture ||
1086  // It might be tempting to skip the
1087  // PointerMayBeCapturedBefore check if
1088  // A->hasNoCaptureAttr() is true, but this is
1089  // incorrect because nocapture only guarantees
1090  // that no copies outlive the function, not
1091  // that the value cannot be locally captured.
1093  /* ReturnCaptures */ false,
1094  /* StoreCaptures */ false, I, &DT)))
1095  NoAliases.push_back(NewScopes[A]);
1096  }
1097 
1098  if (!NoAliases.empty())
1102  MDNode::get(CalledFunc->getContext(), NoAliases)));
1103 
1104  // Next, we want to figure out all of the sets to which we might belong.
1105  // We might belong to a set if the noalias argument is in the set of
1106  // underlying objects. If there is some non-noalias argument in our list
1107  // of underlying objects, then we cannot add a scope because the fact
1108  // that some access does not alias with any set of our noalias arguments
1109  // cannot itself guarantee that it does not alias with this access
1110  // (because there is some pointer of unknown origin involved and the
1111  // other access might also depend on this pointer). We also cannot add
1112  // scopes to arbitrary functions unless we know they don't access any
1113  // non-parameter pointer-values.
1114  bool CanAddScopes = !UsesAliasingPtr;
1115  if (CanAddScopes && IsFuncCall)
1116  CanAddScopes = IsArgMemOnlyCall;
1117 
1118  if (CanAddScopes)
1119  for (const Argument *A : NoAliasArgs) {
1120  if (ObjSet.count(A))
1121  Scopes.push_back(NewScopes[A]);
1122  }
1123 
1124  if (!Scopes.empty())
1125  NI->setMetadata(
1128  MDNode::get(CalledFunc->getContext(), Scopes)));
1129  }
1130  }
1131 }
1132 
1133 /// If the inlined function has non-byval align arguments, then
1134 /// add @llvm.assume-based alignment assumptions to preserve this information.
1137  return;
1138 
1139  AssumptionCache *AC = &(*IFI.GetAssumptionCache)(*CS.getCaller());
1140  auto &DL = CS.getCaller()->getParent()->getDataLayout();
1141 
1142  // To avoid inserting redundant assumptions, we should check for assumptions
1143  // already in the caller. To do this, we might need a DT of the caller.
1144  DominatorTree DT;
1145  bool DTCalculated = false;
1146 
1147  Function *CalledFunc = CS.getCalledFunction();
1148  for (Argument &Arg : CalledFunc->args()) {
1149  unsigned Align = Arg.getType()->isPointerTy() ? Arg.getParamAlignment() : 0;
1150  if (Align && !Arg.hasByValOrInAllocaAttr() && !Arg.hasNUses(0)) {
1151  if (!DTCalculated) {
1152  DT.recalculate(*CS.getCaller());
1153  DTCalculated = true;
1154  }
1155 
1156  // If we can already prove the asserted alignment in the context of the
1157  // caller, then don't bother inserting the assumption.
1158  Value *ArgVal = CS.getArgument(Arg.getArgNo());
1159  if (getKnownAlignment(ArgVal, DL, CS.getInstruction(), AC, &DT) >= Align)
1160  continue;
1161 
1162  CallInst *NewAsmp = IRBuilder<>(CS.getInstruction())
1163  .CreateAlignmentAssumption(DL, ArgVal, Align);
1164  AC->registerAssumption(NewAsmp);
1165  }
1166  }
1167 }
1168 
1169 /// Once we have cloned code over from a callee into the caller,
1170 /// update the specified callgraph to reflect the changes we made.
1171 /// Note that it's possible that not all code was copied over, so only
1172 /// some edges of the callgraph may remain.
1174  Function::iterator FirstNewBlock,
1175  ValueToValueMapTy &VMap,
1176  InlineFunctionInfo &IFI) {
1177  CallGraph &CG = *IFI.CG;
1178  const Function *Caller = CS.getCaller();
1179  const Function *Callee = CS.getCalledFunction();
1180  CallGraphNode *CalleeNode = CG[Callee];
1181  CallGraphNode *CallerNode = CG[Caller];
1182 
1183  // Since we inlined some uninlined call sites in the callee into the caller,
1184  // add edges from the caller to all of the callees of the callee.
1185  CallGraphNode::iterator I = CalleeNode->begin(), E = CalleeNode->end();
1186 
1187  // Consider the case where CalleeNode == CallerNode.
1189  if (CalleeNode == CallerNode) {
1190  CallCache.assign(I, E);
1191  I = CallCache.begin();
1192  E = CallCache.end();
1193  }
1194 
1195  for (; I != E; ++I) {
1196  const Value *OrigCall = I->first;
1197 
1198  ValueToValueMapTy::iterator VMI = VMap.find(OrigCall);
1199  // Only copy the edge if the call was inlined!
1200  if (VMI == VMap.end() || VMI->second == nullptr)
1201  continue;
1202 
1203  // If the call was inlined, but then constant folded, there is no edge to
1204  // add. Check for this case.
1205  Instruction *NewCall = dyn_cast<Instruction>(VMI->second);
1206  if (!NewCall)
1207  continue;
1208 
1209  // We do not treat intrinsic calls like real function calls because we
1210  // expect them to become inline code; do not add an edge for an intrinsic.
1211  CallSite CS = CallSite(NewCall);
1212  if (CS && CS.getCalledFunction() && CS.getCalledFunction()->isIntrinsic())
1213  continue;
1214 
1215  // Remember that this call site got inlined for the client of
1216  // InlineFunction.
1217  IFI.InlinedCalls.push_back(NewCall);
1218 
1219  // It's possible that inlining the callsite will cause it to go from an
1220  // indirect to a direct call by resolving a function pointer. If this
1221  // happens, set the callee of the new call site to a more precise
1222  // destination. This can also happen if the call graph node of the caller
1223  // was just unnecessarily imprecise.
1224  if (!I->second->getFunction())
1225  if (Function *F = CallSite(NewCall).getCalledFunction()) {
1226  // Indirect call site resolved to direct call.
1227  CallerNode->addCalledFunction(CallSite(NewCall), CG[F]);
1228 
1229  continue;
1230  }
1231 
1232  CallerNode->addCalledFunction(CallSite(NewCall), I->second);
1233  }
1234 
1235  // Update the call graph by deleting the edge from Callee to Caller. We must
1236  // do this after the loop above in case Caller and Callee are the same.
1237  CallerNode->removeCallEdgeFor(CS);
1238 }
1239 
1240 static void HandleByValArgumentInit(Value *Dst, Value *Src, Module *M,
1241  BasicBlock *InsertBlock,
1242  InlineFunctionInfo &IFI) {
1243  Type *AggTy = cast<PointerType>(Src->getType())->getElementType();
1244  IRBuilder<> Builder(InsertBlock, InsertBlock->begin());
1245 
1246  Value *Size = Builder.getInt64(M->getDataLayout().getTypeStoreSize(AggTy));
1247 
1248  // Always generate a memcpy of alignment 1 here because we don't know
1249  // the alignment of the src pointer. Other optimizations can infer
1250  // better alignment.
1251  Builder.CreateMemCpy(Dst, /*DstAlign*/1, Src, /*SrcAlign*/1, Size);
1252 }
1253 
1254 /// When inlining a call site that has a byval argument,
1255 /// we have to make the implicit memcpy explicit by adding it.
1257  const Function *CalledFunc,
1258  InlineFunctionInfo &IFI,
1259  unsigned ByValAlignment) {
1260  PointerType *ArgTy = cast<PointerType>(Arg->getType());
1261  Type *AggTy = ArgTy->getElementType();
1262 
1263  Function *Caller = TheCall->getFunction();
1264  const DataLayout &DL = Caller->getParent()->getDataLayout();
1265 
1266  // If the called function is readonly, then it could not mutate the caller's
1267  // copy of the byval'd memory. In this case, it is safe to elide the copy and
1268  // temporary.
1269  if (CalledFunc->onlyReadsMemory()) {
1270  // If the byval argument has a specified alignment that is greater than the
1271  // passed in pointer, then we either have to round up the input pointer or
1272  // give up on this transformation.
1273  if (ByValAlignment <= 1) // 0 = unspecified, 1 = no particular alignment.
1274  return Arg;
1275 
1276  AssumptionCache *AC =
1277  IFI.GetAssumptionCache ? &(*IFI.GetAssumptionCache)(*Caller) : nullptr;
1278 
1279  // If the pointer is already known to be sufficiently aligned, or if we can
1280  // round it up to a larger alignment, then we don't need a temporary.
1281  if (getOrEnforceKnownAlignment(Arg, ByValAlignment, DL, TheCall, AC) >=
1282  ByValAlignment)
1283  return Arg;
1284 
1285  // Otherwise, we have to make a memcpy to get a safe alignment. This is bad
1286  // for code quality, but rarely happens and is required for correctness.
1287  }
1288 
1289  // Create the alloca. If we have DataLayout, use nice alignment.
1290  unsigned Align = DL.getPrefTypeAlignment(AggTy);
1291 
1292  // If the byval had an alignment specified, we *must* use at least that
1293  // alignment, as it is required by the byval argument (and uses of the
1294  // pointer inside the callee).
1295  Align = std::max(Align, ByValAlignment);
1296 
1297  Value *NewAlloca = new AllocaInst(AggTy, DL.getAllocaAddrSpace(),
1298  nullptr, Align, Arg->getName(),
1299  &*Caller->begin()->begin());
1300  IFI.StaticAllocas.push_back(cast<AllocaInst>(NewAlloca));
1301 
1302  // Uses of the argument in the function should use our new alloca
1303  // instead.
1304  return NewAlloca;
1305 }
1306 
1307 // Check whether this Value is used by a lifetime intrinsic.
1309  for (User *U : V->users()) {
1310  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {
1311  switch (II->getIntrinsicID()) {
1312  default: break;
1313  case Intrinsic::lifetime_start:
1314  case Intrinsic::lifetime_end:
1315  return true;
1316  }
1317  }
1318  }
1319  return false;
1320 }
1321 
1322 // Check whether the given alloca already has
1323 // lifetime.start or lifetime.end intrinsics.
1324 static bool hasLifetimeMarkers(AllocaInst *AI) {
1325  Type *Ty = AI->getType();
1326  Type *Int8PtrTy = Type::getInt8PtrTy(Ty->getContext(),
1327  Ty->getPointerAddressSpace());
1328  if (Ty == Int8PtrTy)
1329  return isUsedByLifetimeMarker(AI);
1330 
1331  // Do a scan to find all the casts to i8*.
1332  for (User *U : AI->users()) {
1333  if (U->getType() != Int8PtrTy) continue;
1334  if (U->stripPointerCasts() != AI) continue;
1335  if (isUsedByLifetimeMarker(U))
1336  return true;
1337  }
1338  return false;
1339 }
1340 
1341 /// Return the result of AI->isStaticAlloca() if AI were moved to the entry
1342 /// block. Allocas used in inalloca calls and allocas of dynamic array size
1343 /// cannot be static.
1344 static bool allocaWouldBeStaticInEntry(const AllocaInst *AI ) {
1345  return isa<Constant>(AI->getArraySize()) && !AI->isUsedWithInAlloca();
1346 }
1347 
1348 /// Update inlined instructions' line numbers to
1349 /// to encode location where these instructions are inlined.
1351  Instruction *TheCall, bool CalleeHasDebugInfo) {
1352  const DebugLoc &TheCallDL = TheCall->getDebugLoc();
1353  if (!TheCallDL)
1354  return;
1355 
1356  auto &Ctx = Fn->getContext();
1357  DILocation *InlinedAtNode = TheCallDL;
1358 
1359  // Create a unique call site, not to be confused with any other call from the
1360  // same location.
1361  InlinedAtNode = DILocation::getDistinct(
1362  Ctx, InlinedAtNode->getLine(), InlinedAtNode->getColumn(),
1363  InlinedAtNode->getScope(), InlinedAtNode->getInlinedAt());
1364 
1365  // Cache the inlined-at nodes as they're built so they are reused, without
1366  // this every instruction's inlined-at chain would become distinct from each
1367  // other.
1369 
1370  for (; FI != Fn->end(); ++FI) {
1371  for (BasicBlock::iterator BI = FI->begin(), BE = FI->end();
1372  BI != BE; ++BI) {
1373  if (DebugLoc DL = BI->getDebugLoc()) {
1374  auto IA = DebugLoc::appendInlinedAt(DL, InlinedAtNode, BI->getContext(),
1375  IANodes);
1376  auto IDL = DebugLoc::get(DL.getLine(), DL.getCol(), DL.getScope(), IA);
1377  BI->setDebugLoc(IDL);
1378  continue;
1379  }
1380 
1381  if (CalleeHasDebugInfo)
1382  continue;
1383 
1384  // If the inlined instruction has no line number, make it look as if it
1385  // originates from the call location. This is important for
1386  // ((__always_inline__, __nodebug__)) functions which must use caller
1387  // location for all instructions in their function body.
1388 
1389  // Don't update static allocas, as they may get moved later.
1390  if (auto *AI = dyn_cast<AllocaInst>(BI))
1392  continue;
1393 
1394  BI->setDebugLoc(TheCallDL);
1395  }
1396  }
1397 }
1398 
1399 /// Update the block frequencies of the caller after a callee has been inlined.
1400 ///
1401 /// Each block cloned into the caller has its block frequency scaled by the
1402 /// ratio of CallSiteFreq/CalleeEntryFreq. This ensures that the cloned copy of
1403 /// callee's entry block gets the same frequency as the callsite block and the
1404 /// relative frequencies of all cloned blocks remain the same after cloning.
1405 static void updateCallerBFI(BasicBlock *CallSiteBlock,
1406  const ValueToValueMapTy &VMap,
1407  BlockFrequencyInfo *CallerBFI,
1408  BlockFrequencyInfo *CalleeBFI,
1409  const BasicBlock &CalleeEntryBlock) {
1411  for (auto const &Entry : VMap) {
1412  if (!isa<BasicBlock>(Entry.first) || !Entry.second)
1413  continue;
1414  auto *OrigBB = cast<BasicBlock>(Entry.first);
1415  auto *ClonedBB = cast<BasicBlock>(Entry.second);
1416  uint64_t Freq = CalleeBFI->getBlockFreq(OrigBB).getFrequency();
1417  if (!ClonedBBs.insert(ClonedBB).second) {
1418  // Multiple blocks in the callee might get mapped to one cloned block in
1419  // the caller since we prune the callee as we clone it. When that happens,
1420  // we want to use the maximum among the original blocks' frequencies.
1421  uint64_t NewFreq = CallerBFI->getBlockFreq(ClonedBB).getFrequency();
1422  if (NewFreq > Freq)
1423  Freq = NewFreq;
1424  }
1425  CallerBFI->setBlockFreq(ClonedBB, Freq);
1426  }
1427  BasicBlock *EntryClone = cast<BasicBlock>(VMap.lookup(&CalleeEntryBlock));
1428  CallerBFI->setBlockFreqAndScale(
1429  EntryClone, CallerBFI->getBlockFreq(CallSiteBlock).getFrequency(),
1430  ClonedBBs);
1431 }
1432 
1433 /// Update the branch metadata for cloned call instructions.
1435  const ProfileCount &CalleeEntryCount,
1436  const Instruction *TheCall,
1437  ProfileSummaryInfo *PSI,
1438  BlockFrequencyInfo *CallerBFI) {
1439  if (!CalleeEntryCount.hasValue() || CalleeEntryCount.isSynthetic() ||
1440  CalleeEntryCount.getCount() < 1)
1441  return;
1442  auto CallSiteCount = PSI ? PSI->getProfileCount(TheCall, CallerBFI) : None;
1443  uint64_t CallCount =
1444  std::min(CallSiteCount.hasValue() ? CallSiteCount.getValue() : 0,
1445  CalleeEntryCount.getCount());
1446 
1447  for (auto const &Entry : VMap)
1448  if (isa<CallInst>(Entry.first))
1449  if (auto *CI = dyn_cast_or_null<CallInst>(Entry.second))
1450  CI->updateProfWeight(CallCount, CalleeEntryCount.getCount());
1451  for (BasicBlock &BB : *Callee)
1452  // No need to update the callsite if it is pruned during inlining.
1453  if (VMap.count(&BB))
1454  for (Instruction &I : BB)
1455  if (CallInst *CI = dyn_cast<CallInst>(&I))
1456  CI->updateProfWeight(CalleeEntryCount.getCount() - CallCount,
1457  CalleeEntryCount.getCount());
1458 }
1459 
1460 /// Update the entry count of callee after inlining.
1461 ///
1462 /// The callsite's block count is subtracted from the callee's function entry
1463 /// count.
1464 static void updateCalleeCount(BlockFrequencyInfo *CallerBFI, BasicBlock *CallBB,
1466  ProfileSummaryInfo *PSI) {
1467  // If the callee has a original count of N, and the estimated count of
1468  // callsite is M, the new callee count is set to N - M. M is estimated from
1469  // the caller's entry count, its entry block frequency and the block frequency
1470  // of the callsite.
1471  auto CalleeCount = Callee->getEntryCount();
1472  if (!CalleeCount.hasValue() || !PSI)
1473  return;
1474  auto CallCount = PSI->getProfileCount(CallInst, CallerBFI);
1475  if (!CallCount.hasValue())
1476  return;
1477  // Since CallSiteCount is an estimate, it could exceed the original callee
1478  // count and has to be set to 0.
1479  if (CallCount.getValue() > CalleeCount.getCount())
1480  CalleeCount.setCount(0);
1481  else
1482  CalleeCount.setCount(CalleeCount.getCount() - CallCount.getValue());
1483  Callee->setEntryCount(CalleeCount);
1484 }
1485 
1486 /// This function inlines the called function into the basic block of the
1487 /// caller. This returns false if it is not possible to inline this call.
1488 /// The program is still in a well defined state if this occurs though.
1489 ///
1490 /// Note that this only does one level of inlining. For example, if the
1491 /// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now
1492 /// exists in the instruction stream. Similarly this will inline a recursive
1493 /// function by one level.
1495  AAResults *CalleeAAR, bool InsertLifetime,
1496  Function *ForwardVarArgsTo) {
1497  Instruction *TheCall = CS.getInstruction();
1498  assert(TheCall->getParent() && TheCall->getFunction()
1499  && "Instruction not in function!");
1500 
1501  // If IFI has any state in it, zap it before we fill it in.
1502  IFI.reset();
1503 
1504  Function *CalledFunc = CS.getCalledFunction();
1505  if (!CalledFunc || // Can't inline external function or indirect
1506  CalledFunc->isDeclaration()) // call!
1507  return false;
1508 
1509  // The inliner does not know how to inline through calls with operand bundles
1510  // in general ...
1511  if (CS.hasOperandBundles()) {
1512  for (int i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
1514  // ... but it knows how to inline through "deopt" operand bundles ...
1515  if (Tag == LLVMContext::OB_deopt)
1516  continue;
1517  // ... and "funclet" operand bundles.
1518  if (Tag == LLVMContext::OB_funclet)
1519  continue;
1520 
1521  return false;
1522  }
1523  }
1524 
1525  // If the call to the callee cannot throw, set the 'nounwind' flag on any
1526  // calls that we inline.
1527  bool MarkNoUnwind = CS.doesNotThrow();
1528 
1529  BasicBlock *OrigBB = TheCall->getParent();
1530  Function *Caller = OrigBB->getParent();
1531 
1532  // GC poses two hazards to inlining, which only occur when the callee has GC:
1533  // 1. If the caller has no GC, then the callee's GC must be propagated to the
1534  // caller.
1535  // 2. If the caller has a differing GC, it is invalid to inline.
1536  if (CalledFunc->hasGC()) {
1537  if (!Caller->hasGC())
1538  Caller->setGC(CalledFunc->getGC());
1539  else if (CalledFunc->getGC() != Caller->getGC())
1540  return false;
1541  }
1542 
1543  // Get the personality function from the callee if it contains a landing pad.
1544  Constant *CalledPersonality =
1545  CalledFunc->hasPersonalityFn()
1546  ? CalledFunc->getPersonalityFn()->stripPointerCasts()
1547  : nullptr;
1548 
1549  // Find the personality function used by the landing pads of the caller. If it
1550  // exists, then check to see that it matches the personality function used in
1551  // the callee.
1552  Constant *CallerPersonality =
1553  Caller->hasPersonalityFn()
1554  ? Caller->getPersonalityFn()->stripPointerCasts()
1555  : nullptr;
1556  if (CalledPersonality) {
1557  if (!CallerPersonality)
1558  Caller->setPersonalityFn(CalledPersonality);
1559  // If the personality functions match, then we can perform the
1560  // inlining. Otherwise, we can't inline.
1561  // TODO: This isn't 100% true. Some personality functions are proper
1562  // supersets of others and can be used in place of the other.
1563  else if (CalledPersonality != CallerPersonality)
1564  return false;
1565  }
1566 
1567  // We need to figure out which funclet the callsite was in so that we may
1568  // properly nest the callee.
1569  Instruction *CallSiteEHPad = nullptr;
1570  if (CallerPersonality) {
1571  EHPersonality Personality = classifyEHPersonality(CallerPersonality);
1572  if (isScopedEHPersonality(Personality)) {
1573  Optional<OperandBundleUse> ParentFunclet =
1575  if (ParentFunclet)
1576  CallSiteEHPad = cast<FuncletPadInst>(ParentFunclet->Inputs.front());
1577 
1578  // OK, the inlining site is legal. What about the target function?
1579 
1580  if (CallSiteEHPad) {
1581  if (Personality == EHPersonality::MSVC_CXX) {
1582  // The MSVC personality cannot tolerate catches getting inlined into
1583  // cleanup funclets.
1584  if (isa<CleanupPadInst>(CallSiteEHPad)) {
1585  // Ok, the call site is within a cleanuppad. Let's check the callee
1586  // for catchpads.
1587  for (const BasicBlock &CalledBB : *CalledFunc) {
1588  if (isa<CatchSwitchInst>(CalledBB.getFirstNonPHI()))
1589  return false;
1590  }
1591  }
1592  } else if (isAsynchronousEHPersonality(Personality)) {
1593  // SEH is even less tolerant, there may not be any sort of exceptional
1594  // funclet in the callee.
1595  for (const BasicBlock &CalledBB : *CalledFunc) {
1596  if (CalledBB.isEHPad())
1597  return false;
1598  }
1599  }
1600  }
1601  }
1602  }
1603 
1604  // Determine if we are dealing with a call in an EHPad which does not unwind
1605  // to caller.
1606  bool EHPadForCallUnwindsLocally = false;
1607  if (CallSiteEHPad && CS.isCall()) {
1608  UnwindDestMemoTy FuncletUnwindMap;
1609  Value *CallSiteUnwindDestToken =
1610  getUnwindDestToken(CallSiteEHPad, FuncletUnwindMap);
1611 
1612  EHPadForCallUnwindsLocally =
1613  CallSiteUnwindDestToken &&
1614  !isa<ConstantTokenNone>(CallSiteUnwindDestToken);
1615  }
1616 
1617  // Get an iterator to the last basic block in the function, which will have
1618  // the new function inlined after it.
1619  Function::iterator LastBlock = --Caller->end();
1620 
1621  // Make sure to capture all of the return instructions from the cloned
1622  // function.
1624  ClonedCodeInfo InlinedFunctionInfo;
1625  Function::iterator FirstNewBlock;
1626 
1627  { // Scope to destroy VMap after cloning.
1628  ValueToValueMapTy VMap;
1629  // Keep a list of pair (dst, src) to emit byval initializations.
1631 
1632  auto &DL = Caller->getParent()->getDataLayout();
1633 
1634  // Calculate the vector of arguments to pass into the function cloner, which
1635  // matches up the formal to the actual argument values.
1637  unsigned ArgNo = 0;
1638  for (Function::arg_iterator I = CalledFunc->arg_begin(),
1639  E = CalledFunc->arg_end(); I != E; ++I, ++AI, ++ArgNo) {
1640  Value *ActualArg = *AI;
1641 
1642  // When byval arguments actually inlined, we need to make the copy implied
1643  // by them explicit. However, we don't do this if the callee is readonly
1644  // or readnone, because the copy would be unneeded: the callee doesn't
1645  // modify the struct.
1646  if (CS.isByValArgument(ArgNo)) {
1647  ActualArg = HandleByValArgument(ActualArg, TheCall, CalledFunc, IFI,
1648  CalledFunc->getParamAlignment(ArgNo));
1649  if (ActualArg != *AI)
1650  ByValInit.push_back(std::make_pair(ActualArg, (Value*) *AI));
1651  }
1652 
1653  VMap[&*I] = ActualArg;
1654  }
1655 
1656  // Add alignment assumptions if necessary. We do this before the inlined
1657  // instructions are actually cloned into the caller so that we can easily
1658  // check what will be known at the start of the inlined code.
1659  AddAlignmentAssumptions(CS, IFI);
1660 
1661  // We want the inliner to prune the code as it copies. We would LOVE to
1662  // have no dead or constant instructions leftover after inlining occurs
1663  // (which can happen, e.g., because an argument was constant), but we'll be
1664  // happy with whatever the cloner can do.
1665  CloneAndPruneFunctionInto(Caller, CalledFunc, VMap,
1666  /*ModuleLevelChanges=*/false, Returns, ".i",
1667  &InlinedFunctionInfo, TheCall);
1668  // Remember the first block that is newly cloned over.
1669  FirstNewBlock = LastBlock; ++FirstNewBlock;
1670 
1671  if (IFI.CallerBFI != nullptr && IFI.CalleeBFI != nullptr)
1672  // Update the BFI of blocks cloned into the caller.
1673  updateCallerBFI(OrigBB, VMap, IFI.CallerBFI, IFI.CalleeBFI,
1674  CalledFunc->front());
1675 
1676  updateCallProfile(CalledFunc, VMap, CalledFunc->getEntryCount(), TheCall,
1677  IFI.PSI, IFI.CallerBFI);
1678  // Update the profile count of callee.
1679  updateCalleeCount(IFI.CallerBFI, OrigBB, TheCall, CalledFunc, IFI.PSI);
1680 
1681  // Inject byval arguments initialization.
1682  for (std::pair<Value*, Value*> &Init : ByValInit)
1683  HandleByValArgumentInit(Init.first, Init.second, Caller->getParent(),
1684  &*FirstNewBlock, IFI);
1685 
1686  Optional<OperandBundleUse> ParentDeopt =
1688  if (ParentDeopt) {
1690 
1691  for (auto &VH : InlinedFunctionInfo.OperandBundleCallSites) {
1692  Instruction *I = dyn_cast_or_null<Instruction>(VH);
1693  if (!I) continue; // instruction was DCE'd or RAUW'ed to undef
1694 
1695  OpDefs.clear();
1696 
1697  CallSite ICS(I);
1698  OpDefs.reserve(ICS.getNumOperandBundles());
1699 
1700  for (unsigned i = 0, e = ICS.getNumOperandBundles(); i < e; ++i) {
1701  auto ChildOB = ICS.getOperandBundleAt(i);
1702  if (ChildOB.getTagID() != LLVMContext::OB_deopt) {
1703  // If the inlined call has other operand bundles, let them be
1704  OpDefs.emplace_back(ChildOB);
1705  continue;
1706  }
1707 
1708  // It may be useful to separate this logic (of handling operand
1709  // bundles) out to a separate "policy" component if this gets crowded.
1710  // Prepend the parent's deoptimization continuation to the newly
1711  // inlined call's deoptimization continuation.
1712  std::vector<Value *> MergedDeoptArgs;
1713  MergedDeoptArgs.reserve(ParentDeopt->Inputs.size() +
1714  ChildOB.Inputs.size());
1715 
1716  MergedDeoptArgs.insert(MergedDeoptArgs.end(),
1717  ParentDeopt->Inputs.begin(),
1718  ParentDeopt->Inputs.end());
1719  MergedDeoptArgs.insert(MergedDeoptArgs.end(), ChildOB.Inputs.begin(),
1720  ChildOB.Inputs.end());
1721 
1722  OpDefs.emplace_back("deopt", std::move(MergedDeoptArgs));
1723  }
1724 
1725  Instruction *NewI = nullptr;
1726  if (isa<CallInst>(I))
1727  NewI = CallInst::Create(cast<CallInst>(I), OpDefs, I);
1728  else
1729  NewI = InvokeInst::Create(cast<InvokeInst>(I), OpDefs, I);
1730 
1731  // Note: the RAUW does the appropriate fixup in VMap, so we need to do
1732  // this even if the call returns void.
1733  I->replaceAllUsesWith(NewI);
1734 
1735  VH = nullptr;
1736  I->eraseFromParent();
1737  }
1738  }
1739 
1740  // Update the callgraph if requested.
1741  if (IFI.CG)
1742  UpdateCallGraphAfterInlining(CS, FirstNewBlock, VMap, IFI);
1743 
1744  // For 'nodebug' functions, the associated DISubprogram is always null.
1745  // Conservatively avoid propagating the callsite debug location to
1746  // instructions inlined from a function whose DISubprogram is not null.
1747  fixupLineNumbers(Caller, FirstNewBlock, TheCall,
1748  CalledFunc->getSubprogram() != nullptr);
1749 
1750  // Clone existing noalias metadata if necessary.
1751  CloneAliasScopeMetadata(CS, VMap);
1752 
1753  // Add noalias metadata if necessary.
1754  AddAliasScopeMetadata(CS, VMap, DL, CalleeAAR);
1755 
1756  // Propagate llvm.mem.parallel_loop_access if necessary.
1758 
1759  // Register any cloned assumptions.
1760  if (IFI.GetAssumptionCache)
1761  for (BasicBlock &NewBlock :
1762  make_range(FirstNewBlock->getIterator(), Caller->end()))
1763  for (Instruction &I : NewBlock) {
1764  if (auto *II = dyn_cast<IntrinsicInst>(&I))
1765  if (II->getIntrinsicID() == Intrinsic::assume)
1766  (*IFI.GetAssumptionCache)(*Caller).registerAssumption(II);
1767  }
1768  }
1769 
1770  // If there are any alloca instructions in the block that used to be the entry
1771  // block for the callee, move them to the entry block of the caller. First
1772  // calculate which instruction they should be inserted before. We insert the
1773  // instructions at the end of the current alloca list.
1774  {
1775  BasicBlock::iterator InsertPoint = Caller->begin()->begin();
1776  for (BasicBlock::iterator I = FirstNewBlock->begin(),
1777  E = FirstNewBlock->end(); I != E; ) {
1778  AllocaInst *AI = dyn_cast<AllocaInst>(I++);
1779  if (!AI) continue;
1780 
1781  // If the alloca is now dead, remove it. This often occurs due to code
1782  // specialization.
1783  if (AI->use_empty()) {
1784  AI->eraseFromParent();
1785  continue;
1786  }
1787 
1788  if (!allocaWouldBeStaticInEntry(AI))
1789  continue;
1790 
1791  // Keep track of the static allocas that we inline into the caller.
1792  IFI.StaticAllocas.push_back(AI);
1793 
1794  // Scan for the block of allocas that we can move over, and move them
1795  // all at once.
1796  while (isa<AllocaInst>(I) &&
1797  allocaWouldBeStaticInEntry(cast<AllocaInst>(I))) {
1798  IFI.StaticAllocas.push_back(cast<AllocaInst>(I));
1799  ++I;
1800  }
1801 
1802  // Transfer all of the allocas over in a block. Using splice means
1803  // that the instructions aren't removed from the symbol table, then
1804  // reinserted.
1805  Caller->getEntryBlock().getInstList().splice(
1806  InsertPoint, FirstNewBlock->getInstList(), AI->getIterator(), I);
1807  }
1808  // Move any dbg.declares describing the allocas into the entry basic block.
1809  DIBuilder DIB(*Caller->getParent());
1810  for (auto &AI : IFI.StaticAllocas)
1813  }
1814 
1815  SmallVector<Value*,4> VarArgsToForward;
1816  SmallVector<AttributeSet, 4> VarArgsAttrs;
1817  for (unsigned i = CalledFunc->getFunctionType()->getNumParams();
1818  i < CS.getNumArgOperands(); i++) {
1819  VarArgsToForward.push_back(CS.getArgOperand(i));
1820  VarArgsAttrs.push_back(CS.getAttributes().getParamAttributes(i));
1821  }
1822 
1823  bool InlinedMustTailCalls = false, InlinedDeoptimizeCalls = false;
1824  if (InlinedFunctionInfo.ContainsCalls) {
1825  CallInst::TailCallKind CallSiteTailKind = CallInst::TCK_None;
1826  if (CallInst *CI = dyn_cast<CallInst>(TheCall))
1827  CallSiteTailKind = CI->getTailCallKind();
1828 
1829  // For inlining purposes, the "notail" marker is the same as no marker.
1830  if (CallSiteTailKind == CallInst::TCK_NoTail)
1831  CallSiteTailKind = CallInst::TCK_None;
1832 
1833  for (Function::iterator BB = FirstNewBlock, E = Caller->end(); BB != E;
1834  ++BB) {
1835  for (auto II = BB->begin(); II != BB->end();) {
1836  Instruction &I = *II++;
1837  CallInst *CI = dyn_cast<CallInst>(&I);
1838  if (!CI)
1839  continue;
1840 
1841  // Forward varargs from inlined call site to calls to the
1842  // ForwardVarArgsTo function, if requested, and to musttail calls.
1843  if (!VarArgsToForward.empty() &&
1844  ((ForwardVarArgsTo &&
1845  CI->getCalledFunction() == ForwardVarArgsTo) ||
1846  CI->isMustTailCall())) {
1847  // Collect attributes for non-vararg parameters.
1850  if (!Attrs.isEmpty() || !VarArgsAttrs.empty()) {
1851  for (unsigned ArgNo = 0;
1852  ArgNo < CI->getFunctionType()->getNumParams(); ++ArgNo)
1853  ArgAttrs.push_back(Attrs.getParamAttributes(ArgNo));
1854  }
1855 
1856  // Add VarArg attributes.
1857  ArgAttrs.append(VarArgsAttrs.begin(), VarArgsAttrs.end());
1858  Attrs = AttributeList::get(CI->getContext(), Attrs.getFnAttributes(),
1859  Attrs.getRetAttributes(), ArgAttrs);
1860  // Add VarArgs to existing parameters.
1861  SmallVector<Value *, 6> Params(CI->arg_operands());
1862  Params.append(VarArgsToForward.begin(), VarArgsToForward.end());
1863  CallInst *NewCI =
1865  : CI->getCalledValue(),
1866  Params, "", CI);
1867  NewCI->setDebugLoc(CI->getDebugLoc());
1868  NewCI->setAttributes(Attrs);
1869  NewCI->setCallingConv(CI->getCallingConv());
1870  CI->replaceAllUsesWith(NewCI);
1871  CI->eraseFromParent();
1872  CI = NewCI;
1873  }
1874 
1875  if (Function *F = CI->getCalledFunction())
1876  InlinedDeoptimizeCalls |=
1877  F->getIntrinsicID() == Intrinsic::experimental_deoptimize;
1878 
1879  // We need to reduce the strength of any inlined tail calls. For
1880  // musttail, we have to avoid introducing potential unbounded stack
1881  // growth. For example, if functions 'f' and 'g' are mutually recursive
1882  // with musttail, we can inline 'g' into 'f' so long as we preserve
1883  // musttail on the cloned call to 'f'. If either the inlined call site
1884  // or the cloned call site is *not* musttail, the program already has
1885  // one frame of stack growth, so it's safe to remove musttail. Here is
1886  // a table of example transformations:
1887  //
1888  // f -> musttail g -> musttail f ==> f -> musttail f
1889  // f -> musttail g -> tail f ==> f -> tail f
1890  // f -> g -> musttail f ==> f -> f
1891  // f -> g -> tail f ==> f -> f
1892  //
1893  // Inlined notail calls should remain notail calls.
1894  CallInst::TailCallKind ChildTCK = CI->getTailCallKind();
1895  if (ChildTCK != CallInst::TCK_NoTail)
1896  ChildTCK = std::min(CallSiteTailKind, ChildTCK);
1897  CI->setTailCallKind(ChildTCK);
1898  InlinedMustTailCalls |= CI->isMustTailCall();
1899 
1900  // Calls inlined through a 'nounwind' call site should be marked
1901  // 'nounwind'.
1902  if (MarkNoUnwind)
1903  CI->setDoesNotThrow();
1904  }
1905  }
1906  }
1907 
1908  // Leave lifetime markers for the static alloca's, scoping them to the
1909  // function we just inlined.
1910  if (InsertLifetime && !IFI.StaticAllocas.empty()) {
1911  IRBuilder<> builder(&FirstNewBlock->front());
1912  for (unsigned ai = 0, ae = IFI.StaticAllocas.size(); ai != ae; ++ai) {
1913  AllocaInst *AI = IFI.StaticAllocas[ai];
1914  // Don't mark swifterror allocas. They can't have bitcast uses.
1915  if (AI->isSwiftError())
1916  continue;
1917 
1918  // If the alloca is already scoped to something smaller than the whole
1919  // function then there's no need to add redundant, less accurate markers.
1920  if (hasLifetimeMarkers(AI))
1921  continue;
1922 
1923  // Try to determine the size of the allocation.
1924  ConstantInt *AllocaSize = nullptr;
1925  if (ConstantInt *AIArraySize =
1926  dyn_cast<ConstantInt>(AI->getArraySize())) {
1927  auto &DL = Caller->getParent()->getDataLayout();
1928  Type *AllocaType = AI->getAllocatedType();
1929  uint64_t AllocaTypeSize = DL.getTypeAllocSize(AllocaType);
1930  uint64_t AllocaArraySize = AIArraySize->getLimitedValue();
1931 
1932  // Don't add markers for zero-sized allocas.
1933  if (AllocaArraySize == 0)
1934  continue;
1935 
1936  // Check that array size doesn't saturate uint64_t and doesn't
1937  // overflow when it's multiplied by type size.
1938  if (AllocaArraySize != std::numeric_limits<uint64_t>::max() &&
1939  std::numeric_limits<uint64_t>::max() / AllocaArraySize >=
1940  AllocaTypeSize) {
1941  AllocaSize = ConstantInt::get(Type::getInt64Ty(AI->getContext()),
1942  AllocaArraySize * AllocaTypeSize);
1943  }
1944  }
1945 
1946  builder.CreateLifetimeStart(AI, AllocaSize);
1947  for (ReturnInst *RI : Returns) {
1948  // Don't insert llvm.lifetime.end calls between a musttail or deoptimize
1949  // call and a return. The return kills all local allocas.
1950  if (InlinedMustTailCalls &&
1952  continue;
1953  if (InlinedDeoptimizeCalls &&
1955  continue;
1956  IRBuilder<>(RI).CreateLifetimeEnd(AI, AllocaSize);
1957  }
1958  }
1959  }
1960 
1961  // If the inlined code contained dynamic alloca instructions, wrap the inlined
1962  // code with llvm.stacksave/llvm.stackrestore intrinsics.
1963  if (InlinedFunctionInfo.ContainsDynamicAllocas) {
1964  Module *M = Caller->getParent();
1965  // Get the two intrinsics we care about.
1966  Function *StackSave = Intrinsic::getDeclaration(M, Intrinsic::stacksave);
1967  Function *StackRestore=Intrinsic::getDeclaration(M,Intrinsic::stackrestore);
1968 
1969  // Insert the llvm.stacksave.
1970  CallInst *SavedPtr = IRBuilder<>(&*FirstNewBlock, FirstNewBlock->begin())
1971  .CreateCall(StackSave, {}, "savedstack");
1972 
1973  // Insert a call to llvm.stackrestore before any return instructions in the
1974  // inlined function.
1975  for (ReturnInst *RI : Returns) {
1976  // Don't insert llvm.stackrestore calls between a musttail or deoptimize
1977  // call and a return. The return will restore the stack pointer.
1978  if (InlinedMustTailCalls && RI->getParent()->getTerminatingMustTailCall())
1979  continue;
1980  if (InlinedDeoptimizeCalls && RI->getParent()->getTerminatingDeoptimizeCall())
1981  continue;
1982  IRBuilder<>(RI).CreateCall(StackRestore, SavedPtr);
1983  }
1984  }
1985 
1986  // If we are inlining for an invoke instruction, we must make sure to rewrite
1987  // any call instructions into invoke instructions. This is sensitive to which
1988  // funclet pads were top-level in the inlinee, so must be done before
1989  // rewriting the "parent pad" links.
1990  if (auto *II = dyn_cast<InvokeInst>(TheCall)) {
1991  BasicBlock *UnwindDest = II->getUnwindDest();
1992  Instruction *FirstNonPHI = UnwindDest->getFirstNonPHI();
1993  if (isa<LandingPadInst>(FirstNonPHI)) {
1994  HandleInlinedLandingPad(II, &*FirstNewBlock, InlinedFunctionInfo);
1995  } else {
1996  HandleInlinedEHPad(II, &*FirstNewBlock, InlinedFunctionInfo);
1997  }
1998  }
1999 
2000  // Update the lexical scopes of the new funclets and callsites.
2001  // Anything that had 'none' as its parent is now nested inside the callsite's
2002  // EHPad.
2003 
2004  if (CallSiteEHPad) {
2005  for (Function::iterator BB = FirstNewBlock->getIterator(),
2006  E = Caller->end();
2007  BB != E; ++BB) {
2008  // Add bundle operands to any top-level call sites.
2010  for (BasicBlock::iterator BBI = BB->begin(), E = BB->end(); BBI != E;) {
2011  Instruction *I = &*BBI++;
2012  CallSite CS(I);
2013  if (!CS)
2014  continue;
2015 
2016  // Skip call sites which are nounwind intrinsics.
2017  auto *CalledFn =
2019  if (CalledFn && CalledFn->isIntrinsic() && CS.doesNotThrow())
2020  continue;
2021 
2022  // Skip call sites which already have a "funclet" bundle.
2024  continue;
2025 
2026  CS.getOperandBundlesAsDefs(OpBundles);
2027  OpBundles.emplace_back("funclet", CallSiteEHPad);
2028 
2029  Instruction *NewInst;
2030  if (CS.isCall())
2031  NewInst = CallInst::Create(cast<CallInst>(I), OpBundles, I);
2032  else
2033  NewInst = InvokeInst::Create(cast<InvokeInst>(I), OpBundles, I);
2034  NewInst->takeName(I);
2035  I->replaceAllUsesWith(NewInst);
2036  I->eraseFromParent();
2037 
2038  OpBundles.clear();
2039  }
2040 
2041  // It is problematic if the inlinee has a cleanupret which unwinds to
2042  // caller and we inline it into a call site which doesn't unwind but into
2043  // an EH pad that does. Such an edge must be dynamically unreachable.
2044  // As such, we replace the cleanupret with unreachable.
2045  if (auto *CleanupRet = dyn_cast<CleanupReturnInst>(BB->getTerminator()))
2046  if (CleanupRet->unwindsToCaller() && EHPadForCallUnwindsLocally)
2047  changeToUnreachable(CleanupRet, /*UseLLVMTrap=*/false);
2048 
2049  Instruction *I = BB->getFirstNonPHI();
2050  if (!I->isEHPad())
2051  continue;
2052 
2053  if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I)) {
2054  if (isa<ConstantTokenNone>(CatchSwitch->getParentPad()))
2055  CatchSwitch->setParentPad(CallSiteEHPad);
2056  } else {
2057  auto *FPI = cast<FuncletPadInst>(I);
2058  if (isa<ConstantTokenNone>(FPI->getParentPad()))
2059  FPI->setParentPad(CallSiteEHPad);
2060  }
2061  }
2062  }
2063 
2064  if (InlinedDeoptimizeCalls) {
2065  // We need to at least remove the deoptimizing returns from the Return set,
2066  // so that the control flow from those returns does not get merged into the
2067  // caller (but terminate it instead). If the caller's return type does not
2068  // match the callee's return type, we also need to change the return type of
2069  // the intrinsic.
2070  if (Caller->getReturnType() == TheCall->getType()) {
2071  auto NewEnd = llvm::remove_if(Returns, [](ReturnInst *RI) {
2072  return RI->getParent()->getTerminatingDeoptimizeCall() != nullptr;
2073  });
2074  Returns.erase(NewEnd, Returns.end());
2075  } else {
2076  SmallVector<ReturnInst *, 8> NormalReturns;
2077  Function *NewDeoptIntrinsic = Intrinsic::getDeclaration(
2078  Caller->getParent(), Intrinsic::experimental_deoptimize,
2079  {Caller->getReturnType()});
2080 
2081  for (ReturnInst *RI : Returns) {
2082  CallInst *DeoptCall = RI->getParent()->getTerminatingDeoptimizeCall();
2083  if (!DeoptCall) {
2084  NormalReturns.push_back(RI);
2085  continue;
2086  }
2087 
2088  // The calling convention on the deoptimize call itself may be bogus,
2089  // since the code we're inlining may have undefined behavior (and may
2090  // never actually execute at runtime); but all
2091  // @llvm.experimental.deoptimize declarations have to have the same
2092  // calling convention in a well-formed module.
2093  auto CallingConv = DeoptCall->getCalledFunction()->getCallingConv();
2094  NewDeoptIntrinsic->setCallingConv(CallingConv);
2095  auto *CurBB = RI->getParent();
2096  RI->eraseFromParent();
2097 
2098  SmallVector<Value *, 4> CallArgs(DeoptCall->arg_begin(),
2099  DeoptCall->arg_end());
2100 
2102  DeoptCall->getOperandBundlesAsDefs(OpBundles);
2103  DeoptCall->eraseFromParent();
2104  assert(!OpBundles.empty() &&
2105  "Expected at least the deopt operand bundle");
2106 
2107  IRBuilder<> Builder(CurBB);
2108  CallInst *NewDeoptCall =
2109  Builder.CreateCall(NewDeoptIntrinsic, CallArgs, OpBundles);
2110  NewDeoptCall->setCallingConv(CallingConv);
2111  if (NewDeoptCall->getType()->isVoidTy())
2112  Builder.CreateRetVoid();
2113  else
2114  Builder.CreateRet(NewDeoptCall);
2115  }
2116 
2117  // Leave behind the normal returns so we can merge control flow.
2118  std::swap(Returns, NormalReturns);
2119  }
2120  }
2121 
2122  // Handle any inlined musttail call sites. In order for a new call site to be
2123  // musttail, the source of the clone and the inlined call site must have been
2124  // musttail. Therefore it's safe to return without merging control into the
2125  // phi below.
2126  if (InlinedMustTailCalls) {
2127  // Check if we need to bitcast the result of any musttail calls.
2128  Type *NewRetTy = Caller->getReturnType();
2129  bool NeedBitCast = !TheCall->use_empty() && TheCall->getType() != NewRetTy;
2130 
2131  // Handle the returns preceded by musttail calls separately.
2132  SmallVector<ReturnInst *, 8> NormalReturns;
2133  for (ReturnInst *RI : Returns) {
2134  CallInst *ReturnedMustTail =
2136  if (!ReturnedMustTail) {
2137  NormalReturns.push_back(RI);
2138  continue;
2139  }
2140  if (!NeedBitCast)
2141  continue;
2142 
2143  // Delete the old return and any preceding bitcast.
2144  BasicBlock *CurBB = RI->getParent();
2145  auto *OldCast = dyn_cast_or_null<BitCastInst>(RI->getReturnValue());
2146  RI->eraseFromParent();
2147  if (OldCast)
2148  OldCast->eraseFromParent();
2149 
2150  // Insert a new bitcast and return with the right type.
2151  IRBuilder<> Builder(CurBB);
2152  Builder.CreateRet(Builder.CreateBitCast(ReturnedMustTail, NewRetTy));
2153  }
2154 
2155  // Leave behind the normal returns so we can merge control flow.
2156  std::swap(Returns, NormalReturns);
2157  }
2158 
2159  // Now that all of the transforms on the inlined code have taken place but
2160  // before we splice the inlined code into the CFG and lose track of which
2161  // blocks were actually inlined, collect the call sites. We only do this if
2162  // call graph updates weren't requested, as those provide value handle based
2163  // tracking of inlined call sites instead.
2164  if (InlinedFunctionInfo.ContainsCalls && !IFI.CG) {
2165  // Otherwise just collect the raw call sites that were inlined.
2166  for (BasicBlock &NewBB :
2167  make_range(FirstNewBlock->getIterator(), Caller->end()))
2168  for (Instruction &I : NewBB)
2169  if (auto CS = CallSite(&I))
2170  IFI.InlinedCallSites.push_back(CS);
2171  }
2172 
2173  // If we cloned in _exactly one_ basic block, and if that block ends in a
2174  // return instruction, we splice the body of the inlined callee directly into
2175  // the calling basic block.
2176  if (Returns.size() == 1 && std::distance(FirstNewBlock, Caller->end()) == 1) {
2177  // Move all of the instructions right before the call.
2178  OrigBB->getInstList().splice(TheCall->getIterator(),
2179  FirstNewBlock->getInstList(),
2180  FirstNewBlock->begin(), FirstNewBlock->end());
2181  // Remove the cloned basic block.
2182  Caller->getBasicBlockList().pop_back();
2183 
2184  // If the call site was an invoke instruction, add a branch to the normal
2185  // destination.
2186  if (InvokeInst *II = dyn_cast<InvokeInst>(TheCall)) {
2187  BranchInst *NewBr = BranchInst::Create(II->getNormalDest(), TheCall);
2188  NewBr->setDebugLoc(Returns[0]->getDebugLoc());
2189  }
2190 
2191  // If the return instruction returned a value, replace uses of the call with
2192  // uses of the returned value.
2193  if (!TheCall->use_empty()) {
2194  ReturnInst *R = Returns[0];
2195  if (TheCall == R->getReturnValue())
2196  TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType()));
2197  else
2198  TheCall->replaceAllUsesWith(R->getReturnValue());
2199  }
2200  // Since we are now done with the Call/Invoke, we can delete it.
2201  TheCall->eraseFromParent();
2202 
2203  // Since we are now done with the return instruction, delete it also.
2204  Returns[0]->eraseFromParent();
2205 
2206  // We are now done with the inlining.
2207  return true;
2208  }
2209 
2210  // Otherwise, we have the normal case, of more than one block to inline or
2211  // multiple return sites.
2212 
2213  // We want to clone the entire callee function into the hole between the
2214  // "starter" and "ender" blocks. How we accomplish this depends on whether
2215  // this is an invoke instruction or a call instruction.
2216  BasicBlock *AfterCallBB;
2217  BranchInst *CreatedBranchToNormalDest = nullptr;
2218  if (InvokeInst *II = dyn_cast<InvokeInst>(TheCall)) {
2219 
2220  // Add an unconditional branch to make this look like the CallInst case...
2221  CreatedBranchToNormalDest = BranchInst::Create(II->getNormalDest(), TheCall);
2222 
2223  // Split the basic block. This guarantees that no PHI nodes will have to be
2224  // updated due to new incoming edges, and make the invoke case more
2225  // symmetric to the call case.
2226  AfterCallBB =
2227  OrigBB->splitBasicBlock(CreatedBranchToNormalDest->getIterator(),
2228  CalledFunc->getName() + ".exit");
2229 
2230  } else { // It's a call
2231  // If this is a call instruction, we need to split the basic block that
2232  // the call lives in.
2233  //
2234  AfterCallBB = OrigBB->splitBasicBlock(TheCall->getIterator(),
2235  CalledFunc->getName() + ".exit");
2236  }
2237 
2238  if (IFI.CallerBFI) {
2239  // Copy original BB's block frequency to AfterCallBB
2240  IFI.CallerBFI->setBlockFreq(
2241  AfterCallBB, IFI.CallerBFI->getBlockFreq(OrigBB).getFrequency());
2242  }
2243 
2244  // Change the branch that used to go to AfterCallBB to branch to the first
2245  // basic block of the inlined function.
2246  //
2247  TerminatorInst *Br = OrigBB->getTerminator();
2248  assert(Br && Br->getOpcode() == Instruction::Br &&
2249  "splitBasicBlock broken!");
2250  Br->setOperand(0, &*FirstNewBlock);
2251 
2252  // Now that the function is correct, make it a little bit nicer. In
2253  // particular, move the basic blocks inserted from the end of the function
2254  // into the space made by splitting the source basic block.
2255  Caller->getBasicBlockList().splice(AfterCallBB->getIterator(),
2256  Caller->getBasicBlockList(), FirstNewBlock,
2257  Caller->end());
2258 
2259  // Handle all of the return instructions that we just cloned in, and eliminate
2260  // any users of the original call/invoke instruction.
2261  Type *RTy = CalledFunc->getReturnType();
2262 
2263  PHINode *PHI = nullptr;
2264  if (Returns.size() > 1) {
2265  // The PHI node should go at the front of the new basic block to merge all
2266  // possible incoming values.
2267  if (!TheCall->use_empty()) {
2268  PHI = PHINode::Create(RTy, Returns.size(), TheCall->getName(),
2269  &AfterCallBB->front());
2270  // Anything that used the result of the function call should now use the
2271  // PHI node as their operand.
2272  TheCall->replaceAllUsesWith(PHI);
2273  }
2274 
2275  // Loop over all of the return instructions adding entries to the PHI node
2276  // as appropriate.
2277  if (PHI) {
2278  for (unsigned i = 0, e = Returns.size(); i != e; ++i) {
2279  ReturnInst *RI = Returns[i];
2280  assert(RI->getReturnValue()->getType() == PHI->getType() &&
2281  "Ret value not consistent in function!");
2282  PHI->addIncoming(RI->getReturnValue(), RI->getParent());
2283  }
2284  }
2285 
2286  // Add a branch to the merge points and remove return instructions.
2287  DebugLoc Loc;
2288  for (unsigned i = 0, e = Returns.size(); i != e; ++i) {
2289  ReturnInst *RI = Returns[i];
2290  BranchInst* BI = BranchInst::Create(AfterCallBB, RI);
2291  Loc = RI->getDebugLoc();
2292  BI->setDebugLoc(Loc);
2293  RI->eraseFromParent();
2294  }
2295  // We need to set the debug location to *somewhere* inside the
2296  // inlined function. The line number may be nonsensical, but the
2297  // instruction will at least be associated with the right
2298  // function.
2299  if (CreatedBranchToNormalDest)
2300  CreatedBranchToNormalDest->setDebugLoc(Loc);
2301  } else if (!Returns.empty()) {
2302  // Otherwise, if there is exactly one return value, just replace anything
2303  // using the return value of the call with the computed value.
2304  if (!TheCall->use_empty()) {
2305  if (TheCall == Returns[0]->getReturnValue())
2306  TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType()));
2307  else
2308  TheCall->replaceAllUsesWith(Returns[0]->getReturnValue());
2309  }
2310 
2311  // Update PHI nodes that use the ReturnBB to use the AfterCallBB.
2312  BasicBlock *ReturnBB = Returns[0]->getParent();
2313  ReturnBB->replaceAllUsesWith(AfterCallBB);
2314 
2315  // Splice the code from the return block into the block that it will return
2316  // to, which contains the code that was after the call.
2317  AfterCallBB->getInstList().splice(AfterCallBB->begin(),
2318  ReturnBB->getInstList());
2319 
2320  if (CreatedBranchToNormalDest)
2321  CreatedBranchToNormalDest->setDebugLoc(Returns[0]->getDebugLoc());
2322 
2323  // Delete the return instruction now and empty ReturnBB now.
2324  Returns[0]->eraseFromParent();
2325  ReturnBB->eraseFromParent();
2326  } else if (!TheCall->use_empty()) {
2327  // No returns, but something is using the return value of the call. Just
2328  // nuke the result.
2329  TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType()));
2330  }
2331 
2332  // Since we are now done with the Call/Invoke, we can delete it.
2333  TheCall->eraseFromParent();
2334 
2335  // If we inlined any musttail calls and the original return is now
2336  // unreachable, delete it. It can only contain a bitcast and ret.
2337  if (InlinedMustTailCalls && pred_begin(AfterCallBB) == pred_end(AfterCallBB))
2338  AfterCallBB->eraseFromParent();
2339 
2340  // We should always be able to fold the entry block of the function into the
2341  // single predecessor of the block...
2342  assert(cast<BranchInst>(Br)->isUnconditional() && "splitBasicBlock broken!");
2343  BasicBlock *CalleeEntry = cast<BranchInst>(Br)->getSuccessor(0);
2344 
2345  // Splice the code entry block into calling block, right before the
2346  // unconditional branch.
2347  CalleeEntry->replaceAllUsesWith(OrigBB); // Update PHI nodes
2348  OrigBB->getInstList().splice(Br->getIterator(), CalleeEntry->getInstList());
2349 
2350  // Remove the unconditional branch.
2351  OrigBB->getInstList().erase(Br);
2352 
2353  // Now we can remove the CalleeEntry block, which is now empty.
2354  Caller->getBasicBlockList().erase(CalleeEntry);
2355 
2356  // If we inserted a phi node, check to see if it has a single value (e.g. all
2357  // the entries are the same or undef). If so, remove the PHI so it doesn't
2358  // block other optimizations.
2359  if (PHI) {
2360  AssumptionCache *AC =
2361  IFI.GetAssumptionCache ? &(*IFI.GetAssumptionCache)(*Caller) : nullptr;
2362  auto &DL = Caller->getParent()->getDataLayout();
2363  if (Value *V = SimplifyInstruction(PHI, {DL, nullptr, nullptr, AC})) {
2364  PHI->replaceAllUsesWith(V);
2365  PHI->eraseFromParent();
2366  }
2367  }
2368 
2369  return true;
2370 }
bool onlyReadsMemory() const
Determine if the function does not access or only reads memory.
Definition: Function.h:454
Return a value (possibly void), from a function.
bool isIntrinsic() const
isIntrinsic - Returns true if the function&#39;s name starts with "llvm.".
Definition: Function.h:185
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:68
void push_back(const T &Elt)
Definition: SmallVector.h:213
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
Function * getCalledFunction() const
Return the function called, or null if this is an indirect function invocation.
void removePredecessor(BasicBlock *Pred, bool DontDeleteUselessPHIs=false)
Notify the BasicBlock that the predecessor Pred is no longer able to reach it.
Definition: BasicBlock.cpp:295
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:22
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static cl::opt< bool > EnableNoAliasConversion("enable-noalias-to-md-conversion", cl::init(true), cl::Hidden, cl::desc("Convert noalias attributes to metadata during inlining."))
unsigned getOrEnforceKnownAlignment(Value *V, unsigned PrefAlign, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to ensure that the alignment of V is at least PrefAlign bytes.
Definition: Local.cpp:1165
static void updateCallProfile(Function *Callee, const ValueToValueMapTy &VMap, const ProfileCount &CalleeEntryCount, const Instruction *TheCall, ProfileSummaryInfo *PSI, BlockFrequencyInfo *CallerBFI)
Update the branch metadata for cloned call instructions.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:30
bool replaceDbgDeclareForAlloca(AllocaInst *AI, Value *NewAllocaAddress, DIBuilder &Builder, bool DerefBefore, int Offset, bool DerefAfter)
Replaces llvm.dbg.declare instruction when the alloca it describes is replaced with a new value...
Definition: Local.cpp:1509
iterator erase(iterator where)
Definition: ilist.h:267
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
CallGraph * CG
CG - If non-null, InlineFunction will update the callgraph to reflect the changes it makes...
Definition: Cloning.h:189
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:63
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:137
iterator end()
Definition: Function.h:644
an instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:514
static DebugLoc appendInlinedAt(DebugLoc DL, DILocation *InlinedAt, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode *> &Cache, bool ReplaceLast=false)
Rebuild the entire inlined-at chain for this instruction so that the top of the chain now is inlined-...
Definition: DebugLoc.cpp:70
std::function< AssumptionCache &(Function &)> * GetAssumptionCache
Definition: Cloning.h:190
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
unsigned getParamAlignment(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Definition: Function.h:414
Analysis providing profile information.
const CallInst * getTerminatingMustTailCall() const
Returns the call instruction marked &#39;musttail&#39; prior to the terminating return instruction of this ba...
Definition: BasicBlock.cpp:143
This class represents a function call, abstracting a target machine&#39;s calling convention.
void setGC(std::string Str)
Definition: Function.cpp:456
This file contains the declarations for metadata subclasses.
static void updateCalleeCount(BlockFrequencyInfo *CallerBFI, BasicBlock *CallBB, Instruction *CallInst, Function *Callee, ProfileSummaryInfo *PSI)
Update the entry count of callee after inlining.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
A cache of @llvm.assume calls within a function.
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:132
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:713
static void AddAlignmentAssumptions(CallSite CS, InlineFunctionInfo &IFI)
If the inlined function has non-byval align arguments, then add .assume-based alignment assumptions t...
Optional< uint64_t > getProfileCount(const Instruction *CallInst, BlockFrequencyInfo *BFI)
Returns the profile count for CallInst.
arg_iterator arg_end()
Definition: Function.h:666
A debug info location.
Definition: DebugLoc.h:34
Metadata node.
Definition: Metadata.h:862
F(f)
static CallInst * Create(Value *Func, ArrayRef< Value *> Args, ArrayRef< OperandBundleDef > Bundles=None, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1067
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:503
An instruction for reading from memory.
Definition: Instructions.h:164
static void AddAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap, const DataLayout &DL, AAResults *CalleeAAR)
If the inlined function has noalias arguments, then add new alias scopes for each noalias argument...
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:177
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:677
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - Get or set the calling convention of this function call...
Constant * getClause(unsigned Idx) const
Get the value of the clause at index Idx.
void reserve(size_type N)
Definition: SmallVector.h:377
bool isMustTailCall() const
InlineFunctionInfo - This class captures the data input to the InlineFunction call, and records the auxiliary results produced by it.
Definition: Cloning.h:176
iterator end()
Get an iterator to the end of the SetVector.
Definition: SetVector.h:93
A node in the call graph for a module.
Definition: CallGraph.h:165
Tuple of metadata.
Definition: Metadata.h:1104
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:130
static void fixupLineNumbers(Function *Fn, Function::iterator FI, Instruction *TheCall, bool CalleeHasDebugInfo)
Update inlined instructions&#39; line numbers to to encode location where these instructions are inlined...
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:344
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:264
Function::ProfileCount ProfileCount
bool InlineFunction(CallInst *C, InlineFunctionInfo &IFI, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true)
InlineFunction - This function inlines the called function into the basic block of the caller...
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:191
void addCalledFunction(CallSite CS, CallGraphNode *M)
Adds a function to the list of functions called by this one.
Definition: CallGraph.h:233
unsigned getAllocaAddrSpace() const
Definition: DataLayout.h:258
const CallInst * getTerminatingDeoptimizeCall() const
Returns the call instruction calling @llvm.experimental.deoptimize prior to the terminating return in...
Definition: BasicBlock.cpp:174
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:361
ProfileCount getEntryCount() const
Get the entry count for this function.
Definition: Function.cpp:1363
FunctionType * getFunctionType() const
void setEntryCount(ProfileCount Count, const DenseSet< GlobalValue::GUID > *Imports=nullptr)
Set the entry count for this function.
Definition: Function.cpp:1345
iterator end()
Definition: CallGraph.h:191
static void HandleInlinedLandingPad(InvokeInst *II, BasicBlock *FirstNewBlock, ClonedCodeInfo &InlinedCodeInfo)
If we inlined an invoke site, we need to convert calls in the body of the inlined function into invok...
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:97
TailCallKind getTailCallKind() const
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
uint32_t getTagID() const
Return the tag of this operand bundle as an integer.
Definition: InstrTypes.h:1250
static void updateCallerBFI(BasicBlock *CallSiteBlock, const ValueToValueMapTy &VMap, BlockFrequencyInfo *CallerBFI, BlockFrequencyInfo *CalleeBFI, const BasicBlock &CalleeEntryBlock)
Update the block frequencies of the caller after a callee has been inlined.
ReturnInst * CreateRet(Value *V)
Create a &#39;ret <val>&#39; instruction.
Definition: IRBuilder.h:817
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:731
AttributeSet getRetAttributes() const
The attributes for the ret value are returned.
InstrTy * getInstruction() const
Definition: CallSite.h:92
static void CloneAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap)
When inlining a function that contains noalias scope metadata, this metadata needs to be cloned so th...
The only memory references in this function (if it has any) are non-volatile loads from objects point...
void setBlockFreqAndScale(const BasicBlock *ReferenceBB, uint64_t Freq, SmallPtrSetImpl< BasicBlock *> &BlocksToScale)
Set the frequency of ReferenceBB to Freq and scale the frequencies of the blocks in BlocksToScale suc...
unsigned getNumClauses() const
Get the number of clauses for this landing pad.
std::vector< CallRecord > CalledFunctionsVector
Definition: CallGraph.h:172
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch, catchpad/ret, and cleanuppad/ret.
void addHandler(BasicBlock *Dest)
Add an entry to the switch instruction...
ValTy * getCalledValue() const
Return the pointer to function that is being called.
Definition: CallSite.h:100
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
static void UpdateCallGraphAfterInlining(CallSite CS, Function::iterator FirstNewBlock, ValueToValueMapTy &VMap, InlineFunctionInfo &IFI)
Once we have cloned code over from a callee into the caller, update the specified callgraph to reflec...
void CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, bool ModuleLevelChanges, SmallVectorImpl< ReturnInst *> &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, Instruction *TheCall=nullptr)
CloneAndPruneFunctionInto - This works exactly like CloneFunctionInto, except that it does some simpl...
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1629
static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB, ArrayRef< BasicBlock *> Preds, BranchInst *BI, bool HasLoopExit)
Update the PHI nodes in OrigBB to include the values coming from NewBB.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
std::vector< WeakTrackingVH > OperandBundleCallSites
All cloned call sites that have operand bundles attached are appended to this vector.
Definition: Cloning.h:79
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:142
LandingPadInst * getLandingPadInst() const
Get the landingpad instruction from the landing pad block (the unwind destination).
ValTy * getArgOperand(unsigned i) const
Definition: CallSite.h:297
LLVMContext & getContext() const
Definition: Metadata.h:922
FunctionModRefBehavior
Summary of how a function affects memory in the program.
AttributeSet getParamAttributes(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
bool isUsedWithInAlloca() const
Return true if this alloca is used as an inalloca argument to a call.
Definition: Instructions.h:121
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
iterator find(const KeyT &Val)
Definition: ValueMap.h:162
iterator begin()
Get an iterator to the beginning of the SetVector.
Definition: SetVector.h:83
BlockFrequencyInfo * CalleeBFI
Definition: Cloning.h:192
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:200
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:126
const std::string & getGC() const
Definition: Function.cpp:451
An instruction for storing to memory.
Definition: Instructions.h:306
bool hasPersonalityFn() const
Check whether this function has a personality function.
Definition: Function.h:688
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:439
Debug location.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:301
iterator begin()
Definition: Function.h:642
AttributeList getAttributes() const
Return the parameter attributes for this call.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:142
amdgpu Simplify well known AMD library false Value * Callee
Function * getDeclaration(Module *M, ID id, ArrayRef< Type *> Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1001
Value * getOperand(unsigned i) const
Definition: User.h:170
Class to represent pointers.
Definition: DerivedTypes.h:467
static BasicBlock * HandleCallsInBlockInlinedThroughInvoke(BasicBlock *BB, BasicBlock *UnwindEdge, UnwindDestMemoTy *FuncletUnwindMap=nullptr)
When we inline a basic block into an invoke, we have to turn all of the calls that can throw into inv...
bool isCall() const
Return true if a CallInst is enclosed.
Definition: CallSite.h:87
static bool isUsedByLifetimeMarker(Value *V)
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:146
static cl::opt< bool > PreserveAlignmentAssumptions("preserve-alignment-assumptions-during-inlining", cl::init(true), cl::Hidden, cl::desc("Convert align attributes to assumptions during inlining."))
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Definition: CallSite.h:555
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:141
const BasicBlock & getEntryBlock() const
Definition: Function.h:626
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
static TempMDTuple getTemporary(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Return a temporary node.
Definition: Metadata.h:1151
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1164
bool doesNotThrow() const
Determine if the call cannot unwind.
void setCallingConv(CallingConv::ID CC)
Definition: Function.h:203
SmallVector< CallSite, 8 > InlinedCallSites
All of the new call sites inlined into the caller.
Definition: Cloning.h:207
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:410
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:155
The landingpad instruction holds all of the information necessary to generate correct exception handl...
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:189
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:55
bool hasNUses(unsigned N) const
Return true if this Value has exactly N users.
Definition: Value.cpp:132
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:287
uint64_t getCount() const
Definition: Function.h:258
bool hasName() const
Definition: Value.h:251
LLVM Basic Block Representation.
Definition: BasicBlock.h:59
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1508
Conditional or Unconditional Branch instruction.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:42
ArrayRef< Use > Inputs
Definition: InstrTypes.h:1223
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:117
Resume the propagation of an exception.
static MDTuple * getDistinct(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1172
Value * getIncomingValueForBlock(const BasicBlock *BB) const
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:221
const Instruction & front() const
Definition: BasicBlock.h:276
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:139
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:371
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:113
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:732
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS)
Return the behavior of the given call site.
bool hasOperandBundles() const
Definition: CallSite.h:535
void splice(iterator where, iplist_impl &L2)
Definition: ilist.h:329
static Value * getUnwindDestTokenHelper(Instruction *EHPad, UnwindDestMemoTy &MemoMap)
Helper for getUnwindDestToken that does the descendant-ward part of the search.
ProfileSummaryInfo * PSI
Definition: Cloning.h:191
static bool hasLifetimeMarkers(AllocaInst *AI)
The only memory references in this function (if it has any) are non-volatile loads and stores from ob...
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:116
arg_iterator arg_begin()
Definition: Function.h:657
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:382
self_iterator getIterator()
Definition: ilist_node.h:82
void setTailCallKind(TailCallKind TCK)
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:60
auto remove_if(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range))
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:948
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:193
static UndefValue * get(Type *T)
Static factory methods - Return an &#39;undef&#39; object of the specified type.
Definition: Constants.cpp:1382
const Constant * stripPointerCasts() const
Definition: Constant.h:169
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:538
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
iterator erase(const_iterator CI)
Definition: SmallVector.h:446
iterator_range< User::op_iterator > arg_operands()
Iteration adapter for range-for loops.
unsigned getNumArgOperands() const
Definition: CallSite.h:293
Class to represent profile counts.
Definition: Function.h:247
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:93
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:220
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1226
static InvokeInst * Create(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value *> Args, const Twine &NameStr, Instruction *InsertBefore=nullptr)
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:102
void setCallingConv(CallingConv::ID CC)
bool isIdentifiedFunctionLocal(const Value *V)
Return true if V is umabigously identified at the function-level.
iterator end()
Definition: ValueMap.h:142
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
static void HandleByValArgumentInit(Value *Dst, Value *Src, Module *M, BasicBlock *InsertBlock, InlineFunctionInfo &IFI)
const InstListType & getInstList() const
Return the underlying instruction list container.
Definition: BasicBlock.h:329
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:298
Iterator for intrusive lists based on ilist_node.
This is the shared class of boolean and integer constants.
Definition: Constants.h:84
static CatchSwitchInst * Create(Value *ParentPad, BasicBlock *UnwindDest, unsigned NumHandlers, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
ValTy * getArgument(unsigned ArgNo) const
Definition: CallSite.h:186
OperandBundleUse getOperandBundleAt(unsigned Index) const
Definition: CallSite.h:551
iterator end()
Definition: BasicBlock.h:266
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:199
IterTy arg_begin() const
Definition: CallSite.h:571
Module.h This file contains the declarations for the Module class.
void setBlockFreq(const BasicBlock *BB, uint64_t Freq)
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:382
static Value * getParentPad(Value *EHPad)
Helper for getUnwindDestToken/getUnwindDestTokenHelper.
static cl::opt< bool > NoAliases("riscv-no-aliases", cl::desc("Disable the emission of assembler pseudo instructions"), cl::init(false), cl::Hidden)
static Value * getUnwindDestToken(Instruction *EHPad, UnwindDestMemoTy &MemoMap)
Given an EH pad, find where it unwinds.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:611
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
std::string utostr(uint64_t X, bool isNeg=false)
Definition: StringExtras.h:203
BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
void setOperand(unsigned i, Value *Val)
Definition: User.h:175
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:924
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:150
bool isCleanup() const
Return &#39;true&#39; if this landingpad instruction is a cleanup.
iterator_range< user_iterator > users()
Definition: Value.h:399
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:395
amdgpu Simplify well known AMD library false Value Value * Arg
bool ContainsCalls
ContainsCalls - This is set to true if the cloned code contains a normal call instruction.
Definition: Cloning.h:68
SmallVector< AllocaInst *, 4 > StaticAllocas
StaticAllocas - InlineFunction fills this in with all static allocas that get copied into the caller...
Definition: Cloning.h:196
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:74
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:121
FunTy * getCaller() const
Return the caller function for this call site.
Definition: CallSite.h:267
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:335
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:290
unsigned getNumOperandBundles() const
Definition: CallSite.h:531
void registerAssumption(CallInst *CI)
Add an @llvm.assume intrinsic to this function&#39;s cache.
void emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:653
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:62
SmallVector< WeakTrackingVH, 8 > InlinedCalls
InlinedCalls - InlineFunction fills this in with callsites that were inlined from the callee...
Definition: Cloning.h:200
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Definition: CallSite.h:582
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:1423
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:224
Establish a view to a call site for examination.
Definition: CallSite.h:713
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:108
static MDNode * concatenate(MDNode *A, MDNode *B)
Methods for metadata merging.
Definition: Metadata.cpp:896
static void PropagateParallelLoopAccessMetadata(CallSite CS, ValueToValueMapTy &VMap)
When inlining a call site that has !llvm.mem.parallel_loop_access metadata, that metadata should be p...
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink &#39;this&#39; from the containing function and delete it.
Definition: BasicBlock.cpp:115
#define I(x, y, z)
Definition: MD5.cpp:58
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:73
bool doesNotThrow() const
Determine if the call cannot unwind.
Definition: CallSite.h:505
static Value * HandleByValArgument(Value *Arg, Instruction *TheCall, const Function *CalledFunc, InlineFunctionInfo &IFI, unsigned ByValAlignment)
When inlining a call site that has a byval argument, we have to make the implicit memcpy explicit by ...
iterator end()
Definition: DenseMap.h:79
void setDoesNotThrow()
ClonedCodeInfo - This struct can be used to capture information about code being cloned, while it is being cloned.
Definition: Cloning.h:65
unsigned getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition: Local.h:260
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
const BasicBlockListType & getBasicBlockList() const
Get the underlying elements of the Function...
Definition: Function.h:619
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1097
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:401
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
std::vector< CallRecord >::iterator iterator
Definition: CallGraph.h:184
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:141
unsigned changeToUnreachable(Instruction *I, bool UseLLVMTrap, bool PreserveLCSSA=false, DeferredDominance *DDT=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:1713
bool PointerMayBeCapturedBefore(const Value *V, bool ReturnCaptures, bool StoreCaptures, const Instruction *I, const DominatorTree *DT, bool IncludeI=false, OrderedBasicBlock *OBB=nullptr)
PointerMayBeCapturedBefore - Return true if this pointer value may be captured by the enclosing funct...
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition: InstrTypes.h:1458
bool isByValArgument(unsigned ArgNo) const
Determine whether this argument is passed by value.
Definition: CallSite.h:598
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:201
FunTy * getCalledFunction() const
Return the function being called if this is a direct call, otherwise return null (if it&#39;s an indirect...
Definition: CallSite.h:107
void GetUnderlyingObjects(Value *V, SmallVectorImpl< Value *> &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
static bool allocaWouldBeStaticInEntry(const AllocaInst *AI)
Return the result of AI->isStaticAlloca() if AI were moved to the entry block.
BlockFrequencyInfo * CallerBFI
Definition: Cloning.h:192
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool ContainsDynamicAllocas
ContainsDynamicAllocas - This is set to true if the cloned code contains a &#39;dynamic&#39; alloca...
Definition: Cloning.h:74
const BasicBlock & front() const
Definition: Function.h:649
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
BasicBlock * getUnwindDest() const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:565
LLVM Value Representation.
Definition: Value.h:73
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1281
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:411
A vector that has set insertion semantics.
Definition: SetVector.h:41
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
void removeCallEdgeFor(CallSite CS)
Removes the edge in the node for the specified call site.
Definition: CallGraph.cpp:189
static CleanupReturnInst * Create(Value *CleanupPad, BasicBlock *UnwindBB=nullptr, Instruction *InsertBefore=nullptr)
AttributeSet getFnAttributes() const
The function attributes are returned.
BasicBlock * changeToInvokeAndSplitBasicBlock(CallInst *CI, BasicBlock *UnwindEdge)
Convert the CallInst to InvokeInst with the specified unwind edge basic block.
Definition: Local.cpp:1777
const Value * getCalledValue() const
Get a pointer to the function that is invoked by this instruction.
Invoke instruction.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
Definition: Instruction.h:552
iterator begin()
Definition: CallGraph.h:190
void setPersonalityFn(Constant *Fn)
Definition: Function.cpp:1286
AttributeList getAttributes() const
Get the parameter attributes of the call.
Definition: CallSite.h:329
const TerminatorInst * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:138
static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock, ClonedCodeInfo &InlinedCodeInfo)
If we inlined an invoke site, we need to convert calls in the body of the inlined function into invok...
void pop_back()
Definition: ilist.h:318
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1073
bool isEmpty() const
Return true if there are no attributes.
Definition: Attributes.h:645
Value * SimplifyInstruction(Instruction *I, const SimplifyQuery &Q, OptimizationRemarkEmitter *ORE=nullptr)
See if we can compute a simplified version of this instruction.
Root of the metadata hierarchy.
Definition: Metadata.h:58
bool use_empty() const
Definition: Value.h:322
iterator begin()
Definition: ValueMap.h:141
Type * getElementType() const
Definition: DerivedTypes.h:486
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:871
iterator_range< arg_iterator > args()
Definition: Function.h:675
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
const BasicBlock * getParent() const
Definition: Instruction.h:67
an instruction to allocate memory on the stack
Definition: Instructions.h:60
bool mayReadOrWriteMemory() const
Return true if this instruction may read or write memory.
Definition: Instruction.h:501