LLVM  10.0.0svn
Coroutines.cpp
Go to the documentation of this file.
1 //===- Coroutines.cpp -----------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the common infrastructure for Coroutine Passes.
10 //
11 //===----------------------------------------------------------------------===//
12 
15 #include "CoroInstr.h"
16 #include "CoroInternal.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
22 #include "llvm/IR/Attributes.h"
23 #include "llvm/IR/CallSite.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/DerivedTypes.h"
26 #include "llvm/IR/Function.h"
27 #include "llvm/IR/InstIterator.h"
28 #include "llvm/IR/Instructions.h"
29 #include "llvm/IR/IntrinsicInst.h"
30 #include "llvm/IR/Intrinsics.h"
32 #include "llvm/IR/Module.h"
33 #include "llvm/IR/Type.h"
34 #include "llvm/Support/Casting.h"
36 #include "llvm/Transforms/IPO.h"
38 #include <cassert>
39 #include <cstddef>
40 #include <utility>
41 
42 using namespace llvm;
43 
45  initializeCoroEarlyPass(Registry);
46  initializeCoroSplitPass(Registry);
47  initializeCoroElidePass(Registry);
48  initializeCoroCleanupPass(Registry);
49 }
50 
51 static void addCoroutineOpt0Passes(const PassManagerBuilder &Builder,
55 
58 }
59 
60 static void addCoroutineEarlyPasses(const PassManagerBuilder &Builder,
63 }
64 
68 }
69 
70 static void addCoroutineSCCPasses(const PassManagerBuilder &Builder,
73 }
74 
78 }
79 
91 }
92 
93 // Construct the lowerer base class and initialize its members.
95  : TheModule(M), Context(M.getContext()),
96  Int8Ptr(Type::getInt8PtrTy(Context)),
97  ResumeFnType(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
98  /*isVarArg=*/false)),
99  NullPtr(ConstantPointerNull::get(Int8Ptr)) {}
100 
101 // Creates a sequence of instructions to obtain a resume function address using
102 // llvm.coro.subfn.addr. It generates the following sequence:
103 //
104 // call i8* @llvm.coro.subfn.addr(i8* %Arg, i8 %index)
105 // bitcast i8* %2 to void(i8*)*
106 
108  Instruction *InsertPt) {
109  auto *IndexVal = ConstantInt::get(Type::getInt8Ty(Context), Index);
110  auto *Fn = Intrinsic::getDeclaration(&TheModule, Intrinsic::coro_subfn_addr);
111 
113  Index < CoroSubFnInst::IndexLast &&
114  "makeSubFnCall: Index value out of range");
115  auto *Call = CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt);
116 
117  auto *Bitcast =
118  new BitCastInst(Call, ResumeFnType->getPointerTo(), "", InsertPt);
119  return Bitcast;
120 }
121 
122 #ifndef NDEBUG
124  // NOTE: Must be sorted!
125  static const char *const CoroIntrinsics[] = {
126  "llvm.coro.alloc",
127  "llvm.coro.begin",
128  "llvm.coro.destroy",
129  "llvm.coro.done",
130  "llvm.coro.end",
131  "llvm.coro.frame",
132  "llvm.coro.free",
133  "llvm.coro.id",
134  "llvm.coro.id.retcon",
135  "llvm.coro.id.retcon.once",
136  "llvm.coro.noop",
137  "llvm.coro.param",
138  "llvm.coro.prepare.retcon",
139  "llvm.coro.promise",
140  "llvm.coro.resume",
141  "llvm.coro.save",
142  "llvm.coro.size",
143  "llvm.coro.subfn.addr",
144  "llvm.coro.suspend",
145  "llvm.coro.suspend.retcon",
146  };
147  return Intrinsic::lookupLLVMIntrinsicByName(CoroIntrinsics, Name) != -1;
148 }
149 #endif
150 
151 // Verifies if a module has named values listed. Also, in debug mode verifies
152 // that names are intrinsic names.
154  std::initializer_list<StringRef> List) {
155  for (StringRef Name : List) {
156  assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
157  if (M.getNamedValue(Name))
158  return true;
159  }
160 
161  return false;
162 }
163 
164 // Replace all coro.frees associated with the provided CoroId either with 'null'
165 // if Elide is true and with its frame parameter otherwise.
166 void coro::replaceCoroFree(CoroIdInst *CoroId, bool Elide) {
168  for (User *U : CoroId->users())
169  if (auto CF = dyn_cast<CoroFreeInst>(U))
170  CoroFrees.push_back(CF);
171 
172  if (CoroFrees.empty())
173  return;
174 
175  Value *Replacement =
177  : CoroFrees.front()->getFrame();
178 
179  for (CoroFreeInst *CF : CoroFrees) {
180  CF->replaceAllUsesWith(Replacement);
181  CF->eraseFromParent();
182  }
183 }
184 
185 // FIXME: This code is stolen from CallGraph::addToCallGraph(Function *F), which
186 // happens to be private. It is better for this functionality exposed by the
187 // CallGraph.
188 static void buildCGN(CallGraph &CG, CallGraphNode *Node) {
189  Function *F = Node->getFunction();
190 
191  // Look for calls by this function.
192  for (Instruction &I : instructions(F))
193  if (auto *Call = dyn_cast<CallBase>(&I)) {
194  const Function *Callee = Call->getCalledFunction();
195  if (!Callee || !Intrinsic::isLeaf(Callee->getIntrinsicID()))
196  // Indirect calls of intrinsics are not allowed so no need to check.
197  // We can be more precise here by using TargetArg returned by
198  // Intrinsic::isLeaf.
199  Node->addCalledFunction(Call, CG.getCallsExternalNode());
200  else if (!Callee->isIntrinsic())
201  Node->addCalledFunction(Call, CG.getOrInsertFunction(Callee));
202  }
203 }
204 
205 // Rebuild CGN after we extracted parts of the code from ParentFunc into
206 // NewFuncs. Builds CGNs for the NewFuncs and adds them to the current SCC.
208  CallGraph &CG, CallGraphSCC &SCC) {
209  // Rebuild CGN from scratch for the ParentFunc
210  auto *ParentNode = CG[&ParentFunc];
211  ParentNode->removeAllCalledFunctions();
212  buildCGN(CG, ParentNode);
213 
214  SmallVector<CallGraphNode *, 8> Nodes(SCC.begin(), SCC.end());
215 
216  for (Function *F : NewFuncs) {
218  Nodes.push_back(Callee);
219  buildCGN(CG, Callee);
220  }
221 
222  SCC.initialize(Nodes);
223 }
224 
225 static void clear(coro::Shape &Shape) {
226  Shape.CoroBegin = nullptr;
227  Shape.CoroEnds.clear();
228  Shape.CoroSizes.clear();
229  Shape.CoroSuspends.clear();
230 
231  Shape.FrameTy = nullptr;
232  Shape.FramePtr = nullptr;
233  Shape.AllocaSpillBlock = nullptr;
234 }
235 
237  CoroSuspendInst *SuspendInst) {
238  Module *M = SuspendInst->getModule();
239  auto *Fn = Intrinsic::getDeclaration(M, Intrinsic::coro_save);
240  auto *SaveInst =
241  cast<CoroSaveInst>(CallInst::Create(Fn, CoroBegin, "", SuspendInst));
242  assert(!SuspendInst->getCoroSave());
243  SuspendInst->setArgOperand(0, SaveInst);
244  return SaveInst;
245 }
246 
247 // Collect "interesting" coroutine intrinsics.
249  bool HasFinalSuspend = false;
250  size_t FinalSuspendIndex = 0;
251  clear(*this);
253  SmallVector<CoroSaveInst *, 2> UnusedCoroSaves;
254 
255  for (Instruction &I : instructions(F)) {
256  if (auto II = dyn_cast<IntrinsicInst>(&I)) {
257  switch (II->getIntrinsicID()) {
258  default:
259  continue;
260  case Intrinsic::coro_size:
261  CoroSizes.push_back(cast<CoroSizeInst>(II));
262  break;
263  case Intrinsic::coro_frame:
264  CoroFrames.push_back(cast<CoroFrameInst>(II));
265  break;
266  case Intrinsic::coro_save:
267  // After optimizations, coro_suspends using this coro_save might have
268  // been removed, remember orphaned coro_saves to remove them later.
269  if (II->use_empty())
270  UnusedCoroSaves.push_back(cast<CoroSaveInst>(II));
271  break;
272  case Intrinsic::coro_suspend_retcon: {
273  auto Suspend = cast<CoroSuspendRetconInst>(II);
274  CoroSuspends.push_back(Suspend);
275  break;
276  }
277  case Intrinsic::coro_suspend: {
278  auto Suspend = cast<CoroSuspendInst>(II);
279  CoroSuspends.push_back(Suspend);
280  if (Suspend->isFinal()) {
281  if (HasFinalSuspend)
283  "Only one suspend point can be marked as final");
284  HasFinalSuspend = true;
285  FinalSuspendIndex = CoroSuspends.size() - 1;
286  }
287  break;
288  }
289  case Intrinsic::coro_begin: {
290  auto CB = cast<CoroBeginInst>(II);
291 
292  // Ignore coro id's that aren't pre-split.
293  auto Id = dyn_cast<CoroIdInst>(CB->getId());
294  if (Id && !Id->getInfo().isPreSplit())
295  break;
296 
297  if (CoroBegin)
299  "coroutine should have exactly one defining @llvm.coro.begin");
300  CB->addAttribute(AttributeList::ReturnIndex, Attribute::NonNull);
302  CB->removeAttribute(AttributeList::FunctionIndex,
303  Attribute::NoDuplicate);
304  CoroBegin = CB;
305  break;
306  }
307  case Intrinsic::coro_end:
308  CoroEnds.push_back(cast<CoroEndInst>(II));
309  if (CoroEnds.back()->isFallthrough()) {
310  // Make sure that the fallthrough coro.end is the first element in the
311  // CoroEnds vector.
312  if (CoroEnds.size() > 1) {
313  if (CoroEnds.front()->isFallthrough())
315  "Only one coro.end can be marked as fallthrough");
316  std::swap(CoroEnds.front(), CoroEnds.back());
317  }
318  }
319  break;
320  }
321  }
322  }
323 
324  // If for some reason, we were not able to find coro.begin, bailout.
325  if (!CoroBegin) {
326  // Replace coro.frame which are supposed to be lowered to the result of
327  // coro.begin with undef.
329  for (CoroFrameInst *CF : CoroFrames) {
330  CF->replaceAllUsesWith(Undef);
331  CF->eraseFromParent();
332  }
333 
334  // Replace all coro.suspend with undef and remove related coro.saves if
335  // present.
336  for (AnyCoroSuspendInst *CS : CoroSuspends) {
337  CS->replaceAllUsesWith(UndefValue::get(CS->getType()));
338  CS->eraseFromParent();
339  if (auto *CoroSave = CS->getCoroSave())
340  CoroSave->eraseFromParent();
341  }
342 
343  // Replace all coro.ends with unreachable instruction.
344  for (CoroEndInst *CE : CoroEnds)
345  changeToUnreachable(CE, /*UseLLVMTrap=*/false);
346 
347  return;
348  }
349 
350  auto Id = CoroBegin->getId();
351  switch (auto IdIntrinsic = Id->getIntrinsicID()) {
352  case Intrinsic::coro_id: {
353  auto SwitchId = cast<CoroIdInst>(Id);
354  this->ABI = coro::ABI::Switch;
355  this->SwitchLowering.HasFinalSuspend = HasFinalSuspend;
356  this->SwitchLowering.ResumeSwitch = nullptr;
357  this->SwitchLowering.PromiseAlloca = SwitchId->getPromise();
358  this->SwitchLowering.ResumeEntryBlock = nullptr;
359 
360  for (auto AnySuspend : CoroSuspends) {
361  auto Suspend = dyn_cast<CoroSuspendInst>(AnySuspend);
362  if (!Suspend) {
363 #ifndef NDEBUG
364  AnySuspend->dump();
365 #endif
366  report_fatal_error("coro.id must be paired with coro.suspend");
367  }
368 
369  if (!Suspend->getCoroSave())
370  createCoroSave(CoroBegin, Suspend);
371  }
372  break;
373  }
374 
375  case Intrinsic::coro_id_retcon:
376  case Intrinsic::coro_id_retcon_once: {
377  auto ContinuationId = cast<AnyCoroIdRetconInst>(Id);
378  ContinuationId->checkWellFormed();
379  this->ABI = (IdIntrinsic == Intrinsic::coro_id_retcon
382  auto Prototype = ContinuationId->getPrototype();
383  this->RetconLowering.ResumePrototype = Prototype;
384  this->RetconLowering.Alloc = ContinuationId->getAllocFunction();
385  this->RetconLowering.Dealloc = ContinuationId->getDeallocFunction();
386  this->RetconLowering.ReturnBlock = nullptr;
387  this->RetconLowering.IsFrameInlineInStorage = false;
388 
389  // Determine the result value types, and make sure they match up with
390  // the values passed to the suspends.
391  auto ResultTys = getRetconResultTypes();
392  auto ResumeTys = getRetconResumeTypes();
393 
394  for (auto AnySuspend : CoroSuspends) {
395  auto Suspend = dyn_cast<CoroSuspendRetconInst>(AnySuspend);
396  if (!Suspend) {
397 #ifndef NDEBUG
398  AnySuspend->dump();
399 #endif
400  report_fatal_error("coro.id.retcon.* must be paired with "
401  "coro.suspend.retcon");
402  }
403 
404  // Check that the argument types of the suspend match the results.
405  auto SI = Suspend->value_begin(), SE = Suspend->value_end();
406  auto RI = ResultTys.begin(), RE = ResultTys.end();
407  for (; SI != SE && RI != RE; ++SI, ++RI) {
408  auto SrcTy = (*SI)->getType();
409  if (SrcTy != *RI) {
410  // The optimizer likes to eliminate bitcasts leading into variadic
411  // calls, but that messes with our invariants. Re-insert the
412  // bitcast and ignore this type mismatch.
413  if (CastInst::isBitCastable(SrcTy, *RI)) {
414  auto BCI = new BitCastInst(*SI, *RI, "", Suspend);
415  SI->set(BCI);
416  continue;
417  }
418 
419 #ifndef NDEBUG
420  Suspend->dump();
421  Prototype->getFunctionType()->dump();
422 #endif
423  report_fatal_error("argument to coro.suspend.retcon does not "
424  "match corresponding prototype function result");
425  }
426  }
427  if (SI != SE || RI != RE) {
428 #ifndef NDEBUG
429  Suspend->dump();
430  Prototype->getFunctionType()->dump();
431 #endif
432  report_fatal_error("wrong number of arguments to coro.suspend.retcon");
433  }
434 
435  // Check that the result type of the suspend matches the resume types.
436  Type *SResultTy = Suspend->getType();
437  ArrayRef<Type*> SuspendResultTys;
438  if (SResultTy->isVoidTy()) {
439  // leave as empty array
440  } else if (auto SResultStructTy = dyn_cast<StructType>(SResultTy)) {
441  SuspendResultTys = SResultStructTy->elements();
442  } else {
443  // forms an ArrayRef using SResultTy, be careful
444  SuspendResultTys = SResultTy;
445  }
446  if (SuspendResultTys.size() != ResumeTys.size()) {
447 #ifndef NDEBUG
448  Suspend->dump();
449  Prototype->getFunctionType()->dump();
450 #endif
451  report_fatal_error("wrong number of results from coro.suspend.retcon");
452  }
453  for (size_t I = 0, E = ResumeTys.size(); I != E; ++I) {
454  if (SuspendResultTys[I] != ResumeTys[I]) {
455 #ifndef NDEBUG
456  Suspend->dump();
457  Prototype->getFunctionType()->dump();
458 #endif
459  report_fatal_error("result from coro.suspend.retcon does not "
460  "match corresponding prototype function param");
461  }
462  }
463  }
464  break;
465  }
466 
467  default:
468  llvm_unreachable("coro.begin is not dependent on a coro.id call");
469  }
470 
471  // The coro.free intrinsic is always lowered to the result of coro.begin.
472  for (CoroFrameInst *CF : CoroFrames) {
473  CF->replaceAllUsesWith(CoroBegin);
474  CF->eraseFromParent();
475  }
476 
477  // Move final suspend to be the last element in the CoroSuspends vector.
478  if (ABI == coro::ABI::Switch &&
479  SwitchLowering.HasFinalSuspend &&
480  FinalSuspendIndex != CoroSuspends.size() - 1)
481  std::swap(CoroSuspends[FinalSuspendIndex], CoroSuspends.back());
482 
483  // Remove orphaned coro.saves.
484  for (CoroSaveInst *CoroSave : UnusedCoroSaves)
485  CoroSave->eraseFromParent();
486 }
487 
488 static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee) {
489  Call->setCallingConv(Callee->getCallingConv());
490  // TODO: attributes?
491 }
492 
493 static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee){
494  if (CG)
495  (*CG)[Call->getFunction()]->addCalledFunction(Call, (*CG)[Callee]);
496 }
497 
499  CallGraph *CG) const {
500  switch (ABI) {
501  case coro::ABI::Switch:
502  llvm_unreachable("can't allocate memory in coro switch-lowering");
503 
504  case coro::ABI::Retcon:
505  case coro::ABI::RetconOnce: {
506  auto Alloc = RetconLowering.Alloc;
507  Size = Builder.CreateIntCast(Size,
508  Alloc->getFunctionType()->getParamType(0),
509  /*is signed*/ false);
510  auto *Call = Builder.CreateCall(Alloc, Size);
511  propagateCallAttrsFromCallee(Call, Alloc);
512  addCallToCallGraph(CG, Call, Alloc);
513  return Call;
514  }
515  }
516  llvm_unreachable("Unknown coro::ABI enum");
517 }
518 
520  CallGraph *CG) const {
521  switch (ABI) {
522  case coro::ABI::Switch:
523  llvm_unreachable("can't allocate memory in coro switch-lowering");
524 
525  case coro::ABI::Retcon:
526  case coro::ABI::RetconOnce: {
527  auto Dealloc = RetconLowering.Dealloc;
528  Ptr = Builder.CreateBitCast(Ptr,
529  Dealloc->getFunctionType()->getParamType(0));
530  auto *Call = Builder.CreateCall(Dealloc, Ptr);
531  propagateCallAttrsFromCallee(Call, Dealloc);
532  addCallToCallGraph(CG, Call, Dealloc);
533  return;
534  }
535  }
536  llvm_unreachable("Unknown coro::ABI enum");
537 }
538 
540 static void fail(const Instruction *I, const char *Reason, Value *V) {
541 #ifndef NDEBUG
542  I->dump();
543  if (V) {
544  errs() << " Value: ";
546  errs() << '\n';
547  }
548 #endif
549  report_fatal_error(Reason);
550 }
551 
552 /// Check that the given value is a well-formed prototype for the
553 /// llvm.coro.id.retcon.* intrinsics.
555  auto F = dyn_cast<Function>(V->stripPointerCasts());
556  if (!F)
557  fail(I, "llvm.coro.id.retcon.* prototype not a Function", V);
558 
559  auto FT = F->getFunctionType();
560 
561  if (isa<CoroIdRetconInst>(I)) {
562  bool ResultOkay;
563  if (FT->getReturnType()->isPointerTy()) {
564  ResultOkay = true;
565  } else if (auto SRetTy = dyn_cast<StructType>(FT->getReturnType())) {
566  ResultOkay = (!SRetTy->isOpaque() &&
567  SRetTy->getNumElements() > 0 &&
568  SRetTy->getElementType(0)->isPointerTy());
569  } else {
570  ResultOkay = false;
571  }
572  if (!ResultOkay)
573  fail(I, "llvm.coro.id.retcon prototype must return pointer as first "
574  "result", F);
575 
576  if (FT->getReturnType() !=
578  fail(I, "llvm.coro.id.retcon prototype return type must be same as"
579  "current function return type", F);
580  } else {
581  // No meaningful validation to do here for llvm.coro.id.unique.once.
582  }
583 
584  if (FT->getNumParams() == 0 || !FT->getParamType(0)->isPointerTy())
585  fail(I, "llvm.coro.id.retcon.* prototype must take pointer as "
586  "its first parameter", F);
587 }
588 
589 /// Check that the given value is a well-formed allocator.
590 static void checkWFAlloc(const Instruction *I, Value *V) {
591  auto F = dyn_cast<Function>(V->stripPointerCasts());
592  if (!F)
593  fail(I, "llvm.coro.* allocator not a Function", V);
594 
595  auto FT = F->getFunctionType();
596  if (!FT->getReturnType()->isPointerTy())
597  fail(I, "llvm.coro.* allocator must return a pointer", F);
598 
599  if (FT->getNumParams() != 1 ||
600  !FT->getParamType(0)->isIntegerTy())
601  fail(I, "llvm.coro.* allocator must take integer as only param", F);
602 }
603 
604 /// Check that the given value is a well-formed deallocator.
605 static void checkWFDealloc(const Instruction *I, Value *V) {
606  auto F = dyn_cast<Function>(V->stripPointerCasts());
607  if (!F)
608  fail(I, "llvm.coro.* deallocator not a Function", V);
609 
610  auto FT = F->getFunctionType();
611  if (!FT->getReturnType()->isVoidTy())
612  fail(I, "llvm.coro.* deallocator must return void", F);
613 
614  if (FT->getNumParams() != 1 ||
615  !FT->getParamType(0)->isPointerTy())
616  fail(I, "llvm.coro.* deallocator must take pointer as only param", F);
617 }
618 
619 static void checkConstantInt(const Instruction *I, Value *V,
620  const char *Reason) {
621  if (!isa<ConstantInt>(V)) {
622  fail(I, Reason, V);
623  }
624 }
625 
627  checkConstantInt(this, getArgOperand(SizeArg),
628  "size argument to coro.id.retcon.* must be constant");
629  checkConstantInt(this, getArgOperand(AlignArg),
630  "alignment argument to coro.id.retcon.* must be constant");
631  checkWFRetconPrototype(this, getArgOperand(PrototypeArg));
632  checkWFAlloc(this, getArgOperand(AllocArg));
633  checkWFDealloc(this, getArgOperand(DeallocArg));
634 }
635 
637  unwrap(PM)->add(createCoroEarlyPass());
638 }
639 
641  unwrap(PM)->add(createCoroSplitPass());
642 }
643 
645  unwrap(PM)->add(createCoroElidePass());
646 }
647 
649  unwrap(PM)->add(createCoroCleanupPass());
650 }
bool isIntrinsic() const
isIntrinsic - Returns true if the function&#39;s name starts with "llvm.".
Definition: Function.h:198
This represents the llvm.coro.suspend.retcon instruction.
Definition: CoroInstr.h:404
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Instruction * FramePtr
Definition: CoroInternal.h:109
LLVMContext & Context
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
PassManagerBuilder - This class is used to set up a standard optimization sequence for languages like...
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:88
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
void initializeCoroEarlyPass(PassRegistry &)
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:65
void initializeCoroElidePass(PassRegistry &)
void LLVMAddCoroElidePass(LLVMPassManagerRef PM)
See llvm::createCoroElidePass function.
Definition: Coroutines.cpp:644
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
iterator end() const
A global registry used in conjunction with static constructors to make pluggable components (like tar...
Definition: Registry.h:44
static void buildCGN(CallGraph &CG, CallGraphNode *Node)
Definition: Coroutines.cpp:188
This class represents a function call, abstracting a target machine&#39;s calling convention.
EP_ScalarOptimizerLate - This extension point allows adding optimization passes after most of the mai...
The two locations do not alias at all.
Definition: AliasAnalysis.h:84
static void addCoroutineSCCPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:70
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:733
virtual void add(Pass *P)=0
Add a pass to the queue of passes to run.
void setArgOperand(unsigned i, Value *v)
Definition: InstrTypes.h:1246
Pass * createCoroEarlyPass()
Lower coroutine intrinsics that are not needed by later passes.
F(f)
A node in the call graph for a module.
Definition: CallGraph.h:164
void dump() const
Support for debugging, callable in GDB: V->dump()
Definition: AsmWriter.cpp:4428
static bool isCoroutineIntrinsicName(StringRef Name)
Definition: Coroutines.cpp:123
Attribute unwrap(LLVMAttributeRef Attr)
Definition: Attributes.h:204
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:654
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:372
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:779
This file contains the simple types necessary to represent the attributes associated with functions a...
void buildFrom(Function &F)
Definition: Coroutines.cpp:248
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
Class to represent function types.
Definition: DerivedTypes.h:103
void initializeCoroSplitPass(PassRegistry &)
EP_EnabledOnOptLevel0 - This extension point allows adding passes that should not be disabled by O0 o...
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1964
FunctionType *const ResumeFnType
Definition: CoroInternal.h:58
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:90
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:498
static void checkConstantInt(const Instruction *I, Value *V, const char *Reason)
Definition: Coroutines.cpp:619
static void addCoroutineOptimizerLastPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:75
This class represents a no-op cast from one type to another.
GlobalValue * getNamedValue(StringRef Name) const
Return the global value in the module with the specified name, of arbitrary type. ...
Definition: Module.cpp:113
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
Function * getDeclaration(Module *M, ID id, ArrayRef< Type *> Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1057
This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.
Definition: CoroInstr.h:204
static void addCoroutineScalarOptimizerPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:65
static void checkWFDealloc(const Instruction *I, Value *V)
Check that the given value is a well-formed deallocator.
Definition: Coroutines.cpp:605
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
iterator begin() const
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1432
bool isLeaf(ID id)
Returns true if the intrinsic is a leaf, i.e.
Definition: Function.cpp:1040
CallGraphNode * getCallsExternalNode() const
Definition: CallGraph.h:138
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
void LLVMAddCoroEarlyPass(LLVMPassManagerRef PM)
See llvm::createCoroEarlyPass function.
Definition: Coroutines.cpp:636
Pass * createCoroCleanupPass()
Lower all remaining coroutine intrinsics.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Pass * createCoroElidePass()
Analyze coroutines use sites, devirtualize resume/destroy calls and elide heap allocation for corouti...
Definition: CoroElide.cpp:343
static bool isBitCastable(Type *SrcTy, Type *DestTy)
Check whether a bitcast between these types is valid.
This represents the llvm.coro.end instruction.
Definition: CoroInstr.h:441
static void addCallToCallGraph(CallGraph *CG, CallInst *Call, Function *Callee)
Definition: Coroutines.cpp:493
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1348
ModulePass * createBarrierNoopPass()
createBarrierNoopPass - This pass is purely a module pass barrier in a pass manager.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This represents the llvm.coro.save instruction.
Definition: CoroInstr.h:325
void initialize(ArrayRef< CallGraphNode *> NewNodes)
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:59
static void checkWFAlloc(const Instruction *I, Value *V)
Check that the given value is a well-formed allocator.
Definition: Coroutines.cpp:590
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:205
static UndefValue * get(Type *T)
Static factory methods - Return an &#39;undef&#39; object of the specified type.
Definition: Constants.cpp:1446
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:525
EP_OptimizerLast – This extension point allows adding passes that run after everything else...
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:219
LLVMContext & Context
Definition: CoroInternal.h:56
EP_EarlyAsPossible - This extension point allows adding passes before any other transformations, allowing them to see the code as it is coming out of the frontend.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4355
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:289
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void dump() const
Definition: AsmWriter.cpp:4432
Function * getFunction() const
Returns the function that this call graph node represents.
Definition: CallGraph.h:187
EP_CGSCCOptimizerLate - This extension point allows adding CallGraphSCC passes at the end of the main...
StructType * FrameTy
Definition: CoroInternal.h:108
static void addCoroutineOpt0Passes(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:51
PassManagerBase - An abstract interface to allow code to add passes to a pass manager without having ...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:212
void initializeCoroutines(PassRegistry &)
Initialize all passes linked into the Coroutines library.
Definition: Coroutines.cpp:44
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2033
static CoroSaveInst * createCoroSave(CoroBeginInst *CoroBegin, CoroSuspendInst *SuspendInst)
Definition: Coroutines.cpp:236
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
Module.h This file contains the declarations for the Module class.
A constant pointer value that points to null.
Definition: Constants.h:538
Type * getReturnType() const
Definition: DerivedTypes.h:124
void checkWellFormed() const
Definition: Coroutines.cpp:626
#define LLVM_ATTRIBUTE_NORETURN
Definition: Compiler.h:234
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:653
static void addCoroutineEarlyPasses(const PassManagerBuilder &Builder, legacy::PassManagerBase &PM)
Definition: Coroutines.cpp:60
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
void LLVMAddCoroSplitPass(LLVMPassManagerRef PM)
See llvm::createCoroSplitPass function.
Definition: Coroutines.cpp:640
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:163
struct LLVMOpaquePassManager * LLVMPassManagerRef
Definition: Types.h:128
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:305
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
Definition: Coroutines.cpp:107
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:91
amdgpu Simplify well known AMD library false FunctionCallee Callee
iterator_range< user_iterator > users()
Definition: Value.h:419
static void clear(coro::Shape &Shape)
Definition: Coroutines.cpp:225
void initializeCoroCleanupPass(PassRegistry &)
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:73
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
Pass * createCoroSplitPass()
Split up coroutines into multiple functions driving their state machines.
Definition: CoroSplit.cpp:1601
const NodeList & List
Definition: RDFGraph.cpp:201
#define I(x, y, z)
Definition: MD5.cpp:58
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:519
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2237
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:166
static LLVM_ATTRIBUTE_NORETURN void fail(const Instruction *I, const char *Reason, Value *V)
Definition: Coroutines.cpp:540
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SmallVector< CoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:89
This represents the llvm.coro.frame instruction.
Definition: CoroInstr.h:277
LLVM Value Representation.
Definition: Value.h:73
CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on.
CallGraphNode * getOrInsertFunction(const Function *F)
Similar to operator[], but this will insert a new CallGraphNode for F if one does not already exist...
Definition: CallGraph.cpp:147
void addCoroutinePassesToExtensionPoints(PassManagerBuilder &Builder)
Add all coroutine passes to appropriate extension points.
Definition: Coroutines.cpp:80
bool declaresIntrinsics(Module &M, std::initializer_list< StringRef >)
Definition: Coroutines.cpp:153
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
inst_range instructions(Function *F)
Definition: InstIterator.h:133
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
Definition: PassRegistry.h:38
int lookupLLVMIntrinsicByName(ArrayRef< const char *> NameTable, StringRef Name)
Looks up Name in NameTable via binary search.
void LLVMAddCoroCleanupPass(LLVMPassManagerRef PM)
See llvm::createCoroCleanupPass function.
Definition: Coroutines.cpp:648
BasicBlock * AllocaSpillBlock
Definition: CoroInternal.h:110
void addExtension(ExtensionPointTy Ty, ExtensionFn Fn)
static void checkWFRetconPrototype(const AnyCoroIdRetconInst *I, Value *V)
Check that the given value is a well-formed prototype for the llvm.coro.id.retcon.
Definition: Coroutines.cpp:554
unsigned changeToUnreachable(Instruction *I, bool UseLLVMTrap, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:1898
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:173
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:376
void updateCallGraph(Function &Caller, ArrayRef< Function *> Funcs, CallGraph &CG, CallGraphSCC &SCC)
Definition: Coroutines.cpp:207
static void propagateCallAttrsFromCallee(CallInst *Call, Function *Callee)
Definition: Coroutines.cpp:488
void addCalledFunction(CallBase *Call, CallGraphNode *M)
Adds a function to the list of functions called by this one.
Definition: CallGraph.h:232