LLVM  12.0.0git
CoroEarly.cpp
Go to the documentation of this file.
1 //===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "CoroInternal.h"
11 #include "llvm/IR/IRBuilder.h"
12 #include "llvm/IR/InstIterator.h"
13 #include "llvm/IR/Module.h"
14 #include "llvm/Pass.h"
15 
16 using namespace llvm;
17 
18 #define DEBUG_TYPE "coro-early"
19 
20 namespace {
21 // Created on demand if the coro-early pass has work to do.
22 class Lowerer : public coro::LowererBase {
24  PointerType *const AnyResumeFnPtrTy;
25  Constant *NoopCoro = nullptr;
26 
27  void lowerResumeOrDestroy(CallBase &CB, CoroSubFnInst::ResumeKind);
28  void lowerCoroPromise(CoroPromiseInst *Intrin);
29  void lowerCoroDone(IntrinsicInst *II);
30  void lowerCoroNoop(IntrinsicInst *II);
31 
32 public:
33  Lowerer(Module &M)
34  : LowererBase(M), Builder(Context),
35  AnyResumeFnPtrTy(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
36  /*isVarArg=*/false)
37  ->getPointerTo()) {}
38  bool lowerEarlyIntrinsics(Function &F);
39 };
40 }
41 
42 // Replace a direct call to coro.resume or coro.destroy with an indirect call to
43 // an address returned by coro.subfn.addr intrinsic. This is done so that
44 // CGPassManager recognizes devirtualization when CoroElide pass replaces a call
45 // to coro.subfn.addr with an appropriate function address.
46 void Lowerer::lowerResumeOrDestroy(CallBase &CB,
48  Value *ResumeAddr = makeSubFnCall(CB.getArgOperand(0), Index, &CB);
49  CB.setCalledOperand(ResumeAddr);
51 }
52 
53 // Coroutine promise field is always at the fixed offset from the beginning of
54 // the coroutine frame. i8* coro.promise(i8*, i1 from) intrinsic adds an offset
55 // to a passed pointer to move from coroutine frame to coroutine promise and
56 // vice versa. Since we don't know exactly which coroutine frame it is, we build
57 // a coroutine frame mock up starting with two function pointers, followed by a
58 // properly aligned coroutine promise field.
59 // TODO: Handle the case when coroutine promise alloca has align override.
60 void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) {
61  Value *Operand = Intrin->getArgOperand(0);
62  Align Alignment = Intrin->getAlignment();
63  Type *Int8Ty = Builder.getInt8Ty();
64 
65  auto *SampleStruct =
66  StructType::get(Context, {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty});
67  const DataLayout &DL = TheModule.getDataLayout();
68  int64_t Offset = alignTo(
69  DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignment);
70  if (Intrin->isFromPromise())
71  Offset = -Offset;
72 
73  Builder.SetInsertPoint(Intrin);
74  Value *Replacement =
75  Builder.CreateConstInBoundsGEP1_32(Int8Ty, Operand, Offset);
76 
77  Intrin->replaceAllUsesWith(Replacement);
78  Intrin->eraseFromParent();
79 }
80 
81 // When a coroutine reaches final suspend point, it zeros out ResumeFnAddr in
82 // the coroutine frame (it is UB to resume from a final suspend point).
83 // The llvm.coro.done intrinsic is used to check whether a coroutine is
84 // suspended at the final suspend point or not.
85 void Lowerer::lowerCoroDone(IntrinsicInst *II) {
86  Value *Operand = II->getArgOperand(0);
87 
88  // ResumeFnAddr is the first pointer sized element of the coroutine frame.
89  static_assert(coro::Shape::SwitchFieldIndex::Resume == 0,
90  "resume function not at offset zero");
91  auto *FrameTy = Int8Ptr;
92  PointerType *FramePtrTy = FrameTy->getPointerTo();
93 
94  Builder.SetInsertPoint(II);
95  auto *BCI = Builder.CreateBitCast(Operand, FramePtrTy);
96  auto *Load = Builder.CreateLoad(FrameTy, BCI);
97  auto *Cond = Builder.CreateICmpEQ(Load, NullPtr);
98 
100  II->eraseFromParent();
101 }
102 
103 void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
104  if (!NoopCoro) {
105  LLVMContext &C = Builder.getContext();
106  Module &M = *II->getModule();
107 
108  // Create a noop.frame struct type.
109  StructType *FrameTy = StructType::create(C, "NoopCoro.Frame");
110  auto *FramePtrTy = FrameTy->getPointerTo();
111  auto *FnTy = FunctionType::get(Type::getVoidTy(C), FramePtrTy,
112  /*isVarArg=*/false);
113  auto *FnPtrTy = FnTy->getPointerTo();
114  FrameTy->setBody({FnPtrTy, FnPtrTy});
115 
116  // Create a Noop function that does nothing.
117  Function *NoopFn =
118  Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
119  "NoopCoro.ResumeDestroy", &M);
121  auto *Entry = BasicBlock::Create(C, "entry", NoopFn);
122  ReturnInst::Create(C, Entry);
123 
124  // Create a constant struct for the frame.
125  Constant* Values[] = {NoopFn, NoopFn};
126  Constant* NoopCoroConst = ConstantStruct::get(FrameTy, Values);
127  NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true,
128  GlobalVariable::PrivateLinkage, NoopCoroConst,
129  "NoopCoro.Frame.Const");
130  }
131 
132  Builder.SetInsertPoint(II);
133  auto *NoopCoroVoidPtr = Builder.CreateBitCast(NoopCoro, Int8Ptr);
134  II->replaceAllUsesWith(NoopCoroVoidPtr);
135  II->eraseFromParent();
136 }
137 
138 // Prior to CoroSplit, calls to coro.begin needs to be marked as NoDuplicate,
139 // as CoroSplit assumes there is exactly one coro.begin. After CoroSplit,
140 // NoDuplicate attribute will be removed from coro.begin otherwise, it will
141 // interfere with inlining.
142 static void setCannotDuplicate(CoroIdInst *CoroId) {
143  for (User *U : CoroId->users())
144  if (auto *CB = dyn_cast<CoroBeginInst>(U))
145  CB->setCannotDuplicate();
146 }
147 
148 bool Lowerer::lowerEarlyIntrinsics(Function &F) {
149  bool Changed = false;
150  CoroIdInst *CoroId = nullptr;
152  for (auto IB = inst_begin(F), IE = inst_end(F); IB != IE;) {
153  Instruction &I = *IB++;
154  if (auto *CB = dyn_cast<CallBase>(&I)) {
155  switch (CB->getIntrinsicID()) {
156  default:
157  continue;
158  case Intrinsic::coro_free:
159  CoroFrees.push_back(cast<CoroFreeInst>(&I));
160  break;
161  case Intrinsic::coro_suspend:
162  // Make sure that final suspend point is not duplicated as CoroSplit
163  // pass expects that there is at most one final suspend point.
164  if (cast<CoroSuspendInst>(&I)->isFinal())
165  CB->setCannotDuplicate();
166  break;
167  case Intrinsic::coro_end_async:
168  case Intrinsic::coro_end:
169  // Make sure that fallthrough coro.end is not duplicated as CoroSplit
170  // pass expects that there is at most one fallthrough coro.end.
171  if (cast<AnyCoroEndInst>(&I)->isFallthrough())
172  CB->setCannotDuplicate();
173  break;
174  case Intrinsic::coro_noop:
175  lowerCoroNoop(cast<IntrinsicInst>(&I));
176  break;
177  case Intrinsic::coro_id:
178  // Mark a function that comes out of the frontend that has a coro.id
179  // with a coroutine attribute.
180  if (auto *CII = cast<CoroIdInst>(&I)) {
181  if (CII->getInfo().isPreSplit()) {
183  setCannotDuplicate(CII);
184  CII->setCoroutineSelf();
185  CoroId = cast<CoroIdInst>(&I);
186  }
187  }
188  break;
189  case Intrinsic::coro_id_retcon:
190  case Intrinsic::coro_id_retcon_once:
191  case Intrinsic::coro_id_async:
193  break;
194  case Intrinsic::coro_resume:
195  lowerResumeOrDestroy(*CB, CoroSubFnInst::ResumeIndex);
196  break;
197  case Intrinsic::coro_destroy:
198  lowerResumeOrDestroy(*CB, CoroSubFnInst::DestroyIndex);
199  break;
200  case Intrinsic::coro_promise:
201  lowerCoroPromise(cast<CoroPromiseInst>(&I));
202  break;
203  case Intrinsic::coro_done:
204  lowerCoroDone(cast<IntrinsicInst>(&I));
205  break;
206  }
207  Changed = true;
208  }
209  }
210  // Make sure that all CoroFree reference the coro.id intrinsic.
211  // Token type is not exposed through coroutine C/C++ builtins to plain C, so
212  // we allow specifying none and fixing it up here.
213  if (CoroId)
214  for (CoroFreeInst *CF : CoroFrees)
215  CF->setArgOperand(0, CoroId);
216  return Changed;
217 }
218 
219 static bool declaresCoroEarlyIntrinsics(const Module &M) {
221  M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once",
222  "llvm.coro.destroy", "llvm.coro.done", "llvm.coro.end",
223  "llvm.coro.end.async", "llvm.coro.noop", "llvm.coro.free",
224  "llvm.coro.promise", "llvm.coro.resume", "llvm.coro.suspend"});
225 }
226 
228  Module &M = *F.getParent();
229  if (!declaresCoroEarlyIntrinsics(M) || !Lowerer(M).lowerEarlyIntrinsics(F))
230  return PreservedAnalyses::all();
231 
233  PA.preserveSet<CFGAnalyses>();
234  return PA;
235 }
236 
237 namespace {
238 
239 struct CoroEarlyLegacy : public FunctionPass {
240  static char ID; // Pass identification, replacement for typeid.
241  CoroEarlyLegacy() : FunctionPass(ID) {
243  }
244 
245  std::unique_ptr<Lowerer> L;
246 
247  // This pass has work to do only if we find intrinsics we are going to lower
248  // in the module.
249  bool doInitialization(Module &M) override {
251  L = std::make_unique<Lowerer>(M);
252  return false;
253  }
254 
255  bool runOnFunction(Function &F) override {
256  if (!L)
257  return false;
258 
259  return L->lowerEarlyIntrinsics(F);
260  }
261 
262  void getAnalysisUsage(AnalysisUsage &AU) const override {
263  AU.setPreservesCFG();
264  }
265  StringRef getPassName() const override {
266  return "Lower early coroutine intrinsics";
267  }
268 };
269 }
270 
271 char CoroEarlyLegacy::ID = 0;
272 INITIALIZE_PASS(CoroEarlyLegacy, "coro-early",
273  "Lower early coroutine intrinsics", false, false)
274 
275 Pass *llvm::createCoroEarlyLegacyPass() { return new CoroEarlyLegacy(); }
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:91
uint64_t CallInst * C
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:77
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:111
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & Context
This represents the llvm.coro.promise instruction.
Definition: CoroInstr.h:440
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
This class represents lattice values for constants.
Definition: AllocatorList.h:23
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1164
F(f)
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: CoroEarly.cpp:227
Align getAlignment() const
The required alignment of the promise.
Definition: CoroInstr.h:452
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1323
inst_iterator inst_begin(Function *F)
Definition: InstIterator.h:131
Class to represent struct types.
Definition: DerivedTypes.h:212
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:691
INITIALIZE_PASS(CoroEarlyLegacy, "coro-early", "Lower early coroutine intrinsics", false, false) Pass *llvm
Definition: CoroEarly.cpp:272
#define PREPARED_FOR_SPLIT
Definition: CoroInternal.h:41
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:246
void setCalledOperand(Value *V)
Definition: InstrTypes.h:1411
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:321
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:523
Pass * createCoroEarlyLegacyPass()
Lower coroutine intrinsics that are not needed by later passes.
Class to represent pointers.
Definition: DerivedTypes.h:655
static bool runOnFunction(Function &F, bool PostInlining)
void setCallingConv(CallingConv::ID CC)
Definition: Function.h:232
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
#define CORO_PRESPLIT_ATTR
Definition: CoroInternal.h:39
void setCannotDuplicate()
Definition: InstrTypes.h:1815
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
This is an important base class in LLVM.
Definition: Constant.h:41
void initializeCoroEarlyLegacyPass(PassRegistry &)
Represent the analysis usage information of a pass.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:180
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1439
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
bool isFromPromise() const
Are we translating from the frame to the promise (false) or from the promise to the frame (true)?
Definition: CoroInstr.h:446
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:100
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1291
assume Assume Builder
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
Definition: Type.cpp:393
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:392
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t Offset
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:160
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:366
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1116
Module.h This file contains the declarations for the Module class.
uint32_t Index
#define UNPREPARED_FOR_SPLIT
Definition: CoroInternal.h:40
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:253
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:65
iterator_range< user_iterator > users()
Definition: Value.h:424
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:116
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:158
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:191
#define I(x, y, z)
Definition: MD5.cpp:59
static void setCannotDuplicate(CoroIdInst *CoroId)
Definition: CoroEarly.cpp:142
SmallVector< MachineOperand, 4 > Cond
LLVM Value Representation.
Definition: Value.h:75
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Definition: Type.cpp:461
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
inst_iterator inst_end(Function *F)
Definition: InstIterator.h:132
A container for analyses that lazily runs them and caches their results.
static bool declaresCoroEarlyIntrinsics(const Module &M)
Definition: CoroEarly.cpp:219
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...