LLVM  13.0.0git
CoroEarly.cpp
Go to the documentation of this file.
1 //===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "CoroInternal.h"
11 #include "llvm/IR/IRBuilder.h"
12 #include "llvm/IR/InstIterator.h"
13 #include "llvm/IR/Module.h"
14 #include "llvm/Pass.h"
15 
16 using namespace llvm;
17 
18 #define DEBUG_TYPE "coro-early"
19 
20 namespace {
21 // Created on demand if the coro-early pass has work to do.
22 class Lowerer : public coro::LowererBase {
24  PointerType *const AnyResumeFnPtrTy;
25  Constant *NoopCoro = nullptr;
26 
27  void lowerResumeOrDestroy(CallBase &CB, CoroSubFnInst::ResumeKind);
28  void lowerCoroPromise(CoroPromiseInst *Intrin);
29  void lowerCoroDone(IntrinsicInst *II);
30  void lowerCoroNoop(IntrinsicInst *II);
31 
32 public:
33  Lowerer(Module &M)
34  : LowererBase(M), Builder(Context),
35  AnyResumeFnPtrTy(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
36  /*isVarArg=*/false)
37  ->getPointerTo()) {}
38  bool lowerEarlyIntrinsics(Function &F);
39 };
40 }
41 
42 // Replace a direct call to coro.resume or coro.destroy with an indirect call to
43 // an address returned by coro.subfn.addr intrinsic. This is done so that
44 // CGPassManager recognizes devirtualization when CoroElide pass replaces a call
45 // to coro.subfn.addr with an appropriate function address.
46 void Lowerer::lowerResumeOrDestroy(CallBase &CB,
48  Value *ResumeAddr = makeSubFnCall(CB.getArgOperand(0), Index, &CB);
49  CB.setCalledOperand(ResumeAddr);
51 }
52 
53 // Coroutine promise field is always at the fixed offset from the beginning of
54 // the coroutine frame. i8* coro.promise(i8*, i1 from) intrinsic adds an offset
55 // to a passed pointer to move from coroutine frame to coroutine promise and
56 // vice versa. Since we don't know exactly which coroutine frame it is, we build
57 // a coroutine frame mock up starting with two function pointers, followed by a
58 // properly aligned coroutine promise field.
59 // TODO: Handle the case when coroutine promise alloca has align override.
60 void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) {
61  Value *Operand = Intrin->getArgOperand(0);
62  Align Alignment = Intrin->getAlignment();
63  Type *Int8Ty = Builder.getInt8Ty();
64 
65  auto *SampleStruct =
66  StructType::get(Context, {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty});
67  const DataLayout &DL = TheModule.getDataLayout();
68  int64_t Offset = alignTo(
69  DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignment);
70  if (Intrin->isFromPromise())
71  Offset = -Offset;
72 
73  Builder.SetInsertPoint(Intrin);
74  Value *Replacement =
75  Builder.CreateConstInBoundsGEP1_32(Int8Ty, Operand, Offset);
76 
77  Intrin->replaceAllUsesWith(Replacement);
78  Intrin->eraseFromParent();
79 }
80 
81 // When a coroutine reaches final suspend point, it zeros out ResumeFnAddr in
82 // the coroutine frame (it is UB to resume from a final suspend point).
83 // The llvm.coro.done intrinsic is used to check whether a coroutine is
84 // suspended at the final suspend point or not.
85 void Lowerer::lowerCoroDone(IntrinsicInst *II) {
86  Value *Operand = II->getArgOperand(0);
87 
88  // ResumeFnAddr is the first pointer sized element of the coroutine frame.
89  static_assert(coro::Shape::SwitchFieldIndex::Resume == 0,
90  "resume function not at offset zero");
91  auto *FrameTy = Int8Ptr;
92  PointerType *FramePtrTy = FrameTy->getPointerTo();
93 
94  Builder.SetInsertPoint(II);
95  auto *BCI = Builder.CreateBitCast(Operand, FramePtrTy);
96  auto *Load = Builder.CreateLoad(FrameTy, BCI);
97  auto *Cond = Builder.CreateICmpEQ(Load, NullPtr);
98 
100  II->eraseFromParent();
101 }
102 
103 void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
104  if (!NoopCoro) {
105  LLVMContext &C = Builder.getContext();
106  Module &M = *II->getModule();
107 
108  // Create a noop.frame struct type.
109  StructType *FrameTy = StructType::create(C, "NoopCoro.Frame");
110  auto *FramePtrTy = FrameTy->getPointerTo();
111  auto *FnTy = FunctionType::get(Type::getVoidTy(C), FramePtrTy,
112  /*isVarArg=*/false);
113  auto *FnPtrTy = FnTy->getPointerTo();
114  FrameTy->setBody({FnPtrTy, FnPtrTy});
115 
116  // Create a Noop function that does nothing.
117  Function *NoopFn =
118  Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
119  "NoopCoro.ResumeDestroy", &M);
121  auto *Entry = BasicBlock::Create(C, "entry", NoopFn);
122  ReturnInst::Create(C, Entry);
123 
124  // Create a constant struct for the frame.
125  Constant* Values[] = {NoopFn, NoopFn};
126  Constant* NoopCoroConst = ConstantStruct::get(FrameTy, Values);
127  NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true,
128  GlobalVariable::PrivateLinkage, NoopCoroConst,
129  "NoopCoro.Frame.Const");
130  }
131 
132  Builder.SetInsertPoint(II);
133  auto *NoopCoroVoidPtr = Builder.CreateBitCast(NoopCoro, Int8Ptr);
134  II->replaceAllUsesWith(NoopCoroVoidPtr);
135  II->eraseFromParent();
136 }
137 
138 // Prior to CoroSplit, calls to coro.begin needs to be marked as NoDuplicate,
139 // as CoroSplit assumes there is exactly one coro.begin. After CoroSplit,
140 // NoDuplicate attribute will be removed from coro.begin otherwise, it will
141 // interfere with inlining.
142 static void setCannotDuplicate(CoroIdInst *CoroId) {
143  for (User *U : CoroId->users())
144  if (auto *CB = dyn_cast<CoroBeginInst>(U))
145  CB->setCannotDuplicate();
146 }
147 
148 bool Lowerer::lowerEarlyIntrinsics(Function &F) {
149  bool Changed = false;
150  CoroIdInst *CoroId = nullptr;
152  for (auto IB = inst_begin(F), IE = inst_end(F); IB != IE;) {
153  Instruction &I = *IB++;
154  if (auto *CB = dyn_cast<CallBase>(&I)) {
155  switch (CB->getIntrinsicID()) {
156  default:
157  continue;
158  case Intrinsic::coro_free:
159  CoroFrees.push_back(cast<CoroFreeInst>(&I));
160  break;
161  case Intrinsic::coro_suspend:
162  // Make sure that final suspend point is not duplicated as CoroSplit
163  // pass expects that there is at most one final suspend point.
164  if (cast<CoroSuspendInst>(&I)->isFinal())
165  CB->setCannotDuplicate();
166  break;
167  case Intrinsic::coro_end_async:
168  case Intrinsic::coro_end:
169  // Make sure that fallthrough coro.end is not duplicated as CoroSplit
170  // pass expects that there is at most one fallthrough coro.end.
171  if (cast<AnyCoroEndInst>(&I)->isFallthrough())
172  CB->setCannotDuplicate();
173  break;
174  case Intrinsic::coro_noop:
175  lowerCoroNoop(cast<IntrinsicInst>(&I));
176  break;
177  case Intrinsic::coro_id:
178  // Mark a function that comes out of the frontend that has a coro.id
179  // with a coroutine attribute.
180  if (auto *CII = cast<CoroIdInst>(&I)) {
181  if (CII->getInfo().isPreSplit()) {
183  setCannotDuplicate(CII);
184  CII->setCoroutineSelf();
185  CoroId = cast<CoroIdInst>(&I);
186  }
187  }
188  break;
189  case Intrinsic::coro_id_retcon:
190  case Intrinsic::coro_id_retcon_once:
191  case Intrinsic::coro_id_async:
193  break;
194  case Intrinsic::coro_resume:
195  lowerResumeOrDestroy(*CB, CoroSubFnInst::ResumeIndex);
196  break;
197  case Intrinsic::coro_destroy:
198  lowerResumeOrDestroy(*CB, CoroSubFnInst::DestroyIndex);
199  break;
200  case Intrinsic::coro_promise:
201  lowerCoroPromise(cast<CoroPromiseInst>(&I));
202  break;
203  case Intrinsic::coro_done:
204  lowerCoroDone(cast<IntrinsicInst>(&I));
205  break;
206  }
207  Changed = true;
208  }
209  }
210  // Make sure that all CoroFree reference the coro.id intrinsic.
211  // Token type is not exposed through coroutine C/C++ builtins to plain C, so
212  // we allow specifying none and fixing it up here.
213  if (CoroId)
214  for (CoroFreeInst *CF : CoroFrees)
215  CF->setArgOperand(0, CoroId);
216  return Changed;
217 }
218 
219 static bool declaresCoroEarlyIntrinsics(const Module &M) {
221  M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once",
222  "llvm.coro.id.async", "llvm.coro.destroy", "llvm.coro.done",
223  "llvm.coro.end", "llvm.coro.end.async", "llvm.coro.noop",
224  "llvm.coro.free", "llvm.coro.promise", "llvm.coro.resume",
225  "llvm.coro.suspend"});
226 }
227 
229  Module &M = *F.getParent();
230  if (!declaresCoroEarlyIntrinsics(M) || !Lowerer(M).lowerEarlyIntrinsics(F))
231  return PreservedAnalyses::all();
232 
234  PA.preserveSet<CFGAnalyses>();
235  return PA;
236 }
237 
238 namespace {
239 
240 struct CoroEarlyLegacy : public FunctionPass {
241  static char ID; // Pass identification, replacement for typeid.
242  CoroEarlyLegacy() : FunctionPass(ID) {
244  }
245 
246  std::unique_ptr<Lowerer> L;
247 
248  // This pass has work to do only if we find intrinsics we are going to lower
249  // in the module.
250  bool doInitialization(Module &M) override {
252  L = std::make_unique<Lowerer>(M);
253  return false;
254  }
255 
256  bool runOnFunction(Function &F) override {
257  if (!L)
258  return false;
259 
260  return L->lowerEarlyIntrinsics(F);
261  }
262 
263  void getAnalysisUsage(AnalysisUsage &AU) const override {
264  AU.setPreservesCFG();
265  }
266  StringRef getPassName() const override {
267  return "Lower early coroutine intrinsics";
268  }
269 };
270 }
271 
272 char CoroEarlyLegacy::ID = 0;
273 INITIALIZE_PASS(CoroEarlyLegacy, "coro-early",
274  "Lower early coroutine intrinsics", false, false)
275 
276 Pass *llvm::createCoroEarlyLegacyPass() { return new CoroEarlyLegacy(); }
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:148
llvm::CoroSubFnInst::ResumeIndex
@ ResumeIndex
Definition: CoroInstr.h:41
PREPARED_FOR_SPLIT
#define PREPARED_FOR_SPLIT
Definition: CoroInternal.h:41
llvm
Definition: AllocatorList.h:23
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:366
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:112
llvm::CoroSubFnInst::DestroyIndex
@ DestroyIndex
Definition: CoroInstr.h:42
InstIterator.h
llvm::Function
Definition: Function.h:61
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1325
Pass.h
llvm::StructType::setBody
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
Definition: Type.cpp:405
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1167
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:40
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:321
llvm::CoroSubFnInst::ResumeKind
ResumeKind
Definition: CoroInstr.h:39
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
Module.h
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:161
llvm::CoroEarlyPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: CoroEarly.cpp:228
llvm::CoroFreeInst
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:404
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
setCannotDuplicate
static void setCannotDuplicate(CoroIdInst *CoroId)
Definition: CoroEarly.cpp:142
llvm::StructType::create
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Definition: Type.cpp:473
llvm::CallingConv::Fast
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
llvm::initializeCoroEarlyLegacyPass
void initializeCoroEarlyLegacyPass(PassRegistry &)
F
#define F(x, y, z)
Definition: MD5.cpp:56
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::User
Definition: User.h:44
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::CoroPromiseInst::getAlignment
Align getAlignment() const
The required alignment of the promise.
Definition: CoroInstr.h:464
llvm::Instruction
Definition: Instruction.h:45
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::ARM_PROC::IE
@ IE
Definition: ARMBaseInfo.h:27
CoroInternal.h
llvm::CoroPromiseInst
This represents the llvm.coro.promise instruction.
Definition: CoroInstr.h:452
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:78
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:634
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
IRBuilder.h
UNPREPARED_FOR_SPLIT
#define UNPREPARED_FOR_SPLIT
Definition: CoroInternal.h:40
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::createCoroEarlyLegacyPass
Pass * createCoroEarlyLegacyPass()
Lower coroutine intrinsics that are not needed by later passes.
llvm::CallBase::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition: Instructions.cpp:307
llvm::CoroPromiseInst::isFromPromise
bool isFromPromise() const
Are we translating from the frame to the promise (false) or from the promise to the frame (true)?
Definition: CoroInstr.h:458
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:212
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:167
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:253
llvm::Function::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: Function.h:242
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:116
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:517
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:100
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
CORO_PRESPLIT_ATTR
#define CORO_PRESPLIT_ATTR
Definition: CoroInternal.h:39
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
llvm::inst_end
inst_iterator inst_end(Function *F)
Definition: InstIterator.h:132
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:728
llvm::ReturnInst::Create
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:2980
llvm::IntrinsicInst
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:45
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition: InstrTypes.h:1429
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:180
llvm::Pass
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:91
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
llvm::PreservedAnalyses::preserveSet
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:191
declaresCoroEarlyIntrinsics
static bool declaresCoroEarlyIntrinsics(const Module &M)
Definition: CoroEarly.cpp:219
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1341
llvm::coro::LowererBase
Definition: CoroInternal.h:60
CoroEarly.h
llvm::coro::Shape::SwitchFieldIndex::Resume
@ Resume
Definition: CoroInternal.h:108
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1164
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
llvm::inst_begin
inst_iterator inst_begin(Function *F)
Definition: InstIterator.h:131
INITIALIZE_PASS
INITIALIZE_PASS(CoroEarlyLegacy, "coro-early", "Lower early coroutine intrinsics", false, false) Pass *llvm
Definition: CoroEarly.cpp:273
llvm::CallBase::setCannotDuplicate
void setCannotDuplicate()
Definition: InstrTypes.h:1849
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:422
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1456
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38