LLVM  14.0.0git
CoroEarly.cpp
Go to the documentation of this file.
1 //===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "CoroInternal.h"
11 #include "llvm/IR/IRBuilder.h"
12 #include "llvm/IR/InstIterator.h"
13 #include "llvm/IR/Module.h"
14 #include "llvm/Pass.h"
15 
16 using namespace llvm;
17 
18 #define DEBUG_TYPE "coro-early"
19 
20 namespace {
21 // Created on demand if the coro-early pass has work to do.
22 class Lowerer : public coro::LowererBase {
24  PointerType *const AnyResumeFnPtrTy;
25  Constant *NoopCoro = nullptr;
26 
27  void lowerResumeOrDestroy(CallBase &CB, CoroSubFnInst::ResumeKind);
28  void lowerCoroPromise(CoroPromiseInst *Intrin);
29  void lowerCoroDone(IntrinsicInst *II);
30  void lowerCoroNoop(IntrinsicInst *II);
31 
32 public:
33  Lowerer(Module &M)
34  : LowererBase(M), Builder(Context),
35  AnyResumeFnPtrTy(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
36  /*isVarArg=*/false)
37  ->getPointerTo()) {}
38  bool lowerEarlyIntrinsics(Function &F);
39 };
40 }
41 
42 // Replace a direct call to coro.resume or coro.destroy with an indirect call to
43 // an address returned by coro.subfn.addr intrinsic. This is done so that
44 // CGPassManager recognizes devirtualization when CoroElide pass replaces a call
45 // to coro.subfn.addr with an appropriate function address.
46 void Lowerer::lowerResumeOrDestroy(CallBase &CB,
48  Value *ResumeAddr = makeSubFnCall(CB.getArgOperand(0), Index, &CB);
49  CB.setCalledOperand(ResumeAddr);
51 }
52 
53 // Coroutine promise field is always at the fixed offset from the beginning of
54 // the coroutine frame. i8* coro.promise(i8*, i1 from) intrinsic adds an offset
55 // to a passed pointer to move from coroutine frame to coroutine promise and
56 // vice versa. Since we don't know exactly which coroutine frame it is, we build
57 // a coroutine frame mock up starting with two function pointers, followed by a
58 // properly aligned coroutine promise field.
59 // TODO: Handle the case when coroutine promise alloca has align override.
60 void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) {
61  Value *Operand = Intrin->getArgOperand(0);
62  Align Alignment = Intrin->getAlignment();
63  Type *Int8Ty = Builder.getInt8Ty();
64 
65  auto *SampleStruct =
66  StructType::get(Context, {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty});
67  const DataLayout &DL = TheModule.getDataLayout();
68  int64_t Offset = alignTo(
69  DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignment);
70  if (Intrin->isFromPromise())
71  Offset = -Offset;
72 
73  Builder.SetInsertPoint(Intrin);
74  Value *Replacement =
75  Builder.CreateConstInBoundsGEP1_32(Int8Ty, Operand, Offset);
76 
77  Intrin->replaceAllUsesWith(Replacement);
78  Intrin->eraseFromParent();
79 }
80 
81 // When a coroutine reaches final suspend point, it zeros out ResumeFnAddr in
82 // the coroutine frame (it is UB to resume from a final suspend point).
83 // The llvm.coro.done intrinsic is used to check whether a coroutine is
84 // suspended at the final suspend point or not.
85 void Lowerer::lowerCoroDone(IntrinsicInst *II) {
86  Value *Operand = II->getArgOperand(0);
87 
88  // ResumeFnAddr is the first pointer sized element of the coroutine frame.
89  static_assert(coro::Shape::SwitchFieldIndex::Resume == 0,
90  "resume function not at offset zero");
91  auto *FrameTy = Int8Ptr;
92  PointerType *FramePtrTy = FrameTy->getPointerTo();
93 
94  Builder.SetInsertPoint(II);
95  auto *BCI = Builder.CreateBitCast(Operand, FramePtrTy);
96  auto *Load = Builder.CreateLoad(FrameTy, BCI);
97  auto *Cond = Builder.CreateICmpEQ(Load, NullPtr);
98 
100  II->eraseFromParent();
101 }
102 
103 void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
104  if (!NoopCoro) {
105  LLVMContext &C = Builder.getContext();
106  Module &M = *II->getModule();
107 
108  // Create a noop.frame struct type.
109  StructType *FrameTy = StructType::create(C, "NoopCoro.Frame");
110  auto *FramePtrTy = FrameTy->getPointerTo();
111  auto *FnTy = FunctionType::get(Type::getVoidTy(C), FramePtrTy,
112  /*isVarArg=*/false);
113  auto *FnPtrTy = FnTy->getPointerTo();
114  FrameTy->setBody({FnPtrTy, FnPtrTy});
115 
116  // Create a Noop function that does nothing.
117  Function *NoopFn =
118  Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
119  "NoopCoro.ResumeDestroy", &M);
121  auto *Entry = BasicBlock::Create(C, "entry", NoopFn);
122  ReturnInst::Create(C, Entry);
123 
124  // Create a constant struct for the frame.
125  Constant* Values[] = {NoopFn, NoopFn};
126  Constant* NoopCoroConst = ConstantStruct::get(FrameTy, Values);
127  NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true,
128  GlobalVariable::PrivateLinkage, NoopCoroConst,
129  "NoopCoro.Frame.Const");
130  }
131 
132  Builder.SetInsertPoint(II);
133  auto *NoopCoroVoidPtr = Builder.CreateBitCast(NoopCoro, Int8Ptr);
134  II->replaceAllUsesWith(NoopCoroVoidPtr);
135  II->eraseFromParent();
136 }
137 
138 // Prior to CoroSplit, calls to coro.begin needs to be marked as NoDuplicate,
139 // as CoroSplit assumes there is exactly one coro.begin. After CoroSplit,
140 // NoDuplicate attribute will be removed from coro.begin otherwise, it will
141 // interfere with inlining.
142 static void setCannotDuplicate(CoroIdInst *CoroId) {
143  for (User *U : CoroId->users())
144  if (auto *CB = dyn_cast<CoroBeginInst>(U))
145  CB->setCannotDuplicate();
146 }
147 
148 bool Lowerer::lowerEarlyIntrinsics(Function &F) {
149  bool Changed = false;
150  CoroIdInst *CoroId = nullptr;
152  bool HasCoroSuspend = false;
154  if (auto *CB = dyn_cast<CallBase>(&I)) {
155  switch (CB->getIntrinsicID()) {
156  default:
157  continue;
158  case Intrinsic::coro_free:
159  CoroFrees.push_back(cast<CoroFreeInst>(&I));
160  break;
161  case Intrinsic::coro_suspend:
162  // Make sure that final suspend point is not duplicated as CoroSplit
163  // pass expects that there is at most one final suspend point.
164  if (cast<CoroSuspendInst>(&I)->isFinal())
165  CB->setCannotDuplicate();
166  HasCoroSuspend = true;
167  break;
168  case Intrinsic::coro_end_async:
169  case Intrinsic::coro_end:
170  // Make sure that fallthrough coro.end is not duplicated as CoroSplit
171  // pass expects that there is at most one fallthrough coro.end.
172  if (cast<AnyCoroEndInst>(&I)->isFallthrough())
173  CB->setCannotDuplicate();
174  break;
175  case Intrinsic::coro_noop:
176  lowerCoroNoop(cast<IntrinsicInst>(&I));
177  break;
178  case Intrinsic::coro_id:
179  // Mark a function that comes out of the frontend that has a coro.id
180  // with a coroutine attribute.
181  if (auto *CII = cast<CoroIdInst>(&I)) {
182  if (CII->getInfo().isPreSplit()) {
184  setCannotDuplicate(CII);
185  CII->setCoroutineSelf();
186  CoroId = cast<CoroIdInst>(&I);
187  }
188  }
189  break;
190  case Intrinsic::coro_id_retcon:
191  case Intrinsic::coro_id_retcon_once:
192  case Intrinsic::coro_id_async:
194  break;
195  case Intrinsic::coro_resume:
196  lowerResumeOrDestroy(*CB, CoroSubFnInst::ResumeIndex);
197  break;
198  case Intrinsic::coro_destroy:
199  lowerResumeOrDestroy(*CB, CoroSubFnInst::DestroyIndex);
200  break;
201  case Intrinsic::coro_promise:
202  lowerCoroPromise(cast<CoroPromiseInst>(&I));
203  break;
204  case Intrinsic::coro_done:
205  lowerCoroDone(cast<IntrinsicInst>(&I));
206  break;
207  }
208  Changed = true;
209  }
210  }
211  // Make sure that all CoroFree reference the coro.id intrinsic.
212  // Token type is not exposed through coroutine C/C++ builtins to plain C, so
213  // we allow specifying none and fixing it up here.
214  if (CoroId)
215  for (CoroFreeInst *CF : CoroFrees)
216  CF->setArgOperand(0, CoroId);
217  // Coroutine suspention could potentially lead to any argument modified
218  // outside of the function, hence arguments should not have noalias
219  // attributes.
220  if (HasCoroSuspend)
221  for (Argument &A : F.args())
222  if (A.hasNoAliasAttr())
223  A.removeAttr(Attribute::NoAlias);
224  return Changed;
225 }
226 
227 static bool declaresCoroEarlyIntrinsics(const Module &M) {
229  M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once",
230  "llvm.coro.id.async", "llvm.coro.destroy", "llvm.coro.done",
231  "llvm.coro.end", "llvm.coro.end.async", "llvm.coro.noop",
232  "llvm.coro.free", "llvm.coro.promise", "llvm.coro.resume",
233  "llvm.coro.suspend"});
234 }
235 
237  Module &M = *F.getParent();
238  if (!declaresCoroEarlyIntrinsics(M) || !Lowerer(M).lowerEarlyIntrinsics(F))
239  return PreservedAnalyses::all();
240 
242  PA.preserveSet<CFGAnalyses>();
243  return PA;
244 }
245 
246 namespace {
247 
248 struct CoroEarlyLegacy : public FunctionPass {
249  static char ID; // Pass identification, replacement for typeid.
250  CoroEarlyLegacy() : FunctionPass(ID) {
252  }
253 
254  std::unique_ptr<Lowerer> L;
255 
256  // This pass has work to do only if we find intrinsics we are going to lower
257  // in the module.
258  bool doInitialization(Module &M) override {
260  L = std::make_unique<Lowerer>(M);
261  return false;
262  }
263 
264  bool runOnFunction(Function &F) override {
265  if (!L)
266  return false;
267 
268  return L->lowerEarlyIntrinsics(F);
269  }
270 
271  void getAnalysisUsage(AnalysisUsage &AU) const override {
272  AU.setPreservesCFG();
273  }
274  StringRef getPassName() const override {
275  return "Lower early coroutine intrinsics";
276  }
277 };
278 }
279 
280 char CoroEarlyLegacy::ID = 0;
281 INITIALIZE_PASS(CoroEarlyLegacy, "coro-early",
282  "Lower early coroutine intrinsics", false, false)
283 
284 Pass *llvm::createCoroEarlyLegacyPass() { return new CoroEarlyLegacy(); }
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:148
llvm::CoroSubFnInst::ResumeIndex
@ ResumeIndex
Definition: CoroInstr.h:41
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
PREPARED_FOR_SPLIT
#define PREPARED_FOR_SPLIT
Definition: CoroInternal.h:41
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:370
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:112
llvm::CoroSubFnInst::DestroyIndex
@ DestroyIndex
Definition: CoroInstr.h:42
InstIterator.h
llvm::Function
Definition: Function.h:61
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1327
Pass.h
llvm::StructType::setBody
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
Definition: Type.cpp:409
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:40
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:325
llvm::CoroSubFnInst::ResumeKind
ResumeKind
Definition: CoroInstr.h:39
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:161
llvm::CoroEarlyPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: CoroEarly.cpp:236
llvm::CoroFreeInst
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:404
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
setCannotDuplicate
static void setCannotDuplicate(CoroIdInst *CoroId)
Definition: CoroEarly.cpp:142
llvm::StructType::create
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Definition: Type.cpp:477
llvm::initializeCoroEarlyLegacyPass
void initializeCoroEarlyLegacyPass(PassRegistry &)
F
#define F(x, y, z)
Definition: MD5.cpp:56
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::User
Definition: User.h:44
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::ARM_PROC::A
@ A
Definition: ARMBaseInfo.h:34
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::CoroPromiseInst::getAlignment
Align getAlignment() const
The required alignment of the promise.
Definition: CoroInstr.h:464
llvm::Instruction
Definition: Instruction.h:45
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
CoroInternal.h
llvm::CoroPromiseInst
This represents the llvm.coro.promise instruction.
Definition: CoroInstr.h:452
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:78
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:572
llvm::coro::Shape::SwitchFieldIndex::Resume
@ Resume
Definition: CoroInternal.h:108
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:138
IRBuilder.h
UNPREPARED_FOR_SPLIT
#define UNPREPARED_FOR_SPLIT
Definition: CoroInternal.h:40
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:650
llvm::createCoroEarlyLegacyPass
Pass * createCoroEarlyLegacyPass()
Lower coroutine intrinsics that are not needed by later passes.
llvm::CallBase::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition: Instructions.cpp:311
llvm::CoroPromiseInst::isFromPromise
bool isFromPromise() const
Are we translating from the frame to the promise (false) or from the promise to the frame (true)?
Definition: CoroInstr.h:458
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:213
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:179
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:253
llvm::Function::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: Function.h:243
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:116
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:520
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:100
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
CORO_PRESPLIT_ATTR
#define CORO_PRESPLIT_ATTR
Definition: CoroInternal.h:39
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:738
llvm::ReturnInst::Create
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3005
llvm::IntrinsicInst
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:45
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition: InstrTypes.h:1426
llvm::CallingConv::Fast
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:186
llvm::Pass
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:91
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
llvm::PreservedAnalyses::preserveSet
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:191
declaresCoroEarlyIntrinsics
static bool declaresCoroEarlyIntrinsics(const Module &M)
Definition: CoroEarly.cpp:227
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1338
llvm::coro::LowererBase
Definition: CoroInternal.h:60
CoroEarly.h
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1161
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
INITIALIZE_PASS
INITIALIZE_PASS(CoroEarlyLegacy, "coro-early", "Lower early coroutine intrinsics", false, false) Pass *llvm
Definition: CoroEarly.cpp:281
llvm::CallBase::setCannotDuplicate
void setCannotDuplicate()
Definition: InstrTypes.h:1864
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:422
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1453
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37