LLVM  16.0.0git
CoroEarly.cpp
Go to the documentation of this file.
1 //===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "CoroInternal.h"
11 #include "llvm/IR/Function.h"
12 #include "llvm/IR/IRBuilder.h"
13 #include "llvm/IR/InstIterator.h"
14 #include "llvm/IR/Module.h"
15 
16 using namespace llvm;
17 
18 #define DEBUG_TYPE "coro-early"
19 
20 namespace {
21 // Created on demand if the coro-early pass has work to do.
22 class Lowerer : public coro::LowererBase {
24  PointerType *const AnyResumeFnPtrTy;
25  Constant *NoopCoro = nullptr;
26 
27  void lowerResumeOrDestroy(CallBase &CB, CoroSubFnInst::ResumeKind);
28  void lowerCoroPromise(CoroPromiseInst *Intrin);
29  void lowerCoroDone(IntrinsicInst *II);
30  void lowerCoroNoop(IntrinsicInst *II);
31 
32 public:
33  Lowerer(Module &M)
34  : LowererBase(M), Builder(Context),
35  AnyResumeFnPtrTy(FunctionType::get(Type::getVoidTy(Context), Int8Ptr,
36  /*isVarArg=*/false)
37  ->getPointerTo()) {}
38  void lowerEarlyIntrinsics(Function &F);
39 };
40 }
41 
42 // Replace a direct call to coro.resume or coro.destroy with an indirect call to
43 // an address returned by coro.subfn.addr intrinsic. This is done so that
44 // CGPassManager recognizes devirtualization when CoroElide pass replaces a call
45 // to coro.subfn.addr with an appropriate function address.
46 void Lowerer::lowerResumeOrDestroy(CallBase &CB,
48  Value *ResumeAddr = makeSubFnCall(CB.getArgOperand(0), Index, &CB);
49  CB.setCalledOperand(ResumeAddr);
51 }
52 
53 // Coroutine promise field is always at the fixed offset from the beginning of
54 // the coroutine frame. i8* coro.promise(i8*, i1 from) intrinsic adds an offset
55 // to a passed pointer to move from coroutine frame to coroutine promise and
56 // vice versa. Since we don't know exactly which coroutine frame it is, we build
57 // a coroutine frame mock up starting with two function pointers, followed by a
58 // properly aligned coroutine promise field.
59 // TODO: Handle the case when coroutine promise alloca has align override.
60 void Lowerer::lowerCoroPromise(CoroPromiseInst *Intrin) {
61  Value *Operand = Intrin->getArgOperand(0);
62  Align Alignment = Intrin->getAlignment();
63  Type *Int8Ty = Builder.getInt8Ty();
64 
65  auto *SampleStruct =
66  StructType::get(Context, {AnyResumeFnPtrTy, AnyResumeFnPtrTy, Int8Ty});
67  const DataLayout &DL = TheModule.getDataLayout();
68  int64_t Offset = alignTo(
69  DL.getStructLayout(SampleStruct)->getElementOffset(2), Alignment);
70  if (Intrin->isFromPromise())
71  Offset = -Offset;
72 
73  Builder.SetInsertPoint(Intrin);
74  Value *Replacement =
75  Builder.CreateConstInBoundsGEP1_32(Int8Ty, Operand, Offset);
76 
77  Intrin->replaceAllUsesWith(Replacement);
78  Intrin->eraseFromParent();
79 }
80 
81 // When a coroutine reaches final suspend point, it zeros out ResumeFnAddr in
82 // the coroutine frame (it is UB to resume from a final suspend point).
83 // The llvm.coro.done intrinsic is used to check whether a coroutine is
84 // suspended at the final suspend point or not.
85 void Lowerer::lowerCoroDone(IntrinsicInst *II) {
86  Value *Operand = II->getArgOperand(0);
87 
88  // ResumeFnAddr is the first pointer sized element of the coroutine frame.
89  static_assert(coro::Shape::SwitchFieldIndex::Resume == 0,
90  "resume function not at offset zero");
91  auto *FrameTy = Int8Ptr;
92  PointerType *FramePtrTy = FrameTy->getPointerTo();
93 
94  Builder.SetInsertPoint(II);
95  auto *BCI = Builder.CreateBitCast(Operand, FramePtrTy);
96  auto *Load = Builder.CreateLoad(FrameTy, BCI);
97  auto *Cond = Builder.CreateICmpEQ(Load, NullPtr);
98 
100  II->eraseFromParent();
101 }
102 
103 void Lowerer::lowerCoroNoop(IntrinsicInst *II) {
104  if (!NoopCoro) {
105  LLVMContext &C = Builder.getContext();
106  Module &M = *II->getModule();
107 
108  // Create a noop.frame struct type.
109  StructType *FrameTy = StructType::create(C, "NoopCoro.Frame");
110  auto *FramePtrTy = FrameTy->getPointerTo();
111  auto *FnTy = FunctionType::get(Type::getVoidTy(C), FramePtrTy,
112  /*isVarArg=*/false);
113  auto *FnPtrTy = FnTy->getPointerTo();
114  FrameTy->setBody({FnPtrTy, FnPtrTy});
115 
116  // Create a Noop function that does nothing.
117  Function *NoopFn =
118  Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
119  "NoopCoro.ResumeDestroy", &M);
121  auto *Entry = BasicBlock::Create(C, "entry", NoopFn);
122  ReturnInst::Create(C, Entry);
123 
124  // Create a constant struct for the frame.
125  Constant* Values[] = {NoopFn, NoopFn};
126  Constant* NoopCoroConst = ConstantStruct::get(FrameTy, Values);
127  NoopCoro = new GlobalVariable(M, NoopCoroConst->getType(), /*isConstant=*/true,
128  GlobalVariable::PrivateLinkage, NoopCoroConst,
129  "NoopCoro.Frame.Const");
130  }
131 
132  Builder.SetInsertPoint(II);
133  auto *NoopCoroVoidPtr = Builder.CreateBitCast(NoopCoro, Int8Ptr);
134  II->replaceAllUsesWith(NoopCoroVoidPtr);
135  II->eraseFromParent();
136 }
137 
138 // Prior to CoroSplit, calls to coro.begin needs to be marked as NoDuplicate,
139 // as CoroSplit assumes there is exactly one coro.begin. After CoroSplit,
140 // NoDuplicate attribute will be removed from coro.begin otherwise, it will
141 // interfere with inlining.
142 static void setCannotDuplicate(CoroIdInst *CoroId) {
143  for (User *U : CoroId->users())
144  if (auto *CB = dyn_cast<CoroBeginInst>(U))
145  CB->setCannotDuplicate();
146 }
147 
148 void Lowerer::lowerEarlyIntrinsics(Function &F) {
149  CoroIdInst *CoroId = nullptr;
151  bool HasCoroSuspend = false;
153  auto *CB = dyn_cast<CallBase>(&I);
154  if (!CB)
155  continue;
156 
157  switch (CB->getIntrinsicID()) {
158  default:
159  continue;
160  case Intrinsic::coro_free:
161  CoroFrees.push_back(cast<CoroFreeInst>(&I));
162  break;
163  case Intrinsic::coro_suspend:
164  // Make sure that final suspend point is not duplicated as CoroSplit
165  // pass expects that there is at most one final suspend point.
166  if (cast<CoroSuspendInst>(&I)->isFinal())
167  CB->setCannotDuplicate();
168  HasCoroSuspend = true;
169  break;
170  case Intrinsic::coro_end_async:
171  case Intrinsic::coro_end:
172  // Make sure that fallthrough coro.end is not duplicated as CoroSplit
173  // pass expects that there is at most one fallthrough coro.end.
174  if (cast<AnyCoroEndInst>(&I)->isFallthrough())
175  CB->setCannotDuplicate();
176  break;
177  case Intrinsic::coro_noop:
178  lowerCoroNoop(cast<IntrinsicInst>(&I));
179  break;
180  case Intrinsic::coro_id:
181  if (auto *CII = cast<CoroIdInst>(&I)) {
182  if (CII->getInfo().isPreSplit()) {
183  assert(F.isPresplitCoroutine() &&
184  "The frontend uses Swtich-Resumed ABI should emit "
185  "\"coroutine.presplit\" attribute for the coroutine.");
186  setCannotDuplicate(CII);
187  CII->setCoroutineSelf();
188  CoroId = cast<CoroIdInst>(&I);
189  }
190  }
191  break;
192  case Intrinsic::coro_id_retcon:
193  case Intrinsic::coro_id_retcon_once:
194  case Intrinsic::coro_id_async:
195  F.setPresplitCoroutine();
196  break;
197  case Intrinsic::coro_resume:
198  lowerResumeOrDestroy(*CB, CoroSubFnInst::ResumeIndex);
199  break;
200  case Intrinsic::coro_destroy:
201  lowerResumeOrDestroy(*CB, CoroSubFnInst::DestroyIndex);
202  break;
203  case Intrinsic::coro_promise:
204  lowerCoroPromise(cast<CoroPromiseInst>(&I));
205  break;
206  case Intrinsic::coro_done:
207  lowerCoroDone(cast<IntrinsicInst>(&I));
208  break;
209  }
210  }
211 
212  // Make sure that all CoroFree reference the coro.id intrinsic.
213  // Token type is not exposed through coroutine C/C++ builtins to plain C, so
214  // we allow specifying none and fixing it up here.
215  if (CoroId)
216  for (CoroFreeInst *CF : CoroFrees)
217  CF->setArgOperand(0, CoroId);
218 
219  // Coroutine suspention could potentially lead to any argument modified
220  // outside of the function, hence arguments should not have noalias
221  // attributes.
222  if (HasCoroSuspend)
223  for (Argument &A : F.args())
224  if (A.hasNoAliasAttr())
225  A.removeAttr(Attribute::NoAlias);
226 }
227 
228 static bool declaresCoroEarlyIntrinsics(const Module &M) {
230  M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once",
231  "llvm.coro.id.async", "llvm.coro.destroy", "llvm.coro.done",
232  "llvm.coro.end", "llvm.coro.end.async", "llvm.coro.noop",
233  "llvm.coro.free", "llvm.coro.promise", "llvm.coro.resume",
234  "llvm.coro.suspend"});
235 }
236 
239  return PreservedAnalyses::all();
240 
241  Lowerer L(M);
242  for (auto &F : M)
243  L.lowerEarlyIntrinsics(F);
244 
246  PA.preserveSet<CFGAnalyses>();
247  return PA;
248 }
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:156
llvm::CoroSubFnInst::ResumeIndex
@ ResumeIndex
Definition: CoroInstr.h:41
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:65
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:406
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::CoroSubFnInst::DestroyIndex
@ DestroyIndex
Definition: CoroInstr.h:42
InstIterator.h
llvm::Function
Definition: Function.h:60
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1306
llvm::StructType::setBody
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
Definition: Type.cpp:445
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1183
llvm::CallingConv::Fast
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
llvm::CoroSubFnInst::ResumeKind
ResumeKind
Definition: CoroInstr.h:39
llvm::coro::Shape::SwitchFieldIndex::Resume
@ Resume
Definition: CoroInternal.h:84
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:117
llvm::CoroFreeInst
This represents the llvm.coro.free instruction.
Definition: CoroInstr.h:404
setCannotDuplicate
static void setCannotDuplicate(CoroIdInst *CoroId)
Definition: CoroEarly.cpp:142
llvm::StructType::create
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Definition: Type.cpp:513
F
#define F(x, y, z)
Definition: MD5.cpp:55
Context
LLVMContext & Context
Definition: NVVMIntrRange.cpp:66
llvm::User
Definition: User.h:44
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::ARM_PROC::A
@ A
Definition: ARMBaseInfo.h:34
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::CoroPromiseInst::getAlignment
Align getAlignment() const
The required alignment of the promise.
Definition: CoroInstr.h:464
llvm::Instruction
Definition: Instruction.h:42
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
CoroInternal.h
llvm::CoroPromiseInst
This represents the llvm.coro.promise instruction.
Definition: CoroInstr.h:452
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:77
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:416
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:619
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::CallBase::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition: Instructions.cpp:311
llvm::CoroPromiseInst::isFromPromise
bool isFromPromise() const
Are we translating from the frame to the promise (false) or from the promise to the frame (true)?
Definition: CoroInstr.h:458
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:213
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:137
llvm::Function::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: Function.h:242
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:113
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:97
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::CoroEarlyPass::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition: CoroEarly.cpp:237
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
Function.h
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:774
llvm::ReturnInst::Create
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3044
llvm::IntrinsicInst
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:46
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition: InstrTypes.h:1432
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:222
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
llvm::PreservedAnalyses::preserveSet
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:188
declaresCoroEarlyIntrinsics
static bool declaresCoroEarlyIntrinsics(const Module &M)
Definition: CoroEarly.cpp:228
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1341
llvm::coro::LowererBase
Definition: CoroInternal.h:35
CoroEarly.h
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1174
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
llvm::CallBase::setCannotDuplicate
void setCannotDuplicate()
Definition: InstrTypes.h:1904
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1459