LLVM  4.0.0
CoroElide.cpp
Go to the documentation of this file.
1 //===- CoroElide.cpp - Coroutine Frame Allocation Elision Pass ------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 // This pass replaces dynamic allocation of coroutine frame with alloca and
10 // replaces calls to llvm.coro.resume and llvm.coro.destroy with direct calls
11 // to coroutine sub-functions.
12 //===----------------------------------------------------------------------===//
13 
14 #include "CoroInternal.h"
17 #include "llvm/IR/InstIterator.h"
18 #include "llvm/Pass.h"
20 
21 using namespace llvm;
22 
23 #define DEBUG_TYPE "coro-elide"
24 
25 namespace {
26 // Created on demand if CoroElide pass has work to do.
27 struct Lowerer : coro::LowererBase {
34 
35  Lowerer(Module &M) : LowererBase(M) {}
36 
37  void elideHeapAllocations(Function *F, Type *FrameTy, AAResults &AA);
38  bool shouldElide() const;
39  bool processCoroId(CoroIdInst *, AAResults &AA);
40 };
41 } // end anonymous namespace
42 
43 // Go through the list of coro.subfn.addr intrinsics and replace them with the
44 // provided constant.
47  if (Users.empty())
48  return;
49 
50  // See if we need to bitcast the constant to match the type of the intrinsic
51  // being replaced. Note: All coro.subfn.addr intrinsics return the same type,
52  // so we only need to examine the type of the first one in the list.
53  Type *IntrTy = Users.front()->getType();
54  Type *ValueTy = Value->getType();
55  if (ValueTy != IntrTy) {
56  // May need to tweak the function type to match the type expected at the
57  // use site.
58  assert(ValueTy->isPointerTy() && IntrTy->isPointerTy());
59  Value = ConstantExpr::getBitCast(Value, IntrTy);
60  }
61 
62  // Now the value type matches the type of the intrinsic. Replace them all!
63  for (CoroSubFnInst *I : Users)
65 }
66 
67 // See if any operand of the call instruction references the coroutine frame.
68 static bool operandReferences(CallInst *CI, AllocaInst *Frame, AAResults &AA) {
69  for (Value *Op : CI->operand_values())
70  if (AA.alias(Op, Frame) != NoAlias)
71  return true;
72  return false;
73 }
74 
75 // Look for any tail calls referencing the coroutine frame and remove tail
76 // attribute from them, since now coroutine frame resides on the stack and tail
77 // call implies that the function does not references anything on the stack.
78 static void removeTailCallAttribute(AllocaInst *Frame, AAResults &AA) {
79  Function &F = *Frame->getFunction();
80  MemoryLocation Mem(Frame);
81  for (Instruction &I : instructions(F))
82  if (auto *Call = dyn_cast<CallInst>(&I))
83  if (Call->isTailCall() && operandReferences(Call, Frame, AA)) {
84  // FIXME: If we ever hit this check. Evaluate whether it is more
85  // appropriate to retain musttail and allow the code to compile.
86  if (Call->isMustTailCall())
87  report_fatal_error("Call referring to the coroutine frame cannot be "
88  "marked as musttail");
89  Call->setTailCall(false);
90  }
91 }
92 
93 // Given a resume function @f.resume(%f.frame* %frame), returns %f.frame type.
94 static Type *getFrameType(Function *Resume) {
95  auto *ArgType = Resume->getArgumentList().front().getType();
96  return cast<PointerType>(ArgType)->getElementType();
97 }
98 
99 // Finds first non alloca instruction in the entry block of a function.
101  for (Instruction &I : F->getEntryBlock())
102  if (!isa<AllocaInst>(&I))
103  return &I;
104  llvm_unreachable("no terminator in the entry block");
105 }
106 
107 // To elide heap allocations we need to suppress code blocks guarded by
108 // llvm.coro.alloc and llvm.coro.free instructions.
109 void Lowerer::elideHeapAllocations(Function *F, Type *FrameTy, AAResults &AA) {
110  LLVMContext &C = FrameTy->getContext();
111  auto *InsertPt =
112  getFirstNonAllocaInTheEntryBlock(CoroIds.front()->getFunction());
113 
114  // Replacing llvm.coro.alloc with false will suppress dynamic
115  // allocation as it is expected for the frontend to generate the code that
116  // looks like:
117  // id = coro.id(...)
118  // mem = coro.alloc(id) ? malloc(coro.size()) : 0;
119  // coro.begin(id, mem)
120  auto *False = ConstantInt::getFalse(C);
121  for (auto *CA : CoroAllocs) {
122  CA->replaceAllUsesWith(False);
123  CA->eraseFromParent();
124  }
125 
126  // FIXME: Design how to transmit alignment information for every alloca that
127  // is spilled into the coroutine frame and recreate the alignment information
128  // here. Possibly we will need to do a mini SROA here and break the coroutine
129  // frame into individual AllocaInst recreating the original alignment.
130  auto *Frame = new AllocaInst(FrameTy, "", InsertPt);
131  auto *FrameVoidPtr =
132  new BitCastInst(Frame, Type::getInt8PtrTy(C), "vFrame", InsertPt);
133 
134  for (auto *CB : CoroBegins) {
135  CB->replaceAllUsesWith(FrameVoidPtr);
136  CB->eraseFromParent();
137  }
138 
139  // Since now coroutine frame lives on the stack we need to make sure that
140  // any tail call referencing it, must be made non-tail call.
141  removeTailCallAttribute(Frame, AA);
142 }
143 
144 bool Lowerer::shouldElide() const {
145  // If no CoroAllocs, we cannot suppress allocation, so elision is not
146  // possible.
147  if (CoroAllocs.empty())
148  return false;
149 
150  // Check that for every coro.begin there is a coro.destroy directly
151  // referencing the SSA value of that coro.begin. If the value escaped, then
152  // coro.destroy would have been referencing a memory location storing that
153  // value and not the virtual register.
154 
155  SmallPtrSet<CoroBeginInst *, 8> ReferencedCoroBegins;
156 
157  for (CoroSubFnInst *DA : DestroyAddr) {
158  if (auto *CB = dyn_cast<CoroBeginInst>(DA->getFrame()))
159  ReferencedCoroBegins.insert(CB);
160  else
161  return false;
162  }
163 
164  // If size of the set is the same as total number of CoroBegins, means we
165  // found a coro.free or coro.destroy mentioning a coro.begin and we can
166  // perform heap elision.
167  return ReferencedCoroBegins.size() == CoroBegins.size();
168 }
169 
170 bool Lowerer::processCoroId(CoroIdInst *CoroId, AAResults &AA) {
171  CoroBegins.clear();
172  CoroAllocs.clear();
173  CoroFrees.clear();
174  ResumeAddr.clear();
175  DestroyAddr.clear();
176 
177  // Collect all coro.begin and coro.allocs associated with this coro.id.
178  for (User *U : CoroId->users()) {
179  if (auto *CB = dyn_cast<CoroBeginInst>(U))
180  CoroBegins.push_back(CB);
181  else if (auto *CA = dyn_cast<CoroAllocInst>(U))
182  CoroAllocs.push_back(CA);
183  else if (auto *CF = dyn_cast<CoroFreeInst>(U))
184  CoroFrees.push_back(CF);
185  }
186 
187  // Collect all coro.subfn.addrs associated with coro.begin.
188  // Note, we only devirtualize the calls if their coro.subfn.addr refers to
189  // coro.begin directly. If we run into cases where this check is too
190  // conservative, we can consider relaxing the check.
191  for (CoroBeginInst *CB : CoroBegins) {
192  for (User *U : CB->users())
193  if (auto *II = dyn_cast<CoroSubFnInst>(U))
194  switch (II->getIndex()) {
196  ResumeAddr.push_back(II);
197  break;
199  DestroyAddr.push_back(II);
200  break;
201  default:
202  llvm_unreachable("unexpected coro.subfn.addr constant");
203  }
204  }
205 
206  // PostSplit coro.id refers to an array of subfunctions in its Info
207  // argument.
208  ConstantArray *Resumers = CoroId->getInfo().Resumers;
209  assert(Resumers && "PostSplit coro.id Info argument must refer to an array"
210  "of coroutine subfunctions");
211  auto *ResumeAddrConstant =
213 
214  replaceWithConstant(ResumeAddrConstant, ResumeAddr);
215 
216  bool ShouldElide = shouldElide();
217 
218  auto *DestroyAddrConstant = ConstantExpr::getExtractValue(
219  Resumers,
221 
222  replaceWithConstant(DestroyAddrConstant, DestroyAddr);
223 
224  if (ShouldElide) {
225  auto *FrameTy = getFrameType(cast<Function>(ResumeAddrConstant));
226  elideHeapAllocations(CoroId->getFunction(), FrameTy, AA);
227  coro::replaceCoroFree(CoroId, /*Elide=*/true);
228  }
229 
230  return true;
231 }
232 
233 // See if there are any coro.subfn.addr instructions referring to coro.devirt
234 // trigger, if so, replace them with a direct call to devirt trigger function.
237  for (auto &I : instructions(F))
238  if (auto *SubFn = dyn_cast<CoroSubFnInst>(&I))
239  if (SubFn->getIndex() == CoroSubFnInst::RestartTrigger)
240  DevirtAddr.push_back(SubFn);
241 
242  if (DevirtAddr.empty())
243  return false;
244 
245  Module &M = *F.getParent();
247  assert(DevirtFn && "coro.devirt.fn not found");
248  replaceWithConstant(DevirtFn, DevirtAddr);
249 
250  return true;
251 }
252 
253 //===----------------------------------------------------------------------===//
254 // Top Level Driver
255 //===----------------------------------------------------------------------===//
256 
257 namespace {
258 struct CoroElide : FunctionPass {
259  static char ID;
260  CoroElide() : FunctionPass(ID) {}
261 
262  std::unique_ptr<Lowerer> L;
263 
264  bool doInitialization(Module &M) override {
265  if (coro::declaresIntrinsics(M, {"llvm.coro.id"}))
266  L = llvm::make_unique<Lowerer>(M);
267  return false;
268  }
269 
270  bool runOnFunction(Function &F) override {
271  if (!L)
272  return false;
273 
274  bool Changed = false;
275 
277  Changed = replaceDevirtTrigger(F);
278 
279  L->CoroIds.clear();
280 
281  // Collect all PostSplit coro.ids.
282  for (auto &I : instructions(F))
283  if (auto *CII = dyn_cast<CoroIdInst>(&I))
284  if (CII->getInfo().isPostSplit())
285  // If it is the coroutine itself, don't touch it.
286  if (CII->getCoroutine() != CII->getFunction())
287  L->CoroIds.push_back(CII);
288 
289  // If we did not find any coro.id, there is nothing to do.
290  if (L->CoroIds.empty())
291  return Changed;
292 
293  AAResults &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
294 
295  for (auto *CII : L->CoroIds)
296  Changed |= L->processCoroId(CII, AA);
297 
298  return Changed;
299  }
300  void getAnalysisUsage(AnalysisUsage &AU) const override {
302  }
303 };
304 }
305 
306 char CoroElide::ID = 0;
308  CoroElide, "coro-elide",
309  "Coroutine frame allocation elision and indirect calls replacement", false,
310  false)
313  CoroElide, "coro-elide",
314  "Coroutine frame allocation elision and indirect calls replacement", false,
315  false)
316 
317 Pass *llvm::createCoroElidePass() { return new CoroElide(); }
MachineLoop * L
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:81
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:513
This represents the llvm.coro.alloc instruction.
Definition: CoroInstr.h:79
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:52
coro elide
Definition: CoroElide.cpp:313
This class represents a function call, abstracting a target machine's calling convention.
libcalls Conditionally eliminate dead library calls
The two locations do not alias at all.
Definition: AliasAnalysis.h:79
iv Induction Variable Users
Definition: IVUsers.cpp:51
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB)
The main low level interface to the alias analysis implementation.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
static bool replaceDevirtTrigger(Function &F)
Definition: CoroElide.cpp:235
This class represents the llvm.coro.subfn.addr instruction.
Definition: CoroInstr.h:32
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
#define F(x, y, z)
Definition: MD5.cpp:51
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
This class represents a no-op cast from one type to another.
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:68
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition: Module.cpp:196
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1695
#define CORO_PRESPLIT_ATTR
Definition: CoroInternal.h:38
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
size_type size() const
Definition: SmallPtrSet.h:99
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:48
This is an important base class in LLVM.
Definition: Constant.h:42
Pass * createCoroElidePass()
Analyze coroutines use sites, devirtualize resume/destroy calls and elide heap allocation for corouti...
Definition: CoroElide.cpp:317
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
Represent the analysis usage information of a pass.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
static void removeTailCallAttribute(AllocaInst *Frame, AAResults &AA)
Definition: CoroElide.cpp:78
Info getInfo() const
Definition: CoroInstr.h:143
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:213
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:213
coro Coroutine frame allocation elision and indirect calls replacement
Definition: CoroElide.cpp:313
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static Type * getFrameType(Function *Resume)
Definition: CoroElide.cpp:94
Representation for a specific memory location.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
coro Coroutine frame allocation elision and indirect calls false
Definition: CoroElide.cpp:313
const BasicBlock & getEntryBlock() const
Definition: Function.h:519
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:212
ConstantArray - Constant Array Declarations.
Definition: Constants.h:411
ConstantArray * Resumers
Definition: CoroInstr.h:137
iterator_range< user_iterator > users()
Definition: Value.h:370
phi node Eliminate PHI nodes for register allocation
static Instruction * getFirstNonAllocaInTheEntryBlock(Function *F)
Definition: CoroElide.cpp:100
static bool operandReferences(CallInst *CI, AllocaInst *Frame, AAResults &AA)
Definition: CoroElide.cpp:68
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:226
#define I(x, y, z)
Definition: MD5.cpp:54
static void replaceWithConstant(Constant *Value, SmallVectorImpl< CoroSubFnInst * > &Users)
Definition: CoroElide.cpp:45
iterator_range< value_op_iterator > operand_values()
Definition: User.h:237
#define CORO_DEVIRT_TRIGGER_FN
Definition: CoroInternal.h:42
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:132
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:537
LLVM Value Representation.
Definition: Value.h:71
const ArgumentListType & getArgumentList() const
Get the underlying elements of the Function...
Definition: Function.h:499
bool declaresIntrinsics(Module &M, std::initializer_list< StringRef >)
Definition: Coroutines.cpp:118
bool empty() const
Definition: LoopInfo.h:136
static Constant * getExtractValue(Constant *Agg, ArrayRef< unsigned > Idxs, Type *OnlyIfReducedTy=nullptr)
Definition: Constants.cpp:2089
inst_range instructions(Function *F)
Definition: InstIterator.h:132
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
INITIALIZE_PASS_BEGIN(CoroElide,"coro-elide","Coroutine frame allocation elision and indirect calls replacement", false, false) INITIALIZE_PASS_END(CoroElide
bool replaceAndRecursivelySimplify(Instruction *I, Value *SimpleV, const TargetLibraryInfo *TLI=nullptr, const DominatorTree *DT=nullptr, AssumptionCache *AC=nullptr)
Replace all uses of 'I' with 'SimpleV' and simplify the uses recursively.
an instruction to allocate memory on the stack
Definition: Instructions.h:60