LLVM  3.7.0
StatepointLowering.cpp
Go to the documentation of this file.
1 //===-- StatepointLowering.cpp - SDAGBuilder's statepoint code -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file includes support code use by SelectionDAGBuilder when lowering a
11 // statepoint sequence in SelectionDAG IR.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "StatepointLowering.h"
16 #include "SelectionDAGBuilder.h"
17 #include "llvm/ADT/SmallSet.h"
18 #include "llvm/ADT/Statistic.h"
23 #include "llvm/CodeGen/StackMaps.h"
24 #include "llvm/IR/CallingConv.h"
25 #include "llvm/IR/Instructions.h"
26 #include "llvm/IR/IntrinsicInst.h"
27 #include "llvm/IR/Intrinsics.h"
28 #include "llvm/IR/Statepoint.h"
30 #include <algorithm>
31 using namespace llvm;
32 
33 #define DEBUG_TYPE "statepoint-lowering"
34 
35 STATISTIC(NumSlotsAllocatedForStatepoints,
36  "Number of stack slots allocated for statepoints");
37 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
38 STATISTIC(StatepointMaxSlotsRequired,
39  "Maximum number of stack slots required for a singe statepoint");
40 
42  SelectionDAGBuilder &Builder, uint64_t Value) {
43  SDLoc L = Builder.getCurSDLoc();
45  MVT::i64));
46  Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
47 }
48 
50  // Consistency check
51  assert(PendingGCRelocateCalls.empty() &&
52  "Trying to visit statepoint before finished processing previous one");
53  Locations.clear();
54  NextSlotToAllocate = 0;
55  // Need to resize this on each safepoint - we need the two to stay in
56  // sync and the clear patterns of a SelectionDAGBuilder have no relation
57  // to FunctionLoweringInfo.
58  AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
59  for (size_t i = 0; i < AllocatedStackSlots.size(); i++) {
60  AllocatedStackSlots[i] = false;
61  }
62 }
63 
65  Locations.clear();
66  AllocatedStackSlots.clear();
67  assert(PendingGCRelocateCalls.empty() &&
68  "cleared before statepoint sequence completed");
69 }
70 
71 SDValue
73  SelectionDAGBuilder &Builder) {
74 
75  NumSlotsAllocatedForStatepoints++;
76 
77  // The basic scheme here is to first look for a previously created stack slot
78  // which is not in use (accounting for the fact arbitrary slots may already
79  // be reserved), or to create a new stack slot and use it.
80 
81  // If this doesn't succeed in 40000 iterations, something is seriously wrong
82  for (int i = 0; i < 40000; i++) {
83  assert(Builder.FuncInfo.StatepointStackSlots.size() ==
84  AllocatedStackSlots.size() &&
85  "broken invariant");
86  const size_t NumSlots = AllocatedStackSlots.size();
87  assert(NextSlotToAllocate <= NumSlots && "broken invariant");
88 
89  if (NextSlotToAllocate >= NumSlots) {
90  assert(NextSlotToAllocate == NumSlots);
91  // record stats
92  if (NumSlots + 1 > StatepointMaxSlotsRequired) {
93  StatepointMaxSlotsRequired = NumSlots + 1;
94  }
95 
96  SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
97  const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
99  AllocatedStackSlots.push_back(true);
100  return SpillSlot;
101  }
102  if (!AllocatedStackSlots[NextSlotToAllocate]) {
103  const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
104  AllocatedStackSlots[NextSlotToAllocate] = true;
105  return Builder.DAG.getFrameIndex(FI, ValueType);
106  }
107  // Note: We deliberately choose to advance this only on the failing path.
108  // Doing so on the suceeding path involes a bit of complexity that caused a
109  // minor bug previously. Unless performance shows this matters, please
110  // keep this code as simple as possible.
111  NextSlotToAllocate++;
112  }
113  llvm_unreachable("infinite loop?");
114 }
115 
116 /// Utility function for reservePreviousStackSlotForValue. Tries to find
117 /// stack slot index to which we have spilled value for previous statepoints.
118 /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
120  SelectionDAGBuilder &Builder,
121  int LookUpDepth) {
122  // Can not look any futher - give up now
123  if (LookUpDepth <= 0)
124  return Optional<int>();
125 
126  // Spill location is known for gc relocates
127  if (isGCRelocate(Val)) {
128  GCRelocateOperands RelocOps(cast<Instruction>(Val));
129 
132 
133  auto It = SpillMap.find(RelocOps.getDerivedPtr());
134  if (It == SpillMap.end())
135  return Optional<int>();
136 
137  return It->second;
138  }
139 
140  // Look through bitcast instructions.
141  if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val)) {
142  return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
143  }
144 
145  // Look through phi nodes
146  // All incoming values should have same known stack slot, otherwise result
147  // is unknown.
148  if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
149  Optional<int> MergedResult = None;
150 
151  for (auto &IncomingValue : Phi->incoming_values()) {
152  Optional<int> SpillSlot =
153  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
154  if (!SpillSlot.hasValue())
155  return Optional<int>();
156 
157  if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
158  return Optional<int>();
159 
160  MergedResult = SpillSlot;
161  }
162  return MergedResult;
163  }
164 
165  // TODO: We can do better for PHI nodes. In cases like this:
166  // ptr = phi(relocated_pointer, not_relocated_pointer)
167  // statepoint(ptr)
168  // We will return that stack slot for ptr is unknown. And later we might
169  // assign different stack slots for ptr and relocated_pointer. This limits
170  // llvm's ability to remove redundant stores.
171  // Unfortunately it's hard to accomplish in current infrastructure.
172  // We use this function to eliminate spill store completely, while
173  // in example we still need to emit store, but instead of any location
174  // we need to use special "preferred" location.
175 
176  // TODO: handle simple updates. If a value is modified and the original
177  // value is no longer live, it would be nice to put the modified value in the
178  // same slot. This allows folding of the memory accesses for some
179  // instructions types (like an increment).
180  // statepoint (i)
181  // i1 = i+1
182  // statepoint (i1)
183  // However we need to be careful for cases like this:
184  // statepoint(i)
185  // i1 = i+1
186  // statepoint(i, i1)
187  // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
188  // put handling of simple modifications in this function like it's done
189  // for bitcasts we might end up reserving i's slot for 'i+1' because order in
190  // which we visit values is unspecified.
191 
192  // Don't know any information about this instruction
193  return Optional<int>();
194 }
195 
196 /// Try to find existing copies of the incoming values in stack slots used for
197 /// statepoint spilling. If we can find a spill slot for the incoming value,
198 /// mark that slot as allocated, and reuse the same slot for this safepoint.
199 /// This helps to avoid series of loads and stores that only serve to resuffle
200 /// values on the stack between calls.
201 static void reservePreviousStackSlotForValue(const Value *IncomingValue,
202  SelectionDAGBuilder &Builder) {
203 
204  SDValue Incoming = Builder.getValue(IncomingValue);
205 
206  if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
207  // We won't need to spill this, so no need to check for previously
208  // allocated stack slots
209  return;
210  }
211 
212  SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
213  if (OldLocation.getNode())
214  // duplicates in input
215  return;
216 
217  const int LookUpDepth = 6;
218  Optional<int> Index =
219  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
220  if (!Index.hasValue())
221  return;
222 
223  auto Itr = std::find(Builder.FuncInfo.StatepointStackSlots.begin(),
224  Builder.FuncInfo.StatepointStackSlots.end(), *Index);
225  assert(Itr != Builder.FuncInfo.StatepointStackSlots.end() &&
226  "value spilled to the unknown stack slot");
227 
228  // This is one of our dedicated lowering slots
229  const int Offset =
230  std::distance(Builder.FuncInfo.StatepointStackSlots.begin(), Itr);
231  if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
232  // stack slot already assigned to someone else, can't use it!
233  // TODO: currently we reserve space for gc arguments after doing
234  // normal allocation for deopt arguments. We should reserve for
235  // _all_ deopt and gc arguments, then start allocating. This
236  // will prevent some moves being inserted when vm state changes,
237  // but gc state doesn't between two calls.
238  return;
239  }
240  // Reserve this stack slot
241  Builder.StatepointLowering.reserveStackSlot(Offset);
242 
243  // Cache this slot so we find it when going through the normal
244  // assignment loop.
245  SDValue Loc = Builder.DAG.getTargetFrameIndex(*Index, Incoming.getValueType());
246  Builder.StatepointLowering.setLocation(Incoming, Loc);
247 }
248 
249 /// Remove any duplicate (as SDValues) from the derived pointer pairs. This
250 /// is not required for correctness. It's purpose is to reduce the size of
251 /// StackMap section. It has no effect on the number of spill slots required
252 /// or the actual lowering.
256  SelectionDAGBuilder &Builder) {
257 
258  // This is horribly ineffecient, but I don't care right now
260 
261  SmallVector<const Value *, 64> NewBases, NewPtrs, NewRelocs;
262  for (size_t i = 0; i < Ptrs.size(); i++) {
263  SDValue SD = Builder.getValue(Ptrs[i]);
264  // Only add non-duplicates
265  if (Seen.count(SD) == 0) {
266  NewBases.push_back(Bases[i]);
267  NewPtrs.push_back(Ptrs[i]);
268  NewRelocs.push_back(Relocs[i]);
269  }
270  Seen.insert(SD);
271  }
272  assert(Bases.size() >= NewBases.size());
273  assert(Ptrs.size() >= NewPtrs.size());
274  assert(Relocs.size() >= NewRelocs.size());
275  Bases = NewBases;
276  Ptrs = NewPtrs;
277  Relocs = NewRelocs;
278  assert(Ptrs.size() == Bases.size());
279  assert(Ptrs.size() == Relocs.size());
280 }
281 
282 /// Extract call from statepoint, lower it and return pointer to the
283 /// call node. Also update NodeMap so that getValue(statepoint) will
284 /// reference lowered call result
285 static SDNode *
287  SelectionDAGBuilder &Builder,
288  SmallVectorImpl<SDValue> &PendingExports) {
289 
291 
292  SDValue ActualCallee = Builder.getValue(ISP.getCalledValue());
293 
294  assert(CS.getCallingConv() != CallingConv::AnyReg &&
295  "anyregcc is not supported on statepoints!");
296 
297  Type *DefTy = ISP.getActualReturnType();
298  bool HasDef = !DefTy->isVoidTy();
299 
300  SDValue ReturnValue, CallEndVal;
301  std::tie(ReturnValue, CallEndVal) = Builder.lowerCallOperands(
303  ISP.getNumCallArgs(), ActualCallee, DefTy, LandingPad,
304  false /* IsPatchPoint */);
305 
306  SDNode *CallEnd = CallEndVal.getNode();
307 
308  // Get a call instruction from the call sequence chain. Tail calls are not
309  // allowed. The following code is essentially reverse engineering X86's
310  // LowerCallTo.
311  //
312  // We are expecting DAG to have the following form:
313  //
314  // ch = eh_label (only in case of invoke statepoint)
315  // ch, glue = callseq_start ch
316  // ch, glue = X86::Call ch, glue
317  // ch, glue = callseq_end ch, glue
318  // get_return_value ch, glue
319  //
320  // get_return_value can either be a CopyFromReg to grab the return value from
321  // %RAX, or it can be a LOAD to load a value returned by reference via a stack
322  // slot.
323 
324  if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg ||
325  CallEnd->getOpcode() == ISD::LOAD))
326  CallEnd = CallEnd->getOperand(0).getNode();
327 
328  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
329 
330  if (HasDef) {
331  if (CS.isInvoke()) {
332  // Result value will be used in different basic block for invokes
333  // so we need to export it now. But statepoint call has a different type
334  // than the actuall call. It means that standart exporting mechanism will
335  // create register of the wrong type. So instead we need to create
336  // register with correct type and save value into it manually.
337  // TODO: To eliminate this problem we can remove gc.result intrinsics
338  // completelly and make statepoint call to return a tuple.
339  unsigned Reg = Builder.FuncInfo.CreateRegs(ISP.getActualReturnType());
340  RegsForValue RFV(
341  *Builder.DAG.getContext(), Builder.DAG.getTargetLoweringInfo(),
342  Builder.DAG.getDataLayout(), Reg, ISP.getActualReturnType());
343  SDValue Chain = Builder.DAG.getEntryNode();
344 
345  RFV.getCopyToRegs(ReturnValue, Builder.DAG, Builder.getCurSDLoc(), Chain,
346  nullptr);
347  PendingExports.push_back(Chain);
348  Builder.FuncInfo.ValueMap[CS.getInstruction()] = Reg;
349  } else {
350  // The value of the statepoint itself will be the value of call itself.
351  // We'll replace the actually call node shortly. gc_result will grab
352  // this value.
353  Builder.setValue(CS.getInstruction(), ReturnValue);
354  }
355  } else {
356  // The token value is never used from here on, just generate a poison value
357  Builder.setValue(CS.getInstruction(),
358  Builder.DAG.getIntPtrConstant(-1, Builder.getCurSDLoc()));
359  }
360 
361  return CallEnd->getOperand(0).getNode();
362 }
363 
364 /// Callect all gc pointers coming into statepoint intrinsic, clean them up,
365 /// and return two arrays:
366 /// Bases - base pointers incoming to this statepoint
367 /// Ptrs - derived pointers incoming to this statepoint
368 /// Relocs - the gc_relocate corresponding to each base/ptr pair
369 /// Elements of this arrays should be in one-to-one correspondence with each
370 /// other i.e Bases[i], Ptrs[i] are from the same gcrelocate call
373  SmallVectorImpl<const Value *> &Relocs, ImmutableStatepoint StatepointSite,
374  SelectionDAGBuilder &Builder) {
375  for (GCRelocateOperands relocateOpers : StatepointSite.getRelocates()) {
376  Relocs.push_back(relocateOpers.getUnderlyingCallSite().getInstruction());
377  Bases.push_back(relocateOpers.getBasePtr());
378  Ptrs.push_back(relocateOpers.getDerivedPtr());
379  }
380 
381  // Remove any redundant llvm::Values which map to the same SDValue as another
382  // input. Also has the effect of removing duplicates in the original
383  // llvm::Value input list as well. This is a useful optimization for
384  // reducing the size of the StackMap section. It has no other impact.
385  removeDuplicatesGCPtrs(Bases, Ptrs, Relocs, Builder);
386 
387  assert(Bases.size() == Ptrs.size() && Ptrs.size() == Relocs.size());
388 }
389 
390 /// Spill a value incoming to the statepoint. It might be either part of
391 /// vmstate
392 /// or gcstate. In both cases unconditionally spill it on the stack unless it
393 /// is a null constant. Return pair with first element being frame index
394 /// containing saved value and second element with outgoing chain from the
395 /// emitted store
396 static std::pair<SDValue, SDValue>
398  SelectionDAGBuilder &Builder) {
399  SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
400 
401  // Emit new store if we didn't do it for this ptr before
402  if (!Loc.getNode()) {
403  Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
404  Builder);
405  assert(isa<FrameIndexSDNode>(Loc));
406  int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
407  // We use TargetFrameIndex so that isel will not select it into LEA
408  Loc = Builder.DAG.getTargetFrameIndex(Index, Incoming.getValueType());
409 
410  // TODO: We can create TokenFactor node instead of
411  // chaining stores one after another, this may allow
412  // a bit more optimal scheduling for them
413  Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
415  false, false, 0);
416 
417  Builder.StatepointLowering.setLocation(Incoming, Loc);
418  }
419 
420  assert(Loc.getNode());
421  return std::make_pair(Loc, Chain);
422 }
423 
424 /// Lower a single value incoming to a statepoint node. This value can be
425 /// either a deopt value or a gc value, the handling is the same. We special
426 /// case constants and allocas, then fall back to spilling if required.
429  SelectionDAGBuilder &Builder) {
430  SDValue Chain = Builder.getRoot();
431 
432  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
433  // If the original value was a constant, make sure it gets recorded as
434  // such in the stackmap. This is required so that the consumer can
435  // parse any internal format to the deopt state. It also handles null
436  // pointers and other constant pointers in GC states
437  pushStackMapConstant(Ops, Builder, C->getSExtValue());
438  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
439  // This handles allocas as arguments to the statepoint (this is only
440  // really meaningful for a deopt value. For GC, we'd be trying to
441  // relocate the address of the alloca itself?)
442  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
443  Incoming.getValueType()));
444  } else {
445  // Otherwise, locate a spill slot and explicitly spill it so it
446  // can be found by the runtime later. We currently do not support
447  // tracking values through callee saved registers to their eventual
448  // spill location. This would be a useful optimization, but would
449  // need to be optional since it requires a lot of complexity on the
450  // runtime side which not all would support.
451  std::pair<SDValue, SDValue> Res =
452  spillIncomingStatepointValue(Incoming, Chain, Builder);
453  Ops.push_back(Res.first);
454  Chain = Res.second;
455  }
456 
457  Builder.DAG.setRoot(Chain);
458 }
459 
460 /// Lower deopt state and gc pointer arguments of the statepoint. The actual
461 /// lowering is described in lowerIncomingStatepointValue. This function is
462 /// responsible for lowering everything in the right position and playing some
463 /// tricks to avoid redundant stack manipulation where possible. On
464 /// completion, 'Ops' will contain ready to use operands for machine code
465 /// statepoint. The chain nodes will have already been created and the DAG root
466 /// will be set to the last value spilled (if any were).
468  ImmutableStatepoint StatepointSite,
469  SelectionDAGBuilder &Builder) {
470 
471  // Lower the deopt and gc arguments for this statepoint. Layout will
472  // be: deopt argument length, deopt arguments.., gc arguments...
473 
474  SmallVector<const Value *, 64> Bases, Ptrs, Relocations;
475  getIncomingStatepointGCValues(Bases, Ptrs, Relocations, StatepointSite,
476  Builder);
477 
478 #ifndef NDEBUG
479  // Check that each of the gc pointer and bases we've gotten out of the
480  // safepoint is something the strategy thinks might be a pointer into the GC
481  // heap. This is basically just here to help catch errors during statepoint
482  // insertion. TODO: This should actually be in the Verifier, but we can't get
483  // to the GCStrategy from there (yet).
484  GCStrategy &S = Builder.GFI->getStrategy();
485  for (const Value *V : Bases) {
486  auto Opt = S.isGCManagedPointer(V);
487  if (Opt.hasValue()) {
488  assert(Opt.getValue() &&
489  "non gc managed base pointer found in statepoint");
490  }
491  }
492  for (const Value *V : Ptrs) {
493  auto Opt = S.isGCManagedPointer(V);
494  if (Opt.hasValue()) {
495  assert(Opt.getValue() &&
496  "non gc managed derived pointer found in statepoint");
497  }
498  }
499  for (const Value *V : Relocations) {
500  auto Opt = S.isGCManagedPointer(V);
501  if (Opt.hasValue()) {
502  assert(Opt.getValue() && "non gc managed pointer relocated");
503  }
504  }
505 #endif
506 
507  // Before we actually start lowering (and allocating spill slots for values),
508  // reserve any stack slots which we judge to be profitable to reuse for a
509  // particular value. This is purely an optimization over the code below and
510  // doesn't change semantics at all. It is important for performance that we
511  // reserve slots for both deopt and gc values before lowering either.
512  for (const Value *V : StatepointSite.vm_state_args()) {
514  }
515  for (unsigned i = 0; i < Bases.size(); ++i) {
516  reservePreviousStackSlotForValue(Bases[i], Builder);
517  reservePreviousStackSlotForValue(Ptrs[i], Builder);
518  }
519 
520  // First, prefix the list with the number of unique values to be
521  // lowered. Note that this is the number of *Values* not the
522  // number of SDValues required to lower them.
523  const int NumVMSArgs = StatepointSite.getNumTotalVMSArgs();
524  pushStackMapConstant(Ops, Builder, NumVMSArgs);
525 
526  assert(NumVMSArgs == std::distance(StatepointSite.vm_state_begin(),
527  StatepointSite.vm_state_end()));
528 
529  // The vm state arguments are lowered in an opaque manner. We do
530  // not know what type of values are contained within. We skip the
531  // first one since that happens to be the total number we lowered
532  // explicitly just above. We could have left it in the loop and
533  // not done it explicitly, but it's far easier to understand this
534  // way.
535  for (const Value *V : StatepointSite.vm_state_args()) {
536  SDValue Incoming = Builder.getValue(V);
537  lowerIncomingStatepointValue(Incoming, Ops, Builder);
538  }
539 
540  // Finally, go ahead and lower all the gc arguments. There's no prefixed
541  // length for this one. After lowering, we'll have the base and pointer
542  // arrays interwoven with each (lowered) base pointer immediately followed by
543  // it's (lowered) derived pointer. i.e
544  // (base[0], ptr[0], base[1], ptr[1], ...)
545  for (unsigned i = 0; i < Bases.size(); ++i) {
546  const Value *Base = Bases[i];
547  lowerIncomingStatepointValue(Builder.getValue(Base), Ops, Builder);
548 
549  const Value *Ptr = Ptrs[i];
550  lowerIncomingStatepointValue(Builder.getValue(Ptr), Ops, Builder);
551  }
552 
553  // If there are any explicit spill slots passed to the statepoint, record
554  // them, but otherwise do not do anything special. These are user provided
555  // allocas and give control over placement to the consumer. In this case,
556  // it is the contents of the slot which may get updated, not the pointer to
557  // the alloca
558  for (Value *V : StatepointSite.gc_args()) {
559  SDValue Incoming = Builder.getValue(V);
560  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
561  // This handles allocas as arguments to the statepoint
562  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
563  Incoming.getValueType()));
564  }
565  }
566 
567  // Record computed locations for all lowered values.
568  // This can not be embedded in lowering loops as we need to record *all*
569  // values, while previous loops account only values with unique SDValues.
570  const Instruction *StatepointInstr =
571  StatepointSite.getCallSite().getInstruction();
573  Builder.FuncInfo.StatepointRelocatedValues[StatepointInstr];
574 
575  for (GCRelocateOperands RelocateOpers : StatepointSite.getRelocates()) {
576  const Value *V = RelocateOpers.getDerivedPtr();
577  SDValue SDV = Builder.getValue(V);
578  SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
579 
580  if (Loc.getNode()) {
581  SpillMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
582  } else {
583  // Record value as visited, but not spilled. This is case for allocas
584  // and constants. For this values we can avoid emiting spill load while
585  // visiting corresponding gc_relocate.
586  // Actually we do not need to record them in this map at all.
587  // We do this only to check that we are not relocating any unvisited value.
588  SpillMap[V] = None;
589 
590  // Default llvm mechanisms for exporting values which are used in
591  // different basic blocks does not work for gc relocates.
592  // Note that it would be incorrect to teach llvm that all relocates are
593  // uses of the corresponging values so that it would automatically
594  // export them. Relocates of the spilled values does not use original
595  // value.
596  if (StatepointSite.getCallSite().isInvoke())
597  Builder.ExportFromCurrentBlock(V);
598  }
599  }
600 }
601 
602 void SelectionDAGBuilder::visitStatepoint(const CallInst &CI) {
603  // Check some preconditions for sanity
604  assert(isStatepoint(&CI) &&
605  "function called must be the statepoint function");
606 
608 }
609 
611  ImmutableStatepoint ISP, MachineBasicBlock *LandingPad /*=nullptr*/) {
612  // The basic scheme here is that information about both the original call and
613  // the safepoint is encoded in the CallInst. We create a temporary call and
614  // lower it, then reverse engineer the calling sequence.
615 
616  NumOfStatepoints++;
617  // Clear state
619 
621 
622 #ifndef NDEBUG
623  // Consistency check. Don't do this for invokes. It would be too
624  // expensive to preserve this information across different basic blocks
625  if (!CS.isInvoke()) {
626  for (const User *U : CS->users()) {
627  const CallInst *Call = cast<CallInst>(U);
628  if (isGCRelocate(Call))
630  }
631  }
632 #endif
633 
634 #ifndef NDEBUG
635  // If this is a malformed statepoint, report it early to simplify debugging.
636  // This should catch any IR level mistake that's made when constructing or
637  // transforming statepoints.
638  ISP.verify();
639 
640  // Check that the associated GCStrategy expects to encounter statepoints.
641  assert(GFI->getStrategy().useStatepoints() &&
642  "GCStrategy does not expect to encounter statepoints");
643 #endif
644 
645  // Lower statepoint vmstate and gcstate arguments
646  SmallVector<SDValue, 10> LoweredMetaArgs;
647  lowerStatepointMetaArgs(LoweredMetaArgs, ISP, *this);
648 
649  // Get call node, we will replace it later with statepoint
650  SDNode *CallNode =
651  lowerCallFromStatepoint(ISP, LandingPad, *this, PendingExports);
652 
653  // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
654  // nodes with all the appropriate arguments and return values.
655 
656  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
657  SDValue Chain = CallNode->getOperand(0);
658 
659  SDValue Glue;
660  bool CallHasIncomingGlue = CallNode->getGluedNode();
661  if (CallHasIncomingGlue) {
662  // Glue is always last operand
663  Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
664  }
665 
666  // Build the GC_TRANSITION_START node if necessary.
667  //
668  // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
669  // order in which they appear in the call to the statepoint intrinsic. If
670  // any of the operands is a pointer-typed, that operand is immediately
671  // followed by a SRCVALUE for the pointer that may be used during lowering
672  // (e.g. to form MachinePointerInfo values for loads/stores).
673  const bool IsGCTransition =
674  (ISP.getFlags() & (uint64_t)StatepointFlags::GCTransition) ==
676  if (IsGCTransition) {
678 
679  // Add chain
680  TSOps.push_back(Chain);
681 
682  // Add GC transition arguments
683  for (const Value *V : ISP.gc_transition_args()) {
684  TSOps.push_back(getValue(V));
685  if (V->getType()->isPointerTy())
686  TSOps.push_back(DAG.getSrcValue(V));
687  }
688 
689  // Add glue if necessary
690  if (CallHasIncomingGlue)
691  TSOps.push_back(Glue);
692 
694 
695  SDValue GCTransitionStart =
696  DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
697 
698  Chain = GCTransitionStart.getValue(0);
699  Glue = GCTransitionStart.getValue(1);
700  }
701 
702  // TODO: Currently, all of these operands are being marked as read/write in
703  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
704  // and flags to be read-only.
706 
707  // Add the <id> and <numBytes> constants.
709  Ops.push_back(
711 
712  // Calculate and push starting position of vmstate arguments
713  // Get number of arguments incoming directly into call node
714  unsigned NumCallRegArgs =
715  CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
716  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
717 
718  // Add call target
719  SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
720  Ops.push_back(CallTarget);
721 
722  // Add call arguments
723  // Get position of register mask in the call
724  SDNode::op_iterator RegMaskIt;
725  if (CallHasIncomingGlue)
726  RegMaskIt = CallNode->op_end() - 2;
727  else
728  RegMaskIt = CallNode->op_end() - 1;
729  Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
730 
731  // Add a constant argument for the calling convention
732  pushStackMapConstant(Ops, *this, CS.getCallingConv());
733 
734  // Add a constant argument for the flags
735  uint64_t Flags = ISP.getFlags();
736  assert(
737  ((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0)
738  && "unknown flag used");
739  pushStackMapConstant(Ops, *this, Flags);
740 
741  // Insert all vmstate and gcstate arguments
742  Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
743 
744  // Add register mask from call node
745  Ops.push_back(*RegMaskIt);
746 
747  // Add chain
748  Ops.push_back(Chain);
749 
750  // Same for the glue, but we add it only if original call had it
751  if (Glue.getNode())
752  Ops.push_back(Glue);
753 
754  // Compute return values. Provide a glue output since we consume one as
755  // input. This allows someone else to chain off us as needed.
757 
758  SDNode *StatepointMCNode =
760 
761  SDNode *SinkNode = StatepointMCNode;
762 
763  // Build the GC_TRANSITION_END node if necessary.
764  //
765  // See the comment above regarding GC_TRANSITION_START for the layout of
766  // the operands to the GC_TRANSITION_END node.
767  if (IsGCTransition) {
769 
770  // Add chain
771  TEOps.push_back(SDValue(StatepointMCNode, 0));
772 
773  // Add GC transition arguments
774  for (const Value *V : ISP.gc_transition_args()) {
775  TEOps.push_back(getValue(V));
776  if (V->getType()->isPointerTy())
777  TEOps.push_back(DAG.getSrcValue(V));
778  }
779 
780  // Add glue
781  TEOps.push_back(SDValue(StatepointMCNode, 1));
782 
784 
785  SDValue GCTransitionStart =
786  DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
787 
788  SinkNode = GCTransitionStart.getNode();
789  }
790 
791  // Replace original call
792  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
793  // Remove originall call node
794  DAG.DeleteNode(CallNode);
795 
796  // DON'T set the root - under the assumption that it's already set past the
797  // inserted node we created.
798 
799  // TODO: A better future implementation would be to emit a single variable
800  // argument, variable return value STATEPOINT node here and then hookup the
801  // return value of each gc.relocate to the respective output of the
802  // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
803  // to actually be possible today.
804 }
805 
806 void SelectionDAGBuilder::visitGCResult(const CallInst &CI) {
807  // The result value of the gc_result is simply the result of the actual
808  // call. We've already emitted this, so just grab the value.
809  Instruction *I = cast<Instruction>(CI.getArgOperand(0));
810  assert(isStatepoint(I) && "first argument must be a statepoint token");
811 
812  if (isa<InvokeInst>(I)) {
813  // For invokes we should have stored call result in a virtual register.
814  // We can not use default getValue() functionality to copy value from this
815  // register because statepoint and actuall call return types can be
816  // different, and getValue() will use CopyFromReg of the wrong type,
817  // which is always i32 in our case.
818  PointerType *CalleeType = cast<PointerType>(
820  Type *RetTy =
821  cast<FunctionType>(CalleeType->getElementType())->getReturnType();
822  SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
823 
824  assert(CopyFromReg.getNode());
825  setValue(&CI, CopyFromReg);
826  } else {
827  setValue(&CI, getValue(I));
828  }
829 }
830 
831 void SelectionDAGBuilder::visitGCRelocate(const CallInst &CI) {
832  GCRelocateOperands RelocateOpers(&CI);
833 
834 #ifndef NDEBUG
835  // Consistency check
836  // We skip this check for invoke statepoints. It would be too expensive to
837  // preserve validation info through different basic blocks.
838  if (!RelocateOpers.isTiedToInvoke()) {
840  }
841 #endif
842 
843  const Value *DerivedPtr = RelocateOpers.getDerivedPtr();
844  SDValue SD = getValue(DerivedPtr);
845 
847  FuncInfo.StatepointRelocatedValues[RelocateOpers.getStatepoint()];
848 
849  // We should have recorded location for this pointer
850  assert(SpillMap.count(DerivedPtr) && "Relocating not lowered gc value");
851  Optional<int> DerivedPtrLocation = SpillMap[DerivedPtr];
852 
853  // We didn't need to spill these special cases (constants and allocas).
854  // See the handling in spillIncomingValueForStatepoint for detail.
855  if (!DerivedPtrLocation) {
856  setValue(&CI, SD);
857  return;
858  }
859 
860  SDValue SpillSlot = DAG.getTargetFrameIndex(*DerivedPtrLocation,
861  SD.getValueType());
862 
863  // Be conservative: flush all pending loads
864  // TODO: Probably we can be less restrictive on this,
865  // it may allow more scheduling opprtunities
866  SDValue Chain = getRoot();
867 
868  SDValue SpillLoad =
869  DAG.getLoad(SpillSlot.getValueType(), getCurSDLoc(), Chain, SpillSlot,
870  MachinePointerInfo::getFixedStack(*DerivedPtrLocation),
871  false, false, false, 0);
872 
873  // Again, be conservative, don't emit pending loads
874  DAG.setRoot(SpillLoad.getValue(1));
875 
876  assert(SpillLoad.getNode());
877  setValue(&CI, SpillLoad);
878 }
const NoneType None
Definition: None.h:23
SDValue getValue(unsigned R) const
ValueTy * getCalledValue() const
Return the value actually being called or invoked.
Definition: Statepoint.h:121
LLVMContext * getContext() const
Definition: SelectionDAG.h:289
STATISTIC(NumFunctions,"Total number of functions")
bool hasValue() const
Definition: Optional.h:125
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block...
void LowerStatepoint(ImmutableStatepoint Statepoint, MachineBasicBlock *LandingPad=nullptr)
InstrTy * getInstruction() const
Definition: CallSite.h:82
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it's base class for read only access to a gc.statepoint.
Definition: Statepoint.h:283
DenseMap< const Instruction *, StatepointSpilledValueMapTy > StatepointRelocatedValues
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
CallInst - This class represents a function call, abstracting a target machine's calling convention...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
void setValue(const Value *V, SDValue NewN)
CallSiteTy::arg_iterator vm_state_begin() const
Definition: Statepoint.h:215
SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:344
void DeleteNode(SDNode *N)
Remove the specified node from the system.
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling...
static MachinePointerInfo getFixedStack(int FI, int64_t offset=0)
getFixedStack - Return a MachinePointerInfo record that refers to the the specified FrameIndex...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
SDValue getRoot()
getRoot - Return the current virtual root of the Selection DAG, flushing any PendingLoad items...
iterator_range< arg_iterator > gc_transition_args() const
range adapter for GC transition arguments
Definition: Statepoint.h:203
SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
static SDNode * lowerCallFromStatepoint(ImmutableStatepoint ISP, MachineBasicBlock *LandingPad, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
Reg
All possible values of the reg field in the ModR/M byte.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
CallSiteTy getCallSite() const
Return the underlying CallSite.
Definition: Statepoint.h:95
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
bool isStatepoint(const ImmutableCallSite &CS)
Definition: Statepoint.cpp:22
bool useStatepoints() const
Returns true if this strategy is expecting the use of gc.statepoints, and false otherwise.
Definition: GCStrategy.h:115
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:473
This class represents a no-op cast from one type to another.
Type * getElementType() const
Definition: DerivedTypes.h:323
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:284
PointerType - Class to represent pointers.
Definition: DerivedTypes.h:449
SDNode * getNode() const
get the SDNode which holds the desired result
static void lowerIncomingStatepointValue(SDValue Incoming, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
iterator_range< arg_iterator > vm_state_args() const
range adapter for vm state arguments
Definition: Statepoint.h:227
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:719
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
static std::pair< SDValue, SDValue > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
bool isGCRelocate(const Value *V)
Definition: Statepoint.cpp:50
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:32
iterator_range< arg_iterator > gc_args() const
range adapter for gc arguments
Definition: Statepoint.h:239
GCStrategy & getStrategy()
getStrategy - Return the GC strategy for the function.
Definition: GCMetadata.h:106
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
void clear()
Clear the memory usage of this object.
SDValue getTargetConstant(uint64_t Val, SDLoc DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:436
void verify()
Asserts if this statepoint is malformed.
Definition: Statepoint.h:264
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:69
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
Wraps a call to a gc.relocate and provides access to it's operands.
Definition: Statepoint.h:308
virtual Optional< bool > isGCManagedPointer(const Value *V) const
If the value specified can be reliably distinguished, returns true for pointers to GC managed locatio...
Definition: GCStrategy.h:124
EVT - Extended Value Type.
Definition: ValueTypes.h:31
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:53
CallSiteTy::arg_iterator vm_state_end() const
Definition: Statepoint.h:220
RegsForValue - This struct represents the registers (physical or virtual) that a particular set of va...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
uint64_t getID() const
Return the ID associated with this statepoint.
Definition: Statepoint.h:106
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Type * getActualReturnType() const
Return the type of the value returned by the call underlying the statepoint.
Definition: Statepoint.h:146
int getNumCallArgs() const
Number of arguments to be passed to the actual callee.
Definition: Statepoint.h:153
unsigned CreateRegs(Type *Ty)
CreateRegs - Allocate the appropriate number of virtual registers of the correctly promoted or expand...
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:119
void setLocation(SDValue val, SDValue Location)
GCFunctionInfo * GFI
GFI - Garbage collection metadata for the function.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
op_iterator op_begin() const
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const Instruction * getStatepoint()
The statepoint with which this gc.relocate is associated.
Definition: Statepoint.h:330
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:481
Represents a use of a SDNode.
GCStrategy describes a garbage collector algorithm's code generation requirements, and provides overridable hooks for those needs which cannot be abstractly described.
Definition: GCStrategy.h:78
static void getIncomingStatepointGCValues(SmallVectorImpl< const Value * > &Bases, SmallVectorImpl< const Value * > &Ptrs, SmallVectorImpl< const Value * > &Relocs, ImmutableStatepoint StatepointSite, SelectionDAGBuilder &Builder)
Callect all gc pointers coming into statepoint intrinsic, clean them up, and return two arrays: Bases...
static void removeDuplicatesGCPtrs(SmallVectorImpl< const Value * > &Bases, SmallVectorImpl< const Value * > &Ptrs, SmallVectorImpl< const Value * > &Relocs, SelectionDAGBuilder &Builder)
Remove any duplicate (as SDValues) from the derived pointer pairs.
std::vector< GCRelocateOperands > getRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition: Statepoint.h:378
Call instruction with associated vm state for deoptimization and list of live pointers for relocation...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:518
ImmutableCallSite - establish a view to a call site for examination.
Definition: CallSite.h:418
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
#define I(x, y, z)
Definition: MD5.cpp:54
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
iterator end()
Definition: DenseMap.h:68
int getNumTotalVMSArgs() const
Number of additional arguments excluding those intended for garbage collection.
Definition: Statepoint.h:210
std::pair< SDValue, SDValue > lowerCallOperands(ImmutableCallSite CS, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, MachineBasicBlock *LandingPad=nullptr, bool IsPatchPoint=false)
Lower an argument list according to the target calling convention.
iterator find(const KeyT &Val)
Definition: DenseMap.h:124
op_iterator op_end() const
MachineSDNode * getMachineNode(unsigned Opcode, SDLoc dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:166
SDValue getCopyFromRegs(const Value *V, Type *Ty)
getCopyFromRegs - If there was virtual register allocated for the value V emit CopyFromReg of the spe...
EVT getValueType() const
Return the ValueType of the referenced return value.
FunctionLoweringInfo & FuncInfo
FuncInfo - Information about the function as a whole.
SDValue getLocation(SDValue val)
Returns the spill location of a value incoming to the current statepoint.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM Value Representation.
Definition: Value.h:69
void scheduleRelocCall(const CallInst &RelocCall)
Record the fact that we expect to encounter a given gc_relocate before the next statepoint.
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:287
uint32_t getNumPatchBytes() const
Return the number of patchable bytes associated with this statepoint.
Definition: Statepoint.h:112
void relocCallVisited(const CallInst &RelocCall)
Remove this gc_relocate from the list we're expecting to see before the next statepoint.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, ImmutableStatepoint StatepointSite, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:338
uint64_t getFlags() const
Definition: Statepoint.h:100
bool isInvoke() const
isInvoke - true if a InvokeInst is enclosed.
Definition: CallSite.h:80
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
A bitmask that includes all valid flags.
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
SDValue getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget=false)
This file describes how to lower LLVM code to machine code.
bool isVoidTy() const
isVoidTy - Return true if this is 'void'.
Definition: Type.h:137
void resize(size_type N)
Definition: SmallVector.h:376