LLVM  6.0.0svn
StatepointLowering.cpp
Go to the documentation of this file.
1 //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file includes support code use by SelectionDAGBuilder when lowering a
11 // statepoint sequence in SelectionDAG IR.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "StatepointLowering.h"
16 #include "SelectionDAGBuilder.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/DenseMap.h"
19 #include "llvm/ADT/None.h"
20 #include "llvm/ADT/Optional.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/Statistic.h"
35 #include "llvm/CodeGen/StackMaps.h"
38 #include "llvm/IR/CallingConv.h"
39 #include "llvm/IR/DerivedTypes.h"
40 #include "llvm/IR/Instruction.h"
41 #include "llvm/IR/Instructions.h"
42 #include "llvm/IR/LLVMContext.h"
43 #include "llvm/IR/Statepoint.h"
44 #include "llvm/IR/Type.h"
45 #include "llvm/Support/Casting.h"
48 #include <cassert>
49 #include <cstddef>
50 #include <cstdint>
51 #include <iterator>
52 #include <tuple>
53 #include <utility>
54 
55 using namespace llvm;
56 
57 #define DEBUG_TYPE "statepoint-lowering"
58 
59 STATISTIC(NumSlotsAllocatedForStatepoints,
60  "Number of stack slots allocated for statepoints");
61 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
62 STATISTIC(StatepointMaxSlotsRequired,
63  "Maximum number of stack slots required for a singe statepoint");
64 
66  SelectionDAGBuilder &Builder, uint64_t Value) {
67  SDLoc L = Builder.getCurSDLoc();
68  Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
69  MVT::i64));
70  Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
71 }
72 
74  // Consistency check
75  assert(PendingGCRelocateCalls.empty() &&
76  "Trying to visit statepoint before finished processing previous one");
77  Locations.clear();
78  NextSlotToAllocate = 0;
79  // Need to resize this on each safepoint - we need the two to stay in sync and
80  // the clear patterns of a SelectionDAGBuilder have no relation to
81  // FunctionLoweringInfo. Also need to ensure used bits get cleared.
82  AllocatedStackSlots.clear();
83  AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
84 }
85 
87  Locations.clear();
88  AllocatedStackSlots.clear();
89  assert(PendingGCRelocateCalls.empty() &&
90  "cleared before statepoint sequence completed");
91 }
92 
93 SDValue
95  SelectionDAGBuilder &Builder) {
96  NumSlotsAllocatedForStatepoints++;
98 
99  unsigned SpillSize = ValueType.getSizeInBits() / 8;
100  assert((SpillSize * 8) == ValueType.getSizeInBits() && "Size not in bytes?");
101 
102  // First look for a previously created stack slot which is not in
103  // use (accounting for the fact arbitrary slots may already be
104  // reserved), or to create a new stack slot and use it.
105 
106  const size_t NumSlots = AllocatedStackSlots.size();
107  assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
108 
109  assert(AllocatedStackSlots.size() ==
110  Builder.FuncInfo.StatepointStackSlots.size() &&
111  "Broken invariant");
112 
113  for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
114  if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
115  const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
116  if (MFI.getObjectSize(FI) == SpillSize) {
117  AllocatedStackSlots.set(NextSlotToAllocate);
118  // TODO: Is ValueType the right thing to use here?
119  return Builder.DAG.getFrameIndex(FI, ValueType);
120  }
121  }
122  }
123 
124  // Couldn't find a free slot, so create a new one:
125 
126  SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
127  const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
129 
131  AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
132  assert(AllocatedStackSlots.size() ==
133  Builder.FuncInfo.StatepointStackSlots.size() &&
134  "Broken invariant");
135 
136  StatepointMaxSlotsRequired.updateMax(
138 
139  return SpillSlot;
140 }
141 
142 /// Utility function for reservePreviousStackSlotForValue. Tries to find
143 /// stack slot index to which we have spilled value for previous statepoints.
144 /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
146  SelectionDAGBuilder &Builder,
147  int LookUpDepth) {
148  // Can not look any further - give up now
149  if (LookUpDepth <= 0)
150  return None;
151 
152  // Spill location is known for gc relocates
153  if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
154  const auto &SpillMap =
155  Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
156 
157  auto It = SpillMap.find(Relocate->getDerivedPtr());
158  if (It == SpillMap.end())
159  return None;
160 
161  return It->second;
162  }
163 
164  // Look through bitcast instructions.
165  if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
166  return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
167 
168  // Look through phi nodes
169  // All incoming values should have same known stack slot, otherwise result
170  // is unknown.
171  if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
172  Optional<int> MergedResult = None;
173 
174  for (auto &IncomingValue : Phi->incoming_values()) {
175  Optional<int> SpillSlot =
176  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
177  if (!SpillSlot.hasValue())
178  return None;
179 
180  if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
181  return None;
182 
183  MergedResult = SpillSlot;
184  }
185  return MergedResult;
186  }
187 
188  // TODO: We can do better for PHI nodes. In cases like this:
189  // ptr = phi(relocated_pointer, not_relocated_pointer)
190  // statepoint(ptr)
191  // We will return that stack slot for ptr is unknown. And later we might
192  // assign different stack slots for ptr and relocated_pointer. This limits
193  // llvm's ability to remove redundant stores.
194  // Unfortunately it's hard to accomplish in current infrastructure.
195  // We use this function to eliminate spill store completely, while
196  // in example we still need to emit store, but instead of any location
197  // we need to use special "preferred" location.
198 
199  // TODO: handle simple updates. If a value is modified and the original
200  // value is no longer live, it would be nice to put the modified value in the
201  // same slot. This allows folding of the memory accesses for some
202  // instructions types (like an increment).
203  // statepoint (i)
204  // i1 = i+1
205  // statepoint (i1)
206  // However we need to be careful for cases like this:
207  // statepoint(i)
208  // i1 = i+1
209  // statepoint(i, i1)
210  // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
211  // put handling of simple modifications in this function like it's done
212  // for bitcasts we might end up reserving i's slot for 'i+1' because order in
213  // which we visit values is unspecified.
214 
215  // Don't know any information about this instruction
216  return None;
217 }
218 
219 /// Try to find existing copies of the incoming values in stack slots used for
220 /// statepoint spilling. If we can find a spill slot for the incoming value,
221 /// mark that slot as allocated, and reuse the same slot for this safepoint.
222 /// This helps to avoid series of loads and stores that only serve to reshuffle
223 /// values on the stack between calls.
224 static void reservePreviousStackSlotForValue(const Value *IncomingValue,
225  SelectionDAGBuilder &Builder) {
226  SDValue Incoming = Builder.getValue(IncomingValue);
227 
228  if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
229  // We won't need to spill this, so no need to check for previously
230  // allocated stack slots
231  return;
232  }
233 
234  SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
235  if (OldLocation.getNode())
236  // Duplicates in input
237  return;
238 
239  const int LookUpDepth = 6;
240  Optional<int> Index =
241  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
242  if (!Index.hasValue())
243  return;
244 
245  const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
246 
247  auto SlotIt = find(StatepointSlots, *Index);
248  assert(SlotIt != StatepointSlots.end() &&
249  "Value spilled to the unknown stack slot");
250 
251  // This is one of our dedicated lowering slots
252  const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
253  if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
254  // stack slot already assigned to someone else, can't use it!
255  // TODO: currently we reserve space for gc arguments after doing
256  // normal allocation for deopt arguments. We should reserve for
257  // _all_ deopt and gc arguments, then start allocating. This
258  // will prevent some moves being inserted when vm state changes,
259  // but gc state doesn't between two calls.
260  return;
261  }
262  // Reserve this stack slot
263  Builder.StatepointLowering.reserveStackSlot(Offset);
264 
265  // Cache this slot so we find it when going through the normal
266  // assignment loop.
267  SDValue Loc =
268  Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
269  Builder.StatepointLowering.setLocation(Incoming, Loc);
270 }
271 
272 /// Remove any duplicate (as SDValues) from the derived pointer pairs. This
273 /// is not required for correctness. It's purpose is to reduce the size of
274 /// StackMap section. It has no effect on the number of spill slots required
275 /// or the actual lowering.
276 static void
280  SelectionDAGBuilder &Builder,
283 
284  SmallVector<const Value *, 64> NewBases, NewPtrs;
286  for (size_t i = 0, e = Ptrs.size(); i < e; i++) {
287  SDValue SD = Builder.getValue(Ptrs[i]);
288  auto SeenIt = Seen.find(SD);
289 
290  if (SeenIt == Seen.end()) {
291  // Only add non-duplicates
292  NewBases.push_back(Bases[i]);
293  NewPtrs.push_back(Ptrs[i]);
294  NewRelocs.push_back(Relocs[i]);
295  Seen[SD] = Ptrs[i];
296  } else {
297  // Duplicate pointer found, note in SSM and move on:
298  SSM.DuplicateMap[Ptrs[i]] = SeenIt->second;
299  }
300  }
301  assert(Bases.size() >= NewBases.size());
302  assert(Ptrs.size() >= NewPtrs.size());
303  assert(Relocs.size() >= NewRelocs.size());
304  Bases = NewBases;
305  Ptrs = NewPtrs;
306  Relocs = NewRelocs;
307  assert(Ptrs.size() == Bases.size());
308  assert(Ptrs.size() == Relocs.size());
309 }
310 
311 /// Extract call from statepoint, lower it and return pointer to the
312 /// call node. Also update NodeMap so that getValue(statepoint) will
313 /// reference lowered call result
314 static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
316  SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
317  SDValue ReturnValue, CallEndVal;
318  std::tie(ReturnValue, CallEndVal) =
319  Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
320  SDNode *CallEnd = CallEndVal.getNode();
321 
322  // Get a call instruction from the call sequence chain. Tail calls are not
323  // allowed. The following code is essentially reverse engineering X86's
324  // LowerCallTo.
325  //
326  // We are expecting DAG to have the following form:
327  //
328  // ch = eh_label (only in case of invoke statepoint)
329  // ch, glue = callseq_start ch
330  // ch, glue = X86::Call ch, glue
331  // ch, glue = callseq_end ch, glue
332  // get_return_value ch, glue
333  //
334  // get_return_value can either be a sequence of CopyFromReg instructions
335  // to grab the return value from the return register(s), or it can be a LOAD
336  // to load a value returned by reference via a stack slot.
337 
338  bool HasDef = !SI.CLI.RetTy->isVoidTy();
339  if (HasDef) {
340  if (CallEnd->getOpcode() == ISD::LOAD)
341  CallEnd = CallEnd->getOperand(0).getNode();
342  else
343  while (CallEnd->getOpcode() == ISD::CopyFromReg)
344  CallEnd = CallEnd->getOperand(0).getNode();
345  }
346 
347  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
348  return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
349 }
350 
351 /// Spill a value incoming to the statepoint. It might be either part of
352 /// vmstate
353 /// or gcstate. In both cases unconditionally spill it on the stack unless it
354 /// is a null constant. Return pair with first element being frame index
355 /// containing saved value and second element with outgoing chain from the
356 /// emitted store
357 static std::pair<SDValue, SDValue>
359  SelectionDAGBuilder &Builder) {
360  SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
361 
362  // Emit new store if we didn't do it for this ptr before
363  if (!Loc.getNode()) {
364  Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
365  Builder);
366  int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
367  // We use TargetFrameIndex so that isel will not select it into LEA
368  Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
369 
370  // TODO: We can create TokenFactor node instead of
371  // chaining stores one after another, this may allow
372  // a bit more optimal scheduling for them
373 
374 #ifndef NDEBUG
375  // Right now we always allocate spill slots that are of the same
376  // size as the value we're about to spill (the size of spillee can
377  // vary since we spill vectors of pointers too). At some point we
378  // can consider allowing spills of smaller values to larger slots
379  // (i.e. change the '==' in the assert below to a '>=').
381  assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
382  "Bad spill: stack slot does not match!");
383 #endif
384 
385  Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
387  Builder.DAG.getMachineFunction(), Index));
388 
389  Builder.StatepointLowering.setLocation(Incoming, Loc);
390  }
391 
392  assert(Loc.getNode());
393  return std::make_pair(Loc, Chain);
394 }
395 
396 /// Lower a single value incoming to a statepoint node. This value can be
397 /// either a deopt value or a gc value, the handling is the same. We special
398 /// case constants and allocas, then fall back to spilling if required.
399 static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
401  SelectionDAGBuilder &Builder) {
402  SDValue Chain = Builder.getRoot();
403 
404  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
405  // If the original value was a constant, make sure it gets recorded as
406  // such in the stackmap. This is required so that the consumer can
407  // parse any internal format to the deopt state. It also handles null
408  // pointers and other constant pointers in GC states. Note the constant
409  // vectors do not appear to actually hit this path and that anything larger
410  // than an i64 value (not type!) will fail asserts here.
411  pushStackMapConstant(Ops, Builder, C->getSExtValue());
412  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
413  // This handles allocas as arguments to the statepoint (this is only
414  // really meaningful for a deopt value. For GC, we'd be trying to
415  // relocate the address of the alloca itself?)
416  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
417  "Incoming value is a frame index!");
418  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
419  Builder.getFrameIndexTy()));
420  } else if (LiveInOnly) {
421  // If this value is live in (not live-on-return, or live-through), we can
422  // treat it the same way patchpoint treats it's "live in" values. We'll
423  // end up folding some of these into stack references, but they'll be
424  // handled by the register allocator. Note that we do not have the notion
425  // of a late use so these values might be placed in registers which are
426  // clobbered by the call. This is fine for live-in.
427  Ops.push_back(Incoming);
428  } else {
429  // Otherwise, locate a spill slot and explicitly spill it so it
430  // can be found by the runtime later. We currently do not support
431  // tracking values through callee saved registers to their eventual
432  // spill location. This would be a useful optimization, but would
433  // need to be optional since it requires a lot of complexity on the
434  // runtime side which not all would support.
435  auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
436  Ops.push_back(Res.first);
437  Chain = Res.second;
438  }
439 
440  Builder.DAG.setRoot(Chain);
441 }
442 
443 /// Lower deopt state and gc pointer arguments of the statepoint. The actual
444 /// lowering is described in lowerIncomingStatepointValue. This function is
445 /// responsible for lowering everything in the right position and playing some
446 /// tricks to avoid redundant stack manipulation where possible. On
447 /// completion, 'Ops' will contain ready to use operands for machine code
448 /// statepoint. The chain nodes will have already been created and the DAG root
449 /// will be set to the last value spilled (if any were).
450 static void
453  SelectionDAGBuilder &Builder) {
454  // Lower the deopt and gc arguments for this statepoint. Layout will be:
455  // deopt argument length, deopt arguments.., gc arguments...
456 #ifndef NDEBUG
457  if (auto *GFI = Builder.GFI) {
458  // Check that each of the gc pointer and bases we've gotten out of the
459  // safepoint is something the strategy thinks might be a pointer (or vector
460  // of pointers) into the GC heap. This is basically just here to help catch
461  // errors during statepoint insertion. TODO: This should actually be in the
462  // Verifier, but we can't get to the GCStrategy from there (yet).
463  GCStrategy &S = GFI->getStrategy();
464  for (const Value *V : SI.Bases) {
465  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
466  if (Opt.hasValue()) {
467  assert(Opt.getValue() &&
468  "non gc managed base pointer found in statepoint");
469  }
470  }
471  for (const Value *V : SI.Ptrs) {
472  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
473  if (Opt.hasValue()) {
474  assert(Opt.getValue() &&
475  "non gc managed derived pointer found in statepoint");
476  }
477  }
478  assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
479  } else {
480  assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
481  assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
482  }
483 #endif
484 
485  // Figure out what lowering strategy we're going to use for each part
486  // Note: Is is conservatively correct to lower both "live-in" and "live-out"
487  // as "live-through". A "live-through" variable is one which is "live-in",
488  // "live-out", and live throughout the lifetime of the call (i.e. we can find
489  // it from any PC within the transitive callee of the statepoint). In
490  // particular, if the callee spills callee preserved registers we may not
491  // be able to find a value placed in that register during the call. This is
492  // fine for live-out, but not for live-through. If we were willing to make
493  // assumptions about the code generator producing the callee, we could
494  // potentially allow live-through values in callee saved registers.
495  const bool LiveInDeopt =
497 
498  auto isGCValue =[&](const Value *V) {
499  return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
500  };
501 
502  // Before we actually start lowering (and allocating spill slots for values),
503  // reserve any stack slots which we judge to be profitable to reuse for a
504  // particular value. This is purely an optimization over the code below and
505  // doesn't change semantics at all. It is important for performance that we
506  // reserve slots for both deopt and gc values before lowering either.
507  for (const Value *V : SI.DeoptState) {
508  if (!LiveInDeopt || isGCValue(V))
510  }
511  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
512  reservePreviousStackSlotForValue(SI.Bases[i], Builder);
513  reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
514  }
515 
516  // First, prefix the list with the number of unique values to be
517  // lowered. Note that this is the number of *Values* not the
518  // number of SDValues required to lower them.
519  const int NumVMSArgs = SI.DeoptState.size();
520  pushStackMapConstant(Ops, Builder, NumVMSArgs);
521 
522  // The vm state arguments are lowered in an opaque manner. We do not know
523  // what type of values are contained within.
524  for (const Value *V : SI.DeoptState) {
525  SDValue Incoming = Builder.getValue(V);
526  const bool LiveInValue = LiveInDeopt && !isGCValue(V);
527  lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder);
528  }
529 
530  // Finally, go ahead and lower all the gc arguments. There's no prefixed
531  // length for this one. After lowering, we'll have the base and pointer
532  // arrays interwoven with each (lowered) base pointer immediately followed by
533  // it's (lowered) derived pointer. i.e
534  // (base[0], ptr[0], base[1], ptr[1], ...)
535  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
536  const Value *Base = SI.Bases[i];
537  lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
538  Ops, Builder);
539 
540  const Value *Ptr = SI.Ptrs[i];
541  lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
542  Ops, Builder);
543  }
544 
545  // If there are any explicit spill slots passed to the statepoint, record
546  // them, but otherwise do not do anything special. These are user provided
547  // allocas and give control over placement to the consumer. In this case,
548  // it is the contents of the slot which may get updated, not the pointer to
549  // the alloca
550  for (Value *V : SI.GCArgs) {
551  SDValue Incoming = Builder.getValue(V);
552  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
553  // This handles allocas as arguments to the statepoint
554  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
555  "Incoming value is a frame index!");
556  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
557  Builder.getFrameIndexTy()));
558  }
559  }
560 
561  // Record computed locations for all lowered values.
562  // This can not be embedded in lowering loops as we need to record *all*
563  // values, while previous loops account only values with unique SDValues.
564  const Instruction *StatepointInstr = SI.StatepointInstr;
565  auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
566 
567  for (const GCRelocateInst *Relocate : SI.GCRelocates) {
568  const Value *V = Relocate->getDerivedPtr();
569  SDValue SDV = Builder.getValue(V);
570  SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
571 
572  if (Loc.getNode()) {
573  SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
574  } else {
575  // Record value as visited, but not spilled. This is case for allocas
576  // and constants. For this values we can avoid emitting spill load while
577  // visiting corresponding gc_relocate.
578  // Actually we do not need to record them in this map at all.
579  // We do this only to check that we are not relocating any unvisited
580  // value.
581  SpillMap.SlotMap[V] = None;
582 
583  // Default llvm mechanisms for exporting values which are used in
584  // different basic blocks does not work for gc relocates.
585  // Note that it would be incorrect to teach llvm that all relocates are
586  // uses of the corresponding values so that it would automatically
587  // export them. Relocates of the spilled values does not use original
588  // value.
589  if (Relocate->getParent() != StatepointInstr->getParent())
590  Builder.ExportFromCurrentBlock(V);
591  }
592  }
593 }
594 
597  // The basic scheme here is that information about both the original call and
598  // the safepoint is encoded in the CallInst. We create a temporary call and
599  // lower it, then reverse engineer the calling sequence.
600 
601  NumOfStatepoints++;
602  // Clear state
603  StatepointLowering.startNewStatepoint(*this);
604 
605 #ifndef NDEBUG
606  // We schedule gc relocates before removeDuplicateGCPtrs since we _will_
607  // encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
608  for (auto *Reloc : SI.GCRelocates)
609  if (Reloc->getParent() == SI.StatepointInstr->getParent())
610  StatepointLowering.scheduleRelocCall(*Reloc);
611 #endif
612 
613  // Remove any redundant llvm::Values which map to the same SDValue as another
614  // input. Also has the effect of removing duplicates in the original
615  // llvm::Value input list as well. This is a useful optimization for
616  // reducing the size of the StackMap section. It has no other impact.
617  removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
618  FuncInfo.StatepointSpillMaps[SI.StatepointInstr]);
619  assert(SI.Bases.size() == SI.Ptrs.size() &&
620  SI.Ptrs.size() == SI.GCRelocates.size());
621 
622  // Lower statepoint vmstate and gcstate arguments
623  SmallVector<SDValue, 10> LoweredMetaArgs;
624  lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this);
625 
626  // Now that we've emitted the spills, we need to update the root so that the
627  // call sequence is ordered correctly.
628  SI.CLI.setChain(getRoot());
629 
630  // Get call node, we will replace it later with statepoint
631  SDValue ReturnVal;
632  SDNode *CallNode;
633  std::tie(ReturnVal, CallNode) =
634  lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
635 
636  // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
637  // nodes with all the appropriate arguments and return values.
638 
639  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
640  SDValue Chain = CallNode->getOperand(0);
641 
642  SDValue Glue;
643  bool CallHasIncomingGlue = CallNode->getGluedNode();
644  if (CallHasIncomingGlue) {
645  // Glue is always last operand
646  Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
647  }
648 
649  // Build the GC_TRANSITION_START node if necessary.
650  //
651  // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
652  // order in which they appear in the call to the statepoint intrinsic. If
653  // any of the operands is a pointer-typed, that operand is immediately
654  // followed by a SRCVALUE for the pointer that may be used during lowering
655  // (e.g. to form MachinePointerInfo values for loads/stores).
656  const bool IsGCTransition =
659  if (IsGCTransition) {
661 
662  // Add chain
663  TSOps.push_back(Chain);
664 
665  // Add GC transition arguments
666  for (const Value *V : SI.GCTransitionArgs) {
667  TSOps.push_back(getValue(V));
668  if (V->getType()->isPointerTy())
669  TSOps.push_back(DAG.getSrcValue(V));
670  }
671 
672  // Add glue if necessary
673  if (CallHasIncomingGlue)
674  TSOps.push_back(Glue);
675 
676  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
677 
678  SDValue GCTransitionStart =
679  DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
680 
681  Chain = GCTransitionStart.getValue(0);
682  Glue = GCTransitionStart.getValue(1);
683  }
684 
685  // TODO: Currently, all of these operands are being marked as read/write in
686  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
687  // and flags to be read-only.
689 
690  // Add the <id> and <numBytes> constants.
691  Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
692  Ops.push_back(
693  DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
694 
695  // Calculate and push starting position of vmstate arguments
696  // Get number of arguments incoming directly into call node
697  unsigned NumCallRegArgs =
698  CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
699  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
700 
701  // Add call target
702  SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
703  Ops.push_back(CallTarget);
704 
705  // Add call arguments
706  // Get position of register mask in the call
707  SDNode::op_iterator RegMaskIt;
708  if (CallHasIncomingGlue)
709  RegMaskIt = CallNode->op_end() - 2;
710  else
711  RegMaskIt = CallNode->op_end() - 1;
712  Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
713 
714  // Add a constant argument for the calling convention
715  pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
716 
717  // Add a constant argument for the flags
718  uint64_t Flags = SI.StatepointFlags;
719  assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
720  "Unknown flag used");
721  pushStackMapConstant(Ops, *this, Flags);
722 
723  // Insert all vmstate and gcstate arguments
724  Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
725 
726  // Add register mask from call node
727  Ops.push_back(*RegMaskIt);
728 
729  // Add chain
730  Ops.push_back(Chain);
731 
732  // Same for the glue, but we add it only if original call had it
733  if (Glue.getNode())
734  Ops.push_back(Glue);
735 
736  // Compute return values. Provide a glue output since we consume one as
737  // input. This allows someone else to chain off us as needed.
738  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
739 
740  SDNode *StatepointMCNode =
741  DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
742 
743  SDNode *SinkNode = StatepointMCNode;
744 
745  // Build the GC_TRANSITION_END node if necessary.
746  //
747  // See the comment above regarding GC_TRANSITION_START for the layout of
748  // the operands to the GC_TRANSITION_END node.
749  if (IsGCTransition) {
751 
752  // Add chain
753  TEOps.push_back(SDValue(StatepointMCNode, 0));
754 
755  // Add GC transition arguments
756  for (const Value *V : SI.GCTransitionArgs) {
757  TEOps.push_back(getValue(V));
758  if (V->getType()->isPointerTy())
759  TEOps.push_back(DAG.getSrcValue(V));
760  }
761 
762  // Add glue
763  TEOps.push_back(SDValue(StatepointMCNode, 1));
764 
765  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
766 
767  SDValue GCTransitionStart =
768  DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
769 
770  SinkNode = GCTransitionStart.getNode();
771  }
772 
773  // Replace original call
774  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
775  // Remove original call node
776  DAG.DeleteNode(CallNode);
777 
778  // DON'T set the root - under the assumption that it's already set past the
779  // inserted node we created.
780 
781  // TODO: A better future implementation would be to emit a single variable
782  // argument, variable return value STATEPOINT node here and then hookup the
783  // return value of each gc.relocate to the respective output of the
784  // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
785  // to actually be possible today.
786 
787  return ReturnVal;
788 }
789 
790 void
792  const BasicBlock *EHPadBB /*= nullptr*/) {
794  "anyregcc is not supported on statepoints!");
795 
796 #ifndef NDEBUG
797  // If this is a malformed statepoint, report it early to simplify debugging.
798  // This should catch any IR level mistake that's made when constructing or
799  // transforming statepoints.
800  ISP.verify();
801 
802  // Check that the associated GCStrategy expects to encounter statepoints.
803  assert(GFI->getStrategy().useStatepoints() &&
804  "GCStrategy does not expect to encounter statepoints");
805 #endif
806 
807  SDValue ActualCallee;
808 
809  if (ISP.getNumPatchBytes() > 0) {
810  // If we've been asked to emit a nop sequence instead of a call instruction
811  // for this statepoint then don't lower the call target, but use a constant
812  // `null` instead. Not lowering the call target lets statepoint clients get
813  // away without providing a physical address for the symbolic call target at
814  // link time.
815 
816  const auto &TLI = DAG.getTargetLoweringInfo();
817  const auto &DL = DAG.getDataLayout();
818 
819  unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
820  ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
821  } else {
822  ActualCallee = getValue(ISP.getCalledValue());
823  }
824 
826  populateCallLoweringInfo(SI.CLI, ISP.getCallSite(),
828  ISP.getNumCallArgs(), ActualCallee,
829  ISP.getActualReturnType(), false /* IsPatchPoint */);
830 
831  for (const GCRelocateInst *Relocate : ISP.getRelocates()) {
832  SI.GCRelocates.push_back(Relocate);
833  SI.Bases.push_back(Relocate->getBasePtr());
834  SI.Ptrs.push_back(Relocate->getDerivedPtr());
835  }
836 
837  SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
838  SI.StatepointInstr = ISP.getInstruction();
839  SI.GCTransitionArgs =
841  SI.ID = ISP.getID();
842  SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end());
843  SI.StatepointFlags = ISP.getFlags();
844  SI.NumPatchBytes = ISP.getNumPatchBytes();
845  SI.EHPadBB = EHPadBB;
846 
847  SDValue ReturnValue = LowerAsSTATEPOINT(SI);
848 
849  // Export the result value if needed
850  const GCResultInst *GCResult = ISP.getGCResult();
851  Type *RetTy = ISP.getActualReturnType();
852  if (!RetTy->isVoidTy() && GCResult) {
853  if (GCResult->getParent() != ISP.getCallSite().getParent()) {
854  // Result value will be used in a different basic block so we need to
855  // export it now. Default exporting mechanism will not work here because
856  // statepoint call has a different type than the actual call. It means
857  // that by default llvm will create export register of the wrong type
858  // (always i32 in our case). So instead we need to create export register
859  // with correct type manually.
860  // TODO: To eliminate this problem we can remove gc.result intrinsics
861  // completely and make statepoint call to return a tuple.
862  unsigned Reg = FuncInfo.CreateRegs(RetTy);
863  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
864  DAG.getDataLayout(), Reg, RetTy, true);
865  SDValue Chain = DAG.getEntryNode();
866 
867  RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
868  PendingExports.push_back(Chain);
869  FuncInfo.ValueMap[ISP.getInstruction()] = Reg;
870  } else {
871  // Result value will be used in a same basic block. Don't export it or
872  // perform any explicit register copies.
873  // We'll replace the actuall call node shortly. gc_result will grab
874  // this value.
875  setValue(ISP.getInstruction(), ReturnValue);
876  }
877  } else {
878  // The token value is never used from here on, just generate a poison value
879  setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
880  }
881 }
882 
884  ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB,
885  bool VarArgDisallowed, bool ForceVoidReturnTy) {
887  unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin();
888  populateCallLoweringInfo(
889  SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee,
890  ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(),
891  false);
892  if (!VarArgDisallowed)
893  SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg();
894 
895  auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt);
896 
898 
900  SI.ID = SD.StatepointID.getValueOr(DefaultID);
901  SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
902 
903  SI.DeoptState =
904  ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
905  SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
906  SI.EHPadBB = EHPadBB;
907 
908  // NB! The GC arguments are deliberately left empty.
909 
910  if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
911  const Instruction *Inst = CS.getInstruction();
912  ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal);
913  setValue(Inst, ReturnVal);
914  }
915 }
916 
918  ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) {
919  LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB,
920  /* VarArgDisallowed = */ false,
921  /* ForceVoidReturnTy = */ false);
922 }
923 
924 void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
925  // The result value of the gc_result is simply the result of the actual
926  // call. We've already emitted this, so just grab the value.
927  const Instruction *I = CI.getStatepoint();
928 
929  if (I->getParent() != CI.getParent()) {
930  // Statepoint is in different basic block so we should have stored call
931  // result in a virtual register.
932  // We can not use default getValue() functionality to copy value from this
933  // register because statepoint and actual call return types can be
934  // different, and getValue() will use CopyFromReg of the wrong type,
935  // which is always i32 in our case.
936  PointerType *CalleeType = cast<PointerType>(
938  Type *RetTy =
939  cast<FunctionType>(CalleeType->getElementType())->getReturnType();
940  SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
941 
942  assert(CopyFromReg.getNode());
943  setValue(&CI, CopyFromReg);
944  } else {
945  setValue(&CI, getValue(I));
946  }
947 }
948 
949 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
950 #ifndef NDEBUG
951  // Consistency check
952  // We skip this check for relocates not in the same basic block as their
953  // statepoint. It would be too expensive to preserve validation info through
954  // different basic blocks.
955  if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
956  StatepointLowering.relocCallVisited(Relocate);
957 
958  auto *Ty = Relocate.getType()->getScalarType();
959  if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
960  assert(*IsManaged && "Non gc managed pointer relocated!");
961 #endif
962 
963  const Value *DerivedPtr = Relocate.getDerivedPtr();
964  SDValue SD = getValue(DerivedPtr);
965 
966  auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
967  auto SlotIt = SpillMap.find(DerivedPtr);
968  assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
969  Optional<int> DerivedPtrLocation = SlotIt->second;
970 
971  // We didn't need to spill these special cases (constants and allocas).
972  // See the handling in spillIncomingValueForStatepoint for detail.
973  if (!DerivedPtrLocation) {
974  setValue(&Relocate, SD);
975  return;
976  }
977 
978  SDValue SpillSlot =
979  DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
980 
981  // Be conservative: flush all pending loads
982  // TODO: Probably we can be less restrictive on this,
983  // it may allow more scheduling opportunities.
984  SDValue Chain = getRoot();
985 
986  SDValue SpillLoad =
987  DAG.getLoad(DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
988  Relocate.getType()),
989  getCurSDLoc(), Chain, SpillSlot,
990  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
991  *DerivedPtrLocation));
992 
993  // Again, be conservative, don't emit pending loads
994  DAG.setRoot(SpillLoad.getValue(1));
995 
996  assert(SpillLoad.getNode());
997  setValue(&Relocate, SpillLoad);
998 }
999 
1001  const auto &TLI = DAG.getTargetLoweringInfo();
1002  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1003  TLI.getPointerTy(DAG.getDataLayout()));
1004 
1005  // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1006  // call. We also do not lower the return value to any virtual register, and
1007  // change the immediately following return to a trap instruction.
1008  LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1009  /* VarArgDisallowed = */ true,
1010  /* ForceVoidReturnTy = */ true);
1011 }
1012 
1014  // We do not lower the return value from llvm.deoptimize to any virtual
1015  // register, and change the immediately following return to a trap
1016  // instruction.
1017  if (DAG.getTarget().Options.TrapUnreachable)
1018  DAG.setRoot(
1019  DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
1020 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
uint64_t CallInst * C
Represents calls to the gc.result intrinsic.
Definition: Statepoint.h:409
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
CallingConv::ID getCallingConv() const
Get the calling convention of the call.
Definition: CallSite.h:312
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn&#39;t known to be exported from the current basic block...
ValueTy * getCalledValue() const
Return the value actually being called or invoked.
Definition: Statepoint.h:141
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:136
void LowerDeoptimizeCall(const CallInst *CI)
TargetLowering::CallLoweringInfo CLI
Information regarding the underlying call instruction.
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:69
SmallVector< const GCRelocateInst *, 16 > GCRelocates
The set of gc.relocate calls associated with this gc.statepoint.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it&#39;s base class for read only access to a gc.statepoint.
Definition: Statepoint.h:305
void LowerStatepoint(ImmutableStatepoint Statepoint, const BasicBlock *EHPadBB=nullptr)
This class represents a function call, abstracting a target machine&#39;s calling convention.
uint64_t getID() const
Return the ID associated with this statepoint.
Definition: Statepoint.h:126
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
STATISTIC(NumFunctions, "Total number of functions")
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:503
CallSiteTy getCallSite() const
Return the underlying CallSite.
Definition: Statepoint.h:115
SDNode * getNode() const
get the SDNode which holds the desired result
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:452
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
op_iterator op_begin()
Definition: User.h:214
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling...
Keep track of frame indices allocated for statepoints as they could be used across basic block bounda...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool test(unsigned Idx) const
unsigned ID
The ID that the resulting STATEPOINT instruction has to report.
SDValue getRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
CallSiteTy::arg_iterator gc_args_begin() const
Definition: Statepoint.h:250
uint64_t StatepointFlags
Flags associated with the meta arguments being lowered.
CallSiteTy::arg_iterator deopt_begin() const
Definition: Statepoint.h:234
Value * getDerivedPtr() const
Definition: Statepoint.h:402
ArrayRef< const Use > GCTransitionArgs
The list of gc transition arguments present in the gc.statepoint being lowered.
CallLoweringInfo & setChain(SDValue InChain)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
op_iterator op_end() const
Reg
All possible values of the reg field in the ModR/M byte.
InstrTy * getInstruction() const
Definition: CallSite.h:92
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static void removeDuplicateGCPtrs(SmallVectorImpl< const Value *> &Bases, SmallVectorImpl< const Value *> &Ptrs, SmallVectorImpl< const GCRelocateInst *> &Relocs, SelectionDAGBuilder &Builder, FunctionLoweringInfo::StatepointSpillMap &SSM)
Remove any duplicate (as SDValues) from the derived pointer pairs.
InstructionTy * getInstruction() const
Definition: Statepoint.h:145
void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB)
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes associated with this statepoint.
Definition: Statepoint.h:132
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:385
CallSiteTy::arg_iterator gc_args_end() const
Definition: Statepoint.h:253
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:606
virtual Optional< bool > isGCManagedPointer(const Type *Ty) const
If the type specified can be reliably distinguished, returns true for pointers to GC managed location...
Definition: GCStrategy.h:127
bool isVarArg() const
Definition: DerivedTypes.h:123
This class represents a no-op cast from one type to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:558
amdgpu Simplify well known AMD library false Value * Callee
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Class to represent pointers.
Definition: DerivedTypes.h:467
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:461
int getNumCallArgs() const
Number of arguments to be passed to the actual callee.
Definition: Statepoint.h:173
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return &#39;this&#39;.
Definition: Type.h:301
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Definition: CallSite.h:555
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:141
const Instruction * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Definition: Statepoint.h:350
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
const Instruction * StatepointInstr
The gc.statepoint instruction.
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:809
ArrayRef< const Use > DeoptState
The deoptimization state associated with this gc.statepoint call, if any.
const GCResultInst * getGCResult() const
Get the experimental_gc_result call tied to this statepoint.
Definition: Statepoint.h:275
LLVM Basic Block Representation.
Definition: BasicBlock.h:59
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
static std::pair< SDValue, SDValue > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
const SDValue & getOperand(unsigned Num) const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:221
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:161
void clear()
Clear the memory usage of this object.
void verify()
Asserts if this statepoint is malformed.
Definition: Statepoint.h:286
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:734
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
DenseMap< const Instruction *, StatepointSpillMap > StatepointSpillMaps
Maps gc.statepoint instructions to their corresponding StatepointSpillMap instances.
Extended Value Type.
Definition: ValueTypes.h:34
const AMDGPUAS & AS
unsigned getNumArgOperands() const
Definition: CallSite.h:293
unsigned getNumOperands() const
Return the number of values used by this operation.
void setLocation(SDValue Val, SDValue Location)
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
SmallBitVector & set()
void clear()
Clear all bits.
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:788
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SLI)
Lower SLI into a STATEPOINT instruction.
static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
IterTy arg_begin() const
Definition: CallSite.h:571
RegsForValue - This struct represents the registers (physical or virtual) that a particular set of va...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:864
DenseMap< const Value *, const Value * > DuplicateMap
Maps llvm IR values to the values they were de-duplicated to.
CallSiteTy::arg_iterator deopt_end() const
Definition: Statepoint.h:239
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
size_t size() const
Returns the number of bits in this bitvector.
Represents one node in the SelectionDAG.
BBTy * getParent() const
Get the basic block containing the call site.
Definition: CallSite.h:97
GCFunctionInfo * GFI
GFI - Garbage collection metadata for the function.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:482
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
bool hasValue() const
Definition: Optional.h:137
Represents a use of a SDNode.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:120
GCStrategy describes a garbage collector algorithm&#39;s code generation requirements, and provides overridable hooks for those needs which cannot be abstractly described.
Definition: GCStrategy.h:80
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:581
Establish a view to a call site for examination.
Definition: CallSite.h:713
#define I(x, y, z)
Definition: MD5.cpp:58
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
SmallVector< const Value *, 16 > Bases
Bases[i] is the base pointer for Ptrs[i].
SDValue getValue(unsigned R) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:175
std::vector< const GCRelocateInst * > getRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition: Statepoint.h:423
Type * getType() const
Return the type of the instruction that generated this call site.
Definition: CallSite.h:264
Type * getActualReturnType() const
Return the type of the value returned by the call underlying the statepoint.
Definition: Statepoint.h:166
unsigned NumPatchBytes
The number of patchable bytes the call needs to get lowered into.
void LowerCallSiteWithDeoptBundleImpl(ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
FunctionLoweringInfo & FuncInfo
FuncInfo - Information about the function as a whole.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const BasicBlock * EHPadBB
The exception handling unwind destination, in case this represents an invoke of gc.statepoint.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Represents calls to the gc.relocate intrinsic.
Definition: Statepoint.h:374
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
Definition: Value.h:73
FunctionType * getFunctionType() const
Definition: CallSite.h:320
ArrayRef< const Use > GCArgs
The full list of gc arguments to the gc.statepoint being lowered.
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
AttributeList getAttributes() const
Get the parameter attributes of the call.
Definition: CallSite.h:329
uint64_t getFlags() const
Definition: Statepoint.h:120
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
A bitmask that includes all valid flags.
Type * getElementType() const
Definition: DerivedTypes.h:486
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
A discriminated union of two pointer types, with the discriminator in the low bit of the pointer...
Definition: PointerUnion.h:87
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:821