LLVM  8.0.0svn
StatepointLowering.cpp
Go to the documentation of this file.
1 //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file includes support code use by SelectionDAGBuilder when lowering a
11 // statepoint sequence in SelectionDAG IR.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "StatepointLowering.h"
16 #include "SelectionDAGBuilder.h"
17 #include "llvm/ADT/ArrayRef.h"
18 #include "llvm/ADT/DenseMap.h"
19 #include "llvm/ADT/None.h"
20 #include "llvm/ADT/Optional.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/Statistic.h"
34 #include "llvm/CodeGen/StackMaps.h"
37 #include "llvm/IR/CallingConv.h"
38 #include "llvm/IR/DerivedTypes.h"
39 #include "llvm/IR/Instruction.h"
40 #include "llvm/IR/Instructions.h"
41 #include "llvm/IR/LLVMContext.h"
42 #include "llvm/IR/Statepoint.h"
43 #include "llvm/IR/Type.h"
44 #include "llvm/Support/Casting.h"
48 #include <cassert>
49 #include <cstddef>
50 #include <cstdint>
51 #include <iterator>
52 #include <tuple>
53 #include <utility>
54 
55 using namespace llvm;
56 
57 #define DEBUG_TYPE "statepoint-lowering"
58 
59 STATISTIC(NumSlotsAllocatedForStatepoints,
60  "Number of stack slots allocated for statepoints");
61 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
62 STATISTIC(StatepointMaxSlotsRequired,
63  "Maximum number of stack slots required for a singe statepoint");
64 
66  SelectionDAGBuilder &Builder, uint64_t Value) {
67  SDLoc L = Builder.getCurSDLoc();
68  Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
69  MVT::i64));
70  Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
71 }
72 
74  // Consistency check
75  assert(PendingGCRelocateCalls.empty() &&
76  "Trying to visit statepoint before finished processing previous one");
77  Locations.clear();
78  NextSlotToAllocate = 0;
79  // Need to resize this on each safepoint - we need the two to stay in sync and
80  // the clear patterns of a SelectionDAGBuilder have no relation to
81  // FunctionLoweringInfo. Also need to ensure used bits get cleared.
82  AllocatedStackSlots.clear();
83  AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
84 }
85 
87  Locations.clear();
88  AllocatedStackSlots.clear();
89  assert(PendingGCRelocateCalls.empty() &&
90  "cleared before statepoint sequence completed");
91 }
92 
93 SDValue
95  SelectionDAGBuilder &Builder) {
96  NumSlotsAllocatedForStatepoints++;
98 
99  unsigned SpillSize = ValueType.getStoreSize();
100  assert((SpillSize * 8) == ValueType.getSizeInBits() && "Size not in bytes?");
101 
102  // First look for a previously created stack slot which is not in
103  // use (accounting for the fact arbitrary slots may already be
104  // reserved), or to create a new stack slot and use it.
105 
106  const size_t NumSlots = AllocatedStackSlots.size();
107  assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
108 
109  assert(AllocatedStackSlots.size() ==
110  Builder.FuncInfo.StatepointStackSlots.size() &&
111  "Broken invariant");
112 
113  for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
114  if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
115  const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
116  if (MFI.getObjectSize(FI) == SpillSize) {
117  AllocatedStackSlots.set(NextSlotToAllocate);
118  // TODO: Is ValueType the right thing to use here?
119  return Builder.DAG.getFrameIndex(FI, ValueType);
120  }
121  }
122  }
123 
124  // Couldn't find a free slot, so create a new one:
125 
126  SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
127  const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
129 
131  AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
132  assert(AllocatedStackSlots.size() ==
133  Builder.FuncInfo.StatepointStackSlots.size() &&
134  "Broken invariant");
135 
136  StatepointMaxSlotsRequired.updateMax(
138 
139  return SpillSlot;
140 }
141 
142 /// Utility function for reservePreviousStackSlotForValue. Tries to find
143 /// stack slot index to which we have spilled value for previous statepoints.
144 /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
146  SelectionDAGBuilder &Builder,
147  int LookUpDepth) {
148  // Can not look any further - give up now
149  if (LookUpDepth <= 0)
150  return None;
151 
152  // Spill location is known for gc relocates
153  if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
154  const auto &SpillMap =
155  Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
156 
157  auto It = SpillMap.find(Relocate->getDerivedPtr());
158  if (It == SpillMap.end())
159  return None;
160 
161  return It->second;
162  }
163 
164  // Look through bitcast instructions.
165  if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
166  return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
167 
168  // Look through phi nodes
169  // All incoming values should have same known stack slot, otherwise result
170  // is unknown.
171  if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
172  Optional<int> MergedResult = None;
173 
174  for (auto &IncomingValue : Phi->incoming_values()) {
175  Optional<int> SpillSlot =
176  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
177  if (!SpillSlot.hasValue())
178  return None;
179 
180  if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
181  return None;
182 
183  MergedResult = SpillSlot;
184  }
185  return MergedResult;
186  }
187 
188  // TODO: We can do better for PHI nodes. In cases like this:
189  // ptr = phi(relocated_pointer, not_relocated_pointer)
190  // statepoint(ptr)
191  // We will return that stack slot for ptr is unknown. And later we might
192  // assign different stack slots for ptr and relocated_pointer. This limits
193  // llvm's ability to remove redundant stores.
194  // Unfortunately it's hard to accomplish in current infrastructure.
195  // We use this function to eliminate spill store completely, while
196  // in example we still need to emit store, but instead of any location
197  // we need to use special "preferred" location.
198 
199  // TODO: handle simple updates. If a value is modified and the original
200  // value is no longer live, it would be nice to put the modified value in the
201  // same slot. This allows folding of the memory accesses for some
202  // instructions types (like an increment).
203  // statepoint (i)
204  // i1 = i+1
205  // statepoint (i1)
206  // However we need to be careful for cases like this:
207  // statepoint(i)
208  // i1 = i+1
209  // statepoint(i, i1)
210  // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
211  // put handling of simple modifications in this function like it's done
212  // for bitcasts we might end up reserving i's slot for 'i+1' because order in
213  // which we visit values is unspecified.
214 
215  // Don't know any information about this instruction
216  return None;
217 }
218 
219 /// Try to find existing copies of the incoming values in stack slots used for
220 /// statepoint spilling. If we can find a spill slot for the incoming value,
221 /// mark that slot as allocated, and reuse the same slot for this safepoint.
222 /// This helps to avoid series of loads and stores that only serve to reshuffle
223 /// values on the stack between calls.
224 static void reservePreviousStackSlotForValue(const Value *IncomingValue,
225  SelectionDAGBuilder &Builder) {
226  SDValue Incoming = Builder.getValue(IncomingValue);
227 
228  if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
229  // We won't need to spill this, so no need to check for previously
230  // allocated stack slots
231  return;
232  }
233 
234  SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
235  if (OldLocation.getNode())
236  // Duplicates in input
237  return;
238 
239  const int LookUpDepth = 6;
241  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
242  if (!Index.hasValue())
243  return;
244 
245  const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
246 
247  auto SlotIt = find(StatepointSlots, *Index);
248  assert(SlotIt != StatepointSlots.end() &&
249  "Value spilled to the unknown stack slot");
250 
251  // This is one of our dedicated lowering slots
252  const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
253  if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
254  // stack slot already assigned to someone else, can't use it!
255  // TODO: currently we reserve space for gc arguments after doing
256  // normal allocation for deopt arguments. We should reserve for
257  // _all_ deopt and gc arguments, then start allocating. This
258  // will prevent some moves being inserted when vm state changes,
259  // but gc state doesn't between two calls.
260  return;
261  }
262  // Reserve this stack slot
263  Builder.StatepointLowering.reserveStackSlot(Offset);
264 
265  // Cache this slot so we find it when going through the normal
266  // assignment loop.
267  SDValue Loc =
268  Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
269  Builder.StatepointLowering.setLocation(Incoming, Loc);
270 }
271 
272 /// Remove any duplicate (as SDValues) from the derived pointer pairs. This
273 /// is not required for correctness. It's purpose is to reduce the size of
274 /// StackMap section. It has no effect on the number of spill slots required
275 /// or the actual lowering.
276 static void
280  SelectionDAGBuilder &Builder,
283 
284  SmallVector<const Value *, 64> NewBases, NewPtrs;
286  for (size_t i = 0, e = Ptrs.size(); i < e; i++) {
287  SDValue SD = Builder.getValue(Ptrs[i]);
288  auto SeenIt = Seen.find(SD);
289 
290  if (SeenIt == Seen.end()) {
291  // Only add non-duplicates
292  NewBases.push_back(Bases[i]);
293  NewPtrs.push_back(Ptrs[i]);
294  NewRelocs.push_back(Relocs[i]);
295  Seen[SD] = Ptrs[i];
296  } else {
297  // Duplicate pointer found, note in SSM and move on:
298  SSM.DuplicateMap[Ptrs[i]] = SeenIt->second;
299  }
300  }
301  assert(Bases.size() >= NewBases.size());
302  assert(Ptrs.size() >= NewPtrs.size());
303  assert(Relocs.size() >= NewRelocs.size());
304  Bases = NewBases;
305  Ptrs = NewPtrs;
306  Relocs = NewRelocs;
307  assert(Ptrs.size() == Bases.size());
308  assert(Ptrs.size() == Relocs.size());
309 }
310 
311 /// Extract call from statepoint, lower it and return pointer to the
312 /// call node. Also update NodeMap so that getValue(statepoint) will
313 /// reference lowered call result
314 static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
316  SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
317  SDValue ReturnValue, CallEndVal;
318  std::tie(ReturnValue, CallEndVal) =
319  Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
320  SDNode *CallEnd = CallEndVal.getNode();
321 
322  // Get a call instruction from the call sequence chain. Tail calls are not
323  // allowed. The following code is essentially reverse engineering X86's
324  // LowerCallTo.
325  //
326  // We are expecting DAG to have the following form:
327  //
328  // ch = eh_label (only in case of invoke statepoint)
329  // ch, glue = callseq_start ch
330  // ch, glue = X86::Call ch, glue
331  // ch, glue = callseq_end ch, glue
332  // get_return_value ch, glue
333  //
334  // get_return_value can either be a sequence of CopyFromReg instructions
335  // to grab the return value from the return register(s), or it can be a LOAD
336  // to load a value returned by reference via a stack slot.
337 
338  bool HasDef = !SI.CLI.RetTy->isVoidTy();
339  if (HasDef) {
340  if (CallEnd->getOpcode() == ISD::LOAD)
341  CallEnd = CallEnd->getOperand(0).getNode();
342  else
343  while (CallEnd->getOpcode() == ISD::CopyFromReg)
344  CallEnd = CallEnd->getOperand(0).getNode();
345  }
346 
347  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
348  return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
349 }
350 
351 /// Spill a value incoming to the statepoint. It might be either part of
352 /// vmstate
353 /// or gcstate. In both cases unconditionally spill it on the stack unless it
354 /// is a null constant. Return pair with first element being frame index
355 /// containing saved value and second element with outgoing chain from the
356 /// emitted store
357 static std::pair<SDValue, SDValue>
359  SelectionDAGBuilder &Builder) {
360  SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
361 
362  // Emit new store if we didn't do it for this ptr before
363  if (!Loc.getNode()) {
364  Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
365  Builder);
366  int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
367  // We use TargetFrameIndex so that isel will not select it into LEA
368  Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
369 
370  // TODO: We can create TokenFactor node instead of
371  // chaining stores one after another, this may allow
372  // a bit more optimal scheduling for them
373 
374 #ifndef NDEBUG
375  // Right now we always allocate spill slots that are of the same
376  // size as the value we're about to spill (the size of spillee can
377  // vary since we spill vectors of pointers too). At some point we
378  // can consider allowing spills of smaller values to larger slots
379  // (i.e. change the '==' in the assert below to a '>=').
381  assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
382  "Bad spill: stack slot does not match!");
383 #endif
384 
385  Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
387  Builder.DAG.getMachineFunction(), Index));
388 
389  Builder.StatepointLowering.setLocation(Incoming, Loc);
390  }
391 
392  assert(Loc.getNode());
393  return std::make_pair(Loc, Chain);
394 }
395 
396 /// Lower a single value incoming to a statepoint node. This value can be
397 /// either a deopt value or a gc value, the handling is the same. We special
398 /// case constants and allocas, then fall back to spilling if required.
399 static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
401  SelectionDAGBuilder &Builder) {
402  SDValue Chain = Builder.getRoot();
403 
404  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
405  // If the original value was a constant, make sure it gets recorded as
406  // such in the stackmap. This is required so that the consumer can
407  // parse any internal format to the deopt state. It also handles null
408  // pointers and other constant pointers in GC states. Note the constant
409  // vectors do not appear to actually hit this path and that anything larger
410  // than an i64 value (not type!) will fail asserts here.
411  pushStackMapConstant(Ops, Builder, C->getSExtValue());
412  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
413  // This handles allocas as arguments to the statepoint (this is only
414  // really meaningful for a deopt value. For GC, we'd be trying to
415  // relocate the address of the alloca itself?)
416  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
417  "Incoming value is a frame index!");
418  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
419  Builder.getFrameIndexTy()));
420  } else if (LiveInOnly) {
421  // If this value is live in (not live-on-return, or live-through), we can
422  // treat it the same way patchpoint treats it's "live in" values. We'll
423  // end up folding some of these into stack references, but they'll be
424  // handled by the register allocator. Note that we do not have the notion
425  // of a late use so these values might be placed in registers which are
426  // clobbered by the call. This is fine for live-in.
427  Ops.push_back(Incoming);
428  } else {
429  // Otherwise, locate a spill slot and explicitly spill it so it
430  // can be found by the runtime later. We currently do not support
431  // tracking values through callee saved registers to their eventual
432  // spill location. This would be a useful optimization, but would
433  // need to be optional since it requires a lot of complexity on the
434  // runtime side which not all would support.
435  auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
436  Ops.push_back(Res.first);
437  Chain = Res.second;
438  }
439 
440  Builder.DAG.setRoot(Chain);
441 }
442 
443 /// Lower deopt state and gc pointer arguments of the statepoint. The actual
444 /// lowering is described in lowerIncomingStatepointValue. This function is
445 /// responsible for lowering everything in the right position and playing some
446 /// tricks to avoid redundant stack manipulation where possible. On
447 /// completion, 'Ops' will contain ready to use operands for machine code
448 /// statepoint. The chain nodes will have already been created and the DAG root
449 /// will be set to the last value spilled (if any were).
450 static void
453  SelectionDAGBuilder &Builder) {
454  // Lower the deopt and gc arguments for this statepoint. Layout will be:
455  // deopt argument length, deopt arguments.., gc arguments...
456 #ifndef NDEBUG
457  if (auto *GFI = Builder.GFI) {
458  // Check that each of the gc pointer and bases we've gotten out of the
459  // safepoint is something the strategy thinks might be a pointer (or vector
460  // of pointers) into the GC heap. This is basically just here to help catch
461  // errors during statepoint insertion. TODO: This should actually be in the
462  // Verifier, but we can't get to the GCStrategy from there (yet).
463  GCStrategy &S = GFI->getStrategy();
464  for (const Value *V : SI.Bases) {
465  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
466  if (Opt.hasValue()) {
467  assert(Opt.getValue() &&
468  "non gc managed base pointer found in statepoint");
469  }
470  }
471  for (const Value *V : SI.Ptrs) {
472  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
473  if (Opt.hasValue()) {
474  assert(Opt.getValue() &&
475  "non gc managed derived pointer found in statepoint");
476  }
477  }
478  assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
479  } else {
480  assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
481  assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
482  }
483 #endif
484 
485  // Figure out what lowering strategy we're going to use for each part
486  // Note: Is is conservatively correct to lower both "live-in" and "live-out"
487  // as "live-through". A "live-through" variable is one which is "live-in",
488  // "live-out", and live throughout the lifetime of the call (i.e. we can find
489  // it from any PC within the transitive callee of the statepoint). In
490  // particular, if the callee spills callee preserved registers we may not
491  // be able to find a value placed in that register during the call. This is
492  // fine for live-out, but not for live-through. If we were willing to make
493  // assumptions about the code generator producing the callee, we could
494  // potentially allow live-through values in callee saved registers.
495  const bool LiveInDeopt =
497 
498  auto isGCValue =[&](const Value *V) {
499  return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
500  };
501 
502  // Before we actually start lowering (and allocating spill slots for values),
503  // reserve any stack slots which we judge to be profitable to reuse for a
504  // particular value. This is purely an optimization over the code below and
505  // doesn't change semantics at all. It is important for performance that we
506  // reserve slots for both deopt and gc values before lowering either.
507  for (const Value *V : SI.DeoptState) {
508  if (!LiveInDeopt || isGCValue(V))
510  }
511  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
512  reservePreviousStackSlotForValue(SI.Bases[i], Builder);
513  reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
514  }
515 
516  // First, prefix the list with the number of unique values to be
517  // lowered. Note that this is the number of *Values* not the
518  // number of SDValues required to lower them.
519  const int NumVMSArgs = SI.DeoptState.size();
520  pushStackMapConstant(Ops, Builder, NumVMSArgs);
521 
522  // The vm state arguments are lowered in an opaque manner. We do not know
523  // what type of values are contained within.
524  for (const Value *V : SI.DeoptState) {
525  SDValue Incoming;
526  // If this is a function argument at a static frame index, generate it as
527  // the frame index.
528  if (const Argument *Arg = dyn_cast<Argument>(V)) {
529  int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);
530  if (FI != INT_MAX)
531  Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());
532  }
533  if (!Incoming.getNode())
534  Incoming = Builder.getValue(V);
535  const bool LiveInValue = LiveInDeopt && !isGCValue(V);
536  lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder);
537  }
538 
539  // Finally, go ahead and lower all the gc arguments. There's no prefixed
540  // length for this one. After lowering, we'll have the base and pointer
541  // arrays interwoven with each (lowered) base pointer immediately followed by
542  // it's (lowered) derived pointer. i.e
543  // (base[0], ptr[0], base[1], ptr[1], ...)
544  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
545  const Value *Base = SI.Bases[i];
546  lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
547  Ops, Builder);
548 
549  const Value *Ptr = SI.Ptrs[i];
550  lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
551  Ops, Builder);
552  }
553 
554  // If there are any explicit spill slots passed to the statepoint, record
555  // them, but otherwise do not do anything special. These are user provided
556  // allocas and give control over placement to the consumer. In this case,
557  // it is the contents of the slot which may get updated, not the pointer to
558  // the alloca
559  for (Value *V : SI.GCArgs) {
560  SDValue Incoming = Builder.getValue(V);
561  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
562  // This handles allocas as arguments to the statepoint
563  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
564  "Incoming value is a frame index!");
565  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
566  Builder.getFrameIndexTy()));
567  }
568  }
569 
570  // Record computed locations for all lowered values.
571  // This can not be embedded in lowering loops as we need to record *all*
572  // values, while previous loops account only values with unique SDValues.
573  const Instruction *StatepointInstr = SI.StatepointInstr;
574  auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
575 
576  for (const GCRelocateInst *Relocate : SI.GCRelocates) {
577  const Value *V = Relocate->getDerivedPtr();
578  SDValue SDV = Builder.getValue(V);
579  SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
580 
581  if (Loc.getNode()) {
582  SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
583  } else {
584  // Record value as visited, but not spilled. This is case for allocas
585  // and constants. For this values we can avoid emitting spill load while
586  // visiting corresponding gc_relocate.
587  // Actually we do not need to record them in this map at all.
588  // We do this only to check that we are not relocating any unvisited
589  // value.
590  SpillMap.SlotMap[V] = None;
591 
592  // Default llvm mechanisms for exporting values which are used in
593  // different basic blocks does not work for gc relocates.
594  // Note that it would be incorrect to teach llvm that all relocates are
595  // uses of the corresponding values so that it would automatically
596  // export them. Relocates of the spilled values does not use original
597  // value.
598  if (Relocate->getParent() != StatepointInstr->getParent())
599  Builder.ExportFromCurrentBlock(V);
600  }
601  }
602 }
603 
606  // The basic scheme here is that information about both the original call and
607  // the safepoint is encoded in the CallInst. We create a temporary call and
608  // lower it, then reverse engineer the calling sequence.
609 
610  NumOfStatepoints++;
611  // Clear state
612  StatepointLowering.startNewStatepoint(*this);
613 
614 #ifndef NDEBUG
615  // We schedule gc relocates before removeDuplicateGCPtrs since we _will_
616  // encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
617  for (auto *Reloc : SI.GCRelocates)
618  if (Reloc->getParent() == SI.StatepointInstr->getParent())
619  StatepointLowering.scheduleRelocCall(*Reloc);
620 #endif
621 
622  // Remove any redundant llvm::Values which map to the same SDValue as another
623  // input. Also has the effect of removing duplicates in the original
624  // llvm::Value input list as well. This is a useful optimization for
625  // reducing the size of the StackMap section. It has no other impact.
626  removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
627  FuncInfo.StatepointSpillMaps[SI.StatepointInstr]);
628  assert(SI.Bases.size() == SI.Ptrs.size() &&
629  SI.Ptrs.size() == SI.GCRelocates.size());
630 
631  // Lower statepoint vmstate and gcstate arguments
632  SmallVector<SDValue, 10> LoweredMetaArgs;
633  lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this);
634 
635  // Now that we've emitted the spills, we need to update the root so that the
636  // call sequence is ordered correctly.
637  SI.CLI.setChain(getRoot());
638 
639  // Get call node, we will replace it later with statepoint
640  SDValue ReturnVal;
641  SDNode *CallNode;
642  std::tie(ReturnVal, CallNode) =
643  lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
644 
645  // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
646  // nodes with all the appropriate arguments and return values.
647 
648  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
649  SDValue Chain = CallNode->getOperand(0);
650 
651  SDValue Glue;
652  bool CallHasIncomingGlue = CallNode->getGluedNode();
653  if (CallHasIncomingGlue) {
654  // Glue is always last operand
655  Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
656  }
657 
658  // Build the GC_TRANSITION_START node if necessary.
659  //
660  // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
661  // order in which they appear in the call to the statepoint intrinsic. If
662  // any of the operands is a pointer-typed, that operand is immediately
663  // followed by a SRCVALUE for the pointer that may be used during lowering
664  // (e.g. to form MachinePointerInfo values for loads/stores).
665  const bool IsGCTransition =
668  if (IsGCTransition) {
670 
671  // Add chain
672  TSOps.push_back(Chain);
673 
674  // Add GC transition arguments
675  for (const Value *V : SI.GCTransitionArgs) {
676  TSOps.push_back(getValue(V));
677  if (V->getType()->isPointerTy())
678  TSOps.push_back(DAG.getSrcValue(V));
679  }
680 
681  // Add glue if necessary
682  if (CallHasIncomingGlue)
683  TSOps.push_back(Glue);
684 
685  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
686 
687  SDValue GCTransitionStart =
688  DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
689 
690  Chain = GCTransitionStart.getValue(0);
691  Glue = GCTransitionStart.getValue(1);
692  }
693 
694  // TODO: Currently, all of these operands are being marked as read/write in
695  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
696  // and flags to be read-only.
698 
699  // Add the <id> and <numBytes> constants.
700  Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
701  Ops.push_back(
702  DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
703 
704  // Calculate and push starting position of vmstate arguments
705  // Get number of arguments incoming directly into call node
706  unsigned NumCallRegArgs =
707  CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
708  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
709 
710  // Add call target
711  SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
712  Ops.push_back(CallTarget);
713 
714  // Add call arguments
715  // Get position of register mask in the call
716  SDNode::op_iterator RegMaskIt;
717  if (CallHasIncomingGlue)
718  RegMaskIt = CallNode->op_end() - 2;
719  else
720  RegMaskIt = CallNode->op_end() - 1;
721  Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
722 
723  // Add a constant argument for the calling convention
724  pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
725 
726  // Add a constant argument for the flags
727  uint64_t Flags = SI.StatepointFlags;
728  assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
729  "Unknown flag used");
730  pushStackMapConstant(Ops, *this, Flags);
731 
732  // Insert all vmstate and gcstate arguments
733  Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
734 
735  // Add register mask from call node
736  Ops.push_back(*RegMaskIt);
737 
738  // Add chain
739  Ops.push_back(Chain);
740 
741  // Same for the glue, but we add it only if original call had it
742  if (Glue.getNode())
743  Ops.push_back(Glue);
744 
745  // Compute return values. Provide a glue output since we consume one as
746  // input. This allows someone else to chain off us as needed.
747  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
748 
749  SDNode *StatepointMCNode =
750  DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
751 
752  SDNode *SinkNode = StatepointMCNode;
753 
754  // Build the GC_TRANSITION_END node if necessary.
755  //
756  // See the comment above regarding GC_TRANSITION_START for the layout of
757  // the operands to the GC_TRANSITION_END node.
758  if (IsGCTransition) {
760 
761  // Add chain
762  TEOps.push_back(SDValue(StatepointMCNode, 0));
763 
764  // Add GC transition arguments
765  for (const Value *V : SI.GCTransitionArgs) {
766  TEOps.push_back(getValue(V));
767  if (V->getType()->isPointerTy())
768  TEOps.push_back(DAG.getSrcValue(V));
769  }
770 
771  // Add glue
772  TEOps.push_back(SDValue(StatepointMCNode, 1));
773 
774  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
775 
776  SDValue GCTransitionStart =
777  DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
778 
779  SinkNode = GCTransitionStart.getNode();
780  }
781 
782  // Replace original call
783  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
784  // Remove original call node
785  DAG.DeleteNode(CallNode);
786 
787  // DON'T set the root - under the assumption that it's already set past the
788  // inserted node we created.
789 
790  // TODO: A better future implementation would be to emit a single variable
791  // argument, variable return value STATEPOINT node here and then hookup the
792  // return value of each gc.relocate to the respective output of the
793  // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
794  // to actually be possible today.
795 
796  return ReturnVal;
797 }
798 
799 void
801  const BasicBlock *EHPadBB /*= nullptr*/) {
803  "anyregcc is not supported on statepoints!");
804 
805 #ifndef NDEBUG
806  // If this is a malformed statepoint, report it early to simplify debugging.
807  // This should catch any IR level mistake that's made when constructing or
808  // transforming statepoints.
809  ISP.verify();
810 
811  // Check that the associated GCStrategy expects to encounter statepoints.
812  assert(GFI->getStrategy().useStatepoints() &&
813  "GCStrategy does not expect to encounter statepoints");
814 #endif
815 
816  SDValue ActualCallee;
817 
818  if (ISP.getNumPatchBytes() > 0) {
819  // If we've been asked to emit a nop sequence instead of a call instruction
820  // for this statepoint then don't lower the call target, but use a constant
821  // `null` instead. Not lowering the call target lets statepoint clients get
822  // away without providing a physical address for the symbolic call target at
823  // link time.
824 
825  const auto &TLI = DAG.getTargetLoweringInfo();
826  const auto &DL = DAG.getDataLayout();
827 
828  unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
829  ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
830  } else {
831  ActualCallee = getValue(ISP.getCalledValue());
832  }
833 
835  populateCallLoweringInfo(SI.CLI, ISP.getCallSite(),
837  ISP.getNumCallArgs(), ActualCallee,
838  ISP.getActualReturnType(), false /* IsPatchPoint */);
839 
840  for (const GCRelocateInst *Relocate : ISP.getRelocates()) {
841  SI.GCRelocates.push_back(Relocate);
842  SI.Bases.push_back(Relocate->getBasePtr());
843  SI.Ptrs.push_back(Relocate->getDerivedPtr());
844  }
845 
846  SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
847  SI.StatepointInstr = ISP.getInstruction();
848  SI.GCTransitionArgs =
850  SI.ID = ISP.getID();
851  SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end());
852  SI.StatepointFlags = ISP.getFlags();
853  SI.NumPatchBytes = ISP.getNumPatchBytes();
854  SI.EHPadBB = EHPadBB;
855 
856  SDValue ReturnValue = LowerAsSTATEPOINT(SI);
857 
858  // Export the result value if needed
859  const GCResultInst *GCResult = ISP.getGCResult();
860  Type *RetTy = ISP.getActualReturnType();
861  if (!RetTy->isVoidTy() && GCResult) {
862  if (GCResult->getParent() != ISP.getCallSite().getParent()) {
863  // Result value will be used in a different basic block so we need to
864  // export it now. Default exporting mechanism will not work here because
865  // statepoint call has a different type than the actual call. It means
866  // that by default llvm will create export register of the wrong type
867  // (always i32 in our case). So instead we need to create export register
868  // with correct type manually.
869  // TODO: To eliminate this problem we can remove gc.result intrinsics
870  // completely and make statepoint call to return a tuple.
871  unsigned Reg = FuncInfo.CreateRegs(RetTy);
872  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
873  DAG.getDataLayout(), Reg, RetTy,
874  ISP.getCallSite().getCallingConv());
875  SDValue Chain = DAG.getEntryNode();
876 
877  RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
878  PendingExports.push_back(Chain);
879  FuncInfo.ValueMap[ISP.getInstruction()] = Reg;
880  } else {
881  // Result value will be used in a same basic block. Don't export it or
882  // perform any explicit register copies.
883  // We'll replace the actuall call node shortly. gc_result will grab
884  // this value.
885  setValue(ISP.getInstruction(), ReturnValue);
886  }
887  } else {
888  // The token value is never used from here on, just generate a poison value
889  setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
890  }
891 }
892 
894  ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB,
895  bool VarArgDisallowed, bool ForceVoidReturnTy) {
897  unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin();
898  populateCallLoweringInfo(
899  SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee,
900  ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(),
901  false);
902  if (!VarArgDisallowed)
903  SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg();
904 
905  auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt);
906 
908 
910  SI.ID = SD.StatepointID.getValueOr(DefaultID);
911  SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
912 
913  SI.DeoptState =
914  ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
915  SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
916  SI.EHPadBB = EHPadBB;
917 
918  // NB! The GC arguments are deliberately left empty.
919 
920  if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
921  const Instruction *Inst = CS.getInstruction();
922  ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal);
923  setValue(Inst, ReturnVal);
924  }
925 }
926 
928  ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) {
929  LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB,
930  /* VarArgDisallowed = */ false,
931  /* ForceVoidReturnTy = */ false);
932 }
933 
934 void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
935  // The result value of the gc_result is simply the result of the actual
936  // call. We've already emitted this, so just grab the value.
937  const Instruction *I = CI.getStatepoint();
938 
939  if (I->getParent() != CI.getParent()) {
940  // Statepoint is in different basic block so we should have stored call
941  // result in a virtual register.
942  // We can not use default getValue() functionality to copy value from this
943  // register because statepoint and actual call return types can be
944  // different, and getValue() will use CopyFromReg of the wrong type,
945  // which is always i32 in our case.
946  PointerType *CalleeType = cast<PointerType>(
948  Type *RetTy =
949  cast<FunctionType>(CalleeType->getElementType())->getReturnType();
950  SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
951 
952  assert(CopyFromReg.getNode());
953  setValue(&CI, CopyFromReg);
954  } else {
955  setValue(&CI, getValue(I));
956  }
957 }
958 
959 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
960 #ifndef NDEBUG
961  // Consistency check
962  // We skip this check for relocates not in the same basic block as their
963  // statepoint. It would be too expensive to preserve validation info through
964  // different basic blocks.
965  if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
966  StatepointLowering.relocCallVisited(Relocate);
967 
968  auto *Ty = Relocate.getType()->getScalarType();
969  if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
970  assert(*IsManaged && "Non gc managed pointer relocated!");
971 #endif
972 
973  const Value *DerivedPtr = Relocate.getDerivedPtr();
974  SDValue SD = getValue(DerivedPtr);
975 
976  auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
977  auto SlotIt = SpillMap.find(DerivedPtr);
978  assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
979  Optional<int> DerivedPtrLocation = SlotIt->second;
980 
981  // We didn't need to spill these special cases (constants and allocas).
982  // See the handling in spillIncomingValueForStatepoint for detail.
983  if (!DerivedPtrLocation) {
984  setValue(&Relocate, SD);
985  return;
986  }
987 
988  SDValue SpillSlot =
989  DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
990 
991  // Be conservative: flush all pending loads
992  // TODO: Probably we can be less restrictive on this,
993  // it may allow more scheduling opportunities.
994  SDValue Chain = getRoot();
995 
996  SDValue SpillLoad =
997  DAG.getLoad(DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
998  Relocate.getType()),
999  getCurSDLoc(), Chain, SpillSlot,
1000  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
1001  *DerivedPtrLocation));
1002 
1003  // Again, be conservative, don't emit pending loads
1004  DAG.setRoot(SpillLoad.getValue(1));
1005 
1006  assert(SpillLoad.getNode());
1007  setValue(&Relocate, SpillLoad);
1008 }
1009 
1011  const auto &TLI = DAG.getTargetLoweringInfo();
1012  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1013  TLI.getPointerTy(DAG.getDataLayout()));
1014 
1015  // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1016  // call. We also do not lower the return value to any virtual register, and
1017  // change the immediately following return to a trap instruction.
1018  LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1019  /* VarArgDisallowed = */ true,
1020  /* ForceVoidReturnTy = */ true);
1021 }
1022 
1024  // We do not lower the return value from llvm.deoptimize to any virtual
1025  // register, and change the immediately following return to a trap
1026  // instruction.
1027  if (DAG.getTarget().Options.TrapUnreachable)
1028  DAG.setRoot(
1029  DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
1030 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
uint64_t CallInst * C
Represents calls to the gc.result intrinsic.
Definition: Statepoint.h:409
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:30
CallingConv::ID getCallingConv() const
Get the calling convention of the call.
Definition: CallSite.h:312
This class represents lattice values for constants.
Definition: AllocatorList.h:24
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn&#39;t known to be exported from the current basic block...
ValueTy * getCalledValue() const
Return the value actually being called or invoked.
Definition: Statepoint.h:141
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
void LowerDeoptimizeCall(const CallInst *CI)
TargetLowering::CallLoweringInfo CLI
Information regarding the underlying call instruction.
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:69
SmallVector< const GCRelocateInst *, 16 > GCRelocates
The set of gc.relocate calls associated with this gc.statepoint.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it&#39;s base class for read only access to a gc.statepoint.
Definition: Statepoint.h:305
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
uint64_t getID() const
Return the ID associated with this statepoint.
Definition: Statepoint.h:126
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
STATISTIC(NumFunctions, "Total number of functions")
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:503
CallSiteTy getCallSite() const
Return the underlying CallSite.
Definition: Statepoint.h:115
SDNode * getNode() const
get the SDNode which holds the desired result
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:463
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
op_iterator op_begin()
Definition: User.h:230
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling...
Keep track of frame indices allocated for statepoints as they could be used across basic block bounda...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool test(unsigned Idx) const
unsigned ID
The ID that the resulting STATEPOINT instruction has to report.
SDValue getRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
CallSiteTy::arg_iterator gc_args_begin() const
Definition: Statepoint.h:250
uint64_t StatepointFlags
Flags associated with the meta arguments being lowered.
CallSiteTy::arg_iterator deopt_begin() const
Definition: Statepoint.h:234
Value * getDerivedPtr() const
Definition: Statepoint.h:402
ArrayRef< const Use > GCTransitionArgs
The list of gc transition arguments present in the gc.statepoint being lowered.
CallLoweringInfo & setChain(SDValue InChain)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
op_iterator op_end() const
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
InstrTy * getInstruction() const
Definition: CallSite.h:92
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static void removeDuplicateGCPtrs(SmallVectorImpl< const Value *> &Bases, SmallVectorImpl< const Value *> &Ptrs, SmallVectorImpl< const GCRelocateInst *> &Relocs, SelectionDAGBuilder &Builder, FunctionLoweringInfo::StatepointSpillMap &SSM)
Remove any duplicate (as SDValues) from the derived pointer pairs.
InstructionTy * getInstruction() const
Definition: Statepoint.h:145
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)
Lower SLI into a STATEPOINT instruction.
void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB)
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes associated with this statepoint.
Definition: Statepoint.h:132
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:395
CallSiteTy::arg_iterator gc_args_end() const
Definition: Statepoint.h:253
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:625
virtual Optional< bool > isGCManagedPointer(const Type *Ty) const
If the type specified can be reliably distinguished, returns true for pointers to GC managed location...
Definition: GCStrategy.h:100
bool isVarArg() const
Definition: DerivedTypes.h:123
This class represents a no-op cast from one type to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:573
amdgpu Simplify well known AMD library false Value * Callee
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Class to represent pointers.
Definition: DerivedTypes.h:467
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:461
int getNumCallArgs() const
Number of arguments to be passed to the actual callee.
Definition: Statepoint.h:173
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return &#39;this&#39;.
Definition: Type.h:304
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Definition: CallSite.h:555
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:141
const Instruction * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Definition: Statepoint.h:350
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
const Instruction * StatepointInstr
The gc.statepoint instruction.
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:845
ArrayRef< const Use > DeoptState
The deoptimization state associated with this gc.statepoint call, if any.
const GCResultInst * getGCResult() const
Get the experimental_gc_result call tied to this statepoint.
Definition: Statepoint.h:275
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
static std::pair< SDValue, SDValue > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
const SDValue & getOperand(unsigned Num) const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:224
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:161
void clear()
Clear the memory usage of this object.
void verify()
Asserts if this statepoint is malformed.
Definition: Statepoint.h:286
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:760
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
DenseMap< const Instruction *, StatepointSpillMap > StatepointSpillMaps
Maps gc.statepoint instructions to their corresponding StatepointSpillMap instances.
Extended Value Type.
Definition: ValueTypes.h:34
unsigned getNumArgOperands() const
Definition: CallSite.h:293
size_t size() const
Definition: SmallVector.h:53
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1197
unsigned getNumOperands() const
Return the number of values used by this operation.
void setLocation(SDValue Val, SDValue Location)
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
SmallBitVector & set()
void clear()
Clear all bits.
static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
IterTy arg_begin() const
Definition: CallSite.h:571
RegsForValue - This struct represents the registers (physical or virtual) that a particular set of va...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:847
DenseMap< const Value *, const Value * > DuplicateMap
Maps llvm IR values to the values they were de-duplicated to.
CallSiteTy::arg_iterator deopt_end() const
Definition: Statepoint.h:239
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
size_t size() const
Returns the number of bits in this bitvector.
Represents one node in the SelectionDAG.
BBTy * getParent() const
Get the basic block containing the call site.
Definition: CallSite.h:97
GCFunctionInfo * GFI
GFI - Garbage collection metadata for the function.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:478
amdgpu Simplify well known AMD library false Value Value * Arg
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
bool hasValue() const
Definition: Optional.h:165
Represents a use of a SDNode.
void LowerStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB=nullptr)
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:133
GCStrategy describes a garbage collector algorithm&#39;s code generation requirements, and provides overridable hooks for those needs which cannot be abstractly described.
Definition: GCStrategy.h:67
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:607
Establish a view to a call site for examination.
Definition: CallSite.h:711
#define I(x, y, z)
Definition: MD5.cpp:58
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
SmallVector< const Value *, 16 > Bases
Bases[i] is the base pointer for Ptrs[i].
SDValue getValue(unsigned R) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:175
std::vector< const GCRelocateInst * > getRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition: Statepoint.h:423
Type * getType() const
Return the type of the instruction that generated this call site.
Definition: CallSite.h:264
Type * getActualReturnType() const
Return the type of the value returned by the call underlying the statepoint.
Definition: Statepoint.h:166
unsigned NumPatchBytes
The number of patchable bytes the call needs to get lowered into.
void LowerCallSiteWithDeoptBundleImpl(ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
FunctionLoweringInfo & FuncInfo
FuncInfo - Information about the function as a whole.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const BasicBlock * EHPadBB
The exception handling unwind destination, in case this represents an invoke of gc.statepoint.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Represents calls to the gc.relocate intrinsic.
Definition: Statepoint.h:374
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
Definition: Value.h:73
FunctionType * getFunctionType() const
Definition: CallSite.h:320
ArrayRef< const Use > GCArgs
The full list of gc arguments to the gc.statepoint being lowered.
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
AttributeList getAttributes() const
Get the parameter attributes of the call.
Definition: CallSite.h:329
uint64_t getFlags() const
Definition: Statepoint.h:120
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
A bitmask that includes all valid flags.
Type * getElementType() const
Definition: DerivedTypes.h:486
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:67
A discriminated union of two pointer types, with the discriminator in the low bit of the pointer...
Definition: PointerUnion.h:87
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1235