LLVM  4.0.0
StatepointLowering.cpp
Go to the documentation of this file.
1 //===-- StatepointLowering.cpp - SDAGBuilder's statepoint code -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file includes support code use by SelectionDAGBuilder when lowering a
11 // statepoint sequence in SelectionDAG IR.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "StatepointLowering.h"
16 #include "SelectionDAGBuilder.h"
17 #include "llvm/ADT/SmallSet.h"
18 #include "llvm/ADT/Statistic.h"
24 #include "llvm/CodeGen/StackMaps.h"
25 #include "llvm/IR/CallingConv.h"
26 #include "llvm/IR/Instructions.h"
27 #include "llvm/IR/IntrinsicInst.h"
28 #include "llvm/IR/Intrinsics.h"
29 #include "llvm/IR/Statepoint.h"
31 #include <algorithm>
32 using namespace llvm;
33 
34 #define DEBUG_TYPE "statepoint-lowering"
35 
36 STATISTIC(NumSlotsAllocatedForStatepoints,
37  "Number of stack slots allocated for statepoints");
38 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
39 STATISTIC(StatepointMaxSlotsRequired,
40  "Maximum number of stack slots required for a singe statepoint");
41 
43  SelectionDAGBuilder &Builder, uint64_t Value) {
44  SDLoc L = Builder.getCurSDLoc();
46  MVT::i64));
47  Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
48 }
49 
51  // Consistency check
52  assert(PendingGCRelocateCalls.empty() &&
53  "Trying to visit statepoint before finished processing previous one");
54  Locations.clear();
55  NextSlotToAllocate = 0;
56  // Need to resize this on each safepoint - we need the two to stay in sync and
57  // the clear patterns of a SelectionDAGBuilder have no relation to
58  // FunctionLoweringInfo. Also need to ensure used bits get cleared.
59  AllocatedStackSlots.clear();
60  AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
61 }
62 
64  Locations.clear();
65  AllocatedStackSlots.clear();
66  assert(PendingGCRelocateCalls.empty() &&
67  "cleared before statepoint sequence completed");
68 }
69 
70 SDValue
72  SelectionDAGBuilder &Builder) {
73  NumSlotsAllocatedForStatepoints++;
75 
76  unsigned SpillSize = ValueType.getSizeInBits() / 8;
77  assert((SpillSize * 8) == ValueType.getSizeInBits() && "Size not in bytes?");
78 
79  // First look for a previously created stack slot which is not in
80  // use (accounting for the fact arbitrary slots may already be
81  // reserved), or to create a new stack slot and use it.
82 
83  const size_t NumSlots = AllocatedStackSlots.size();
84  assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
85 
86  assert(AllocatedStackSlots.size() ==
88  "Broken invariant");
89 
90  for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
91  if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
92  const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
93  if (MFI.getObjectSize(FI) == SpillSize) {
94  AllocatedStackSlots.set(NextSlotToAllocate);
95  // TODO: Is ValueType the right thing to use here?
96  return Builder.DAG.getFrameIndex(FI, ValueType);
97  }
98  }
99  }
100 
101  // Couldn't find a free slot, so create a new one:
102 
103  SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
104  const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
106 
108  AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
109  assert(AllocatedStackSlots.size() ==
110  Builder.FuncInfo.StatepointStackSlots.size() &&
111  "Broken invariant");
112 
113  StatepointMaxSlotsRequired = std::max<unsigned long>(
114  StatepointMaxSlotsRequired, Builder.FuncInfo.StatepointStackSlots.size());
115 
116  return SpillSlot;
117 }
118 
119 /// Utility function for reservePreviousStackSlotForValue. Tries to find
120 /// stack slot index to which we have spilled value for previous statepoints.
121 /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
123  SelectionDAGBuilder &Builder,
124  int LookUpDepth) {
125  // Can not look any further - give up now
126  if (LookUpDepth <= 0)
127  return None;
128 
129  // Spill location is known for gc relocates
130  if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
131  const auto &SpillMap =
132  Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
133 
134  auto It = SpillMap.find(Relocate->getDerivedPtr());
135  if (It == SpillMap.end())
136  return None;
137 
138  return It->second;
139  }
140 
141  // Look through bitcast instructions.
142  if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
143  return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
144 
145  // Look through phi nodes
146  // All incoming values should have same known stack slot, otherwise result
147  // is unknown.
148  if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
149  Optional<int> MergedResult = None;
150 
151  for (auto &IncomingValue : Phi->incoming_values()) {
152  Optional<int> SpillSlot =
153  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
154  if (!SpillSlot.hasValue())
155  return None;
156 
157  if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
158  return None;
159 
160  MergedResult = SpillSlot;
161  }
162  return MergedResult;
163  }
164 
165  // TODO: We can do better for PHI nodes. In cases like this:
166  // ptr = phi(relocated_pointer, not_relocated_pointer)
167  // statepoint(ptr)
168  // We will return that stack slot for ptr is unknown. And later we might
169  // assign different stack slots for ptr and relocated_pointer. This limits
170  // llvm's ability to remove redundant stores.
171  // Unfortunately it's hard to accomplish in current infrastructure.
172  // We use this function to eliminate spill store completely, while
173  // in example we still need to emit store, but instead of any location
174  // we need to use special "preferred" location.
175 
176  // TODO: handle simple updates. If a value is modified and the original
177  // value is no longer live, it would be nice to put the modified value in the
178  // same slot. This allows folding of the memory accesses for some
179  // instructions types (like an increment).
180  // statepoint (i)
181  // i1 = i+1
182  // statepoint (i1)
183  // However we need to be careful for cases like this:
184  // statepoint(i)
185  // i1 = i+1
186  // statepoint(i, i1)
187  // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
188  // put handling of simple modifications in this function like it's done
189  // for bitcasts we might end up reserving i's slot for 'i+1' because order in
190  // which we visit values is unspecified.
191 
192  // Don't know any information about this instruction
193  return None;
194 }
195 
196 /// Try to find existing copies of the incoming values in stack slots used for
197 /// statepoint spilling. If we can find a spill slot for the incoming value,
198 /// mark that slot as allocated, and reuse the same slot for this safepoint.
199 /// This helps to avoid series of loads and stores that only serve to reshuffle
200 /// values on the stack between calls.
201 static void reservePreviousStackSlotForValue(const Value *IncomingValue,
202  SelectionDAGBuilder &Builder) {
203 
204  SDValue Incoming = Builder.getValue(IncomingValue);
205 
206  if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
207  // We won't need to spill this, so no need to check for previously
208  // allocated stack slots
209  return;
210  }
211 
212  SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
213  if (OldLocation.getNode())
214  // Duplicates in input
215  return;
216 
217  const int LookUpDepth = 6;
218  Optional<int> Index =
219  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
220  if (!Index.hasValue())
221  return;
222 
223  const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
224 
225  auto SlotIt = find(StatepointSlots, *Index);
226  assert(SlotIt != StatepointSlots.end() &&
227  "Value spilled to the unknown stack slot");
228 
229  // This is one of our dedicated lowering slots
230  const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
231  if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
232  // stack slot already assigned to someone else, can't use it!
233  // TODO: currently we reserve space for gc arguments after doing
234  // normal allocation for deopt arguments. We should reserve for
235  // _all_ deopt and gc arguments, then start allocating. This
236  // will prevent some moves being inserted when vm state changes,
237  // but gc state doesn't between two calls.
238  return;
239  }
240  // Reserve this stack slot
241  Builder.StatepointLowering.reserveStackSlot(Offset);
242 
243  // Cache this slot so we find it when going through the normal
244  // assignment loop.
245  SDValue Loc = Builder.DAG.getTargetFrameIndex(*Index, Incoming.getValueType());
246  Builder.StatepointLowering.setLocation(Incoming, Loc);
247 }
248 
249 /// Remove any duplicate (as SDValues) from the derived pointer pairs. This
250 /// is not required for correctness. It's purpose is to reduce the size of
251 /// StackMap section. It has no effect on the number of spill slots required
252 /// or the actual lowering.
253 static void
257  SelectionDAGBuilder &Builder,
260 
261  SmallVector<const Value *, 64> NewBases, NewPtrs;
263  for (size_t i = 0, e = Ptrs.size(); i < e; i++) {
264  SDValue SD = Builder.getValue(Ptrs[i]);
265  auto SeenIt = Seen.find(SD);
266 
267  if (SeenIt == Seen.end()) {
268  // Only add non-duplicates
269  NewBases.push_back(Bases[i]);
270  NewPtrs.push_back(Ptrs[i]);
271  NewRelocs.push_back(Relocs[i]);
272  Seen[SD] = Ptrs[i];
273  } else {
274  // Duplicate pointer found, note in SSM and move on:
275  SSM.DuplicateMap[Ptrs[i]] = SeenIt->second;
276  }
277  }
278  assert(Bases.size() >= NewBases.size());
279  assert(Ptrs.size() >= NewPtrs.size());
280  assert(Relocs.size() >= NewRelocs.size());
281  Bases = NewBases;
282  Ptrs = NewPtrs;
283  Relocs = NewRelocs;
284  assert(Ptrs.size() == Bases.size());
285  assert(Ptrs.size() == Relocs.size());
286 }
287 
288 /// Extract call from statepoint, lower it and return pointer to the
289 /// call node. Also update NodeMap so that getValue(statepoint) will
290 /// reference lowered call result
291 static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
293  SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
294 
295  SDValue ReturnValue, CallEndVal;
296  std::tie(ReturnValue, CallEndVal) =
297  Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
298  SDNode *CallEnd = CallEndVal.getNode();
299 
300  // Get a call instruction from the call sequence chain. Tail calls are not
301  // allowed. The following code is essentially reverse engineering X86's
302  // LowerCallTo.
303  //
304  // We are expecting DAG to have the following form:
305  //
306  // ch = eh_label (only in case of invoke statepoint)
307  // ch, glue = callseq_start ch
308  // ch, glue = X86::Call ch, glue
309  // ch, glue = callseq_end ch, glue
310  // get_return_value ch, glue
311  //
312  // get_return_value can either be a sequence of CopyFromReg instructions
313  // to grab the return value from the return register(s), or it can be a LOAD
314  // to load a value returned by reference via a stack slot.
315 
316  bool HasDef = !SI.CLI.RetTy->isVoidTy();
317  if (HasDef) {
318  if (CallEnd->getOpcode() == ISD::LOAD)
319  CallEnd = CallEnd->getOperand(0).getNode();
320  else
321  while (CallEnd->getOpcode() == ISD::CopyFromReg)
322  CallEnd = CallEnd->getOperand(0).getNode();
323  }
324 
325  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
326  return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
327 }
328 
329 /// Spill a value incoming to the statepoint. It might be either part of
330 /// vmstate
331 /// or gcstate. In both cases unconditionally spill it on the stack unless it
332 /// is a null constant. Return pair with first element being frame index
333 /// containing saved value and second element with outgoing chain from the
334 /// emitted store
335 static std::pair<SDValue, SDValue>
337  SelectionDAGBuilder &Builder) {
338  SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
339 
340  // Emit new store if we didn't do it for this ptr before
341  if (!Loc.getNode()) {
342  Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
343  Builder);
344  int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
345  // We use TargetFrameIndex so that isel will not select it into LEA
346  Loc = Builder.DAG.getTargetFrameIndex(Index, Incoming.getValueType());
347 
348  // TODO: We can create TokenFactor node instead of
349  // chaining stores one after another, this may allow
350  // a bit more optimal scheduling for them
351 
352 #ifndef NDEBUG
353  // Right now we always allocate spill slots that are of the same
354  // size as the value we're about to spill (the size of spillee can
355  // vary since we spill vectors of pointers too). At some point we
356  // can consider allowing spills of smaller values to larger slots
357  // (i.e. change the '==' in the assert below to a '>=').
359  assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
360  "Bad spill: stack slot does not match!");
361 #endif
362 
363  Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
365  Builder.DAG.getMachineFunction(), Index));
366 
367  Builder.StatepointLowering.setLocation(Incoming, Loc);
368  }
369 
370  assert(Loc.getNode());
371  return std::make_pair(Loc, Chain);
372 }
373 
374 /// Lower a single value incoming to a statepoint node. This value can be
375 /// either a deopt value or a gc value, the handling is the same. We special
376 /// case constants and allocas, then fall back to spilling if required.
377 static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
379  SelectionDAGBuilder &Builder) {
380  SDValue Chain = Builder.getRoot();
381 
382  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
383  // If the original value was a constant, make sure it gets recorded as
384  // such in the stackmap. This is required so that the consumer can
385  // parse any internal format to the deopt state. It also handles null
386  // pointers and other constant pointers in GC states. Note the constant
387  // vectors do not appear to actually hit this path and that anything larger
388  // than an i64 value (not type!) will fail asserts here.
389  pushStackMapConstant(Ops, Builder, C->getSExtValue());
390  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
391  // This handles allocas as arguments to the statepoint (this is only
392  // really meaningful for a deopt value. For GC, we'd be trying to
393  // relocate the address of the alloca itself?)
394  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
395  Incoming.getValueType()));
396  } else if (LiveInOnly) {
397  // If this value is live in (not live-on-return, or live-through), we can
398  // treat it the same way patchpoint treats it's "live in" values. We'll
399  // end up folding some of these into stack references, but they'll be
400  // handled by the register allocator. Note that we do not have the notion
401  // of a late use so these values might be placed in registers which are
402  // clobbered by the call. This is fine for live-in.
403  Ops.push_back(Incoming);
404  } else {
405  // Otherwise, locate a spill slot and explicitly spill it so it
406  // can be found by the runtime later. We currently do not support
407  // tracking values through callee saved registers to their eventual
408  // spill location. This would be a useful optimization, but would
409  // need to be optional since it requires a lot of complexity on the
410  // runtime side which not all would support.
411  auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
412  Ops.push_back(Res.first);
413  Chain = Res.second;
414  }
415 
416  Builder.DAG.setRoot(Chain);
417 }
418 
419 /// Lower deopt state and gc pointer arguments of the statepoint. The actual
420 /// lowering is described in lowerIncomingStatepointValue. This function is
421 /// responsible for lowering everything in the right position and playing some
422 /// tricks to avoid redundant stack manipulation where possible. On
423 /// completion, 'Ops' will contain ready to use operands for machine code
424 /// statepoint. The chain nodes will have already been created and the DAG root
425 /// will be set to the last value spilled (if any were).
426 static void
429  SelectionDAGBuilder &Builder) {
430  // Lower the deopt and gc arguments for this statepoint. Layout will be:
431  // deopt argument length, deopt arguments.., gc arguments...
432 #ifndef NDEBUG
433  if (auto *GFI = Builder.GFI) {
434  // Check that each of the gc pointer and bases we've gotten out of the
435  // safepoint is something the strategy thinks might be a pointer (or vector
436  // of pointers) into the GC heap. This is basically just here to help catch
437  // errors during statepoint insertion. TODO: This should actually be in the
438  // Verifier, but we can't get to the GCStrategy from there (yet).
439  GCStrategy &S = GFI->getStrategy();
440  for (const Value *V : SI.Bases) {
441  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
442  if (Opt.hasValue()) {
443  assert(Opt.getValue() &&
444  "non gc managed base pointer found in statepoint");
445  }
446  }
447  for (const Value *V : SI.Ptrs) {
448  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
449  if (Opt.hasValue()) {
450  assert(Opt.getValue() &&
451  "non gc managed derived pointer found in statepoint");
452  }
453  }
454  assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
455  } else {
456  assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
457  assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
458  }
459 #endif
460 
461  // Figure out what lowering strategy we're going to use for each part
462  // Note: Is is conservatively correct to lower both "live-in" and "live-out"
463  // as "live-through". A "live-through" variable is one which is "live-in",
464  // "live-out", and live throughout the lifetime of the call (i.e. we can find
465  // it from any PC within the transitive callee of the statepoint). In
466  // particular, if the callee spills callee preserved registers we may not
467  // be able to find a value placed in that register during the call. This is
468  // fine for live-out, but not for live-through. If we were willing to make
469  // assumptions about the code generator producing the callee, we could
470  // potentially allow live-through values in callee saved registers.
471  const bool LiveInDeopt =
473 
474  auto isGCValue =[&](const Value *V) {
475  return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
476  };
477 
478  // Before we actually start lowering (and allocating spill slots for values),
479  // reserve any stack slots which we judge to be profitable to reuse for a
480  // particular value. This is purely an optimization over the code below and
481  // doesn't change semantics at all. It is important for performance that we
482  // reserve slots for both deopt and gc values before lowering either.
483  for (const Value *V : SI.DeoptState) {
484  if (!LiveInDeopt || isGCValue(V))
486  }
487  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
490  }
491 
492  // First, prefix the list with the number of unique values to be
493  // lowered. Note that this is the number of *Values* not the
494  // number of SDValues required to lower them.
495  const int NumVMSArgs = SI.DeoptState.size();
496  pushStackMapConstant(Ops, Builder, NumVMSArgs);
497 
498  // The vm state arguments are lowered in an opaque manner. We do not know
499  // what type of values are contained within.
500  for (const Value *V : SI.DeoptState) {
501  SDValue Incoming = Builder.getValue(V);
502  const bool LiveInValue = LiveInDeopt && !isGCValue(V);
503  lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder);
504  }
505 
506  // Finally, go ahead and lower all the gc arguments. There's no prefixed
507  // length for this one. After lowering, we'll have the base and pointer
508  // arrays interwoven with each (lowered) base pointer immediately followed by
509  // it's (lowered) derived pointer. i.e
510  // (base[0], ptr[0], base[1], ptr[1], ...)
511  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
512  const Value *Base = SI.Bases[i];
513  lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
514  Ops, Builder);
515 
516  const Value *Ptr = SI.Ptrs[i];
517  lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
518  Ops, Builder);
519  }
520 
521  // If there are any explicit spill slots passed to the statepoint, record
522  // them, but otherwise do not do anything special. These are user provided
523  // allocas and give control over placement to the consumer. In this case,
524  // it is the contents of the slot which may get updated, not the pointer to
525  // the alloca
526  for (Value *V : SI.GCArgs) {
527  SDValue Incoming = Builder.getValue(V);
528  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
529  // This handles allocas as arguments to the statepoint
530  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
531  Incoming.getValueType()));
532  }
533  }
534 
535  // Record computed locations for all lowered values.
536  // This can not be embedded in lowering loops as we need to record *all*
537  // values, while previous loops account only values with unique SDValues.
538  const Instruction *StatepointInstr = SI.StatepointInstr;
539  auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
540 
541  for (const GCRelocateInst *Relocate : SI.GCRelocates) {
542  const Value *V = Relocate->getDerivedPtr();
543  SDValue SDV = Builder.getValue(V);
544  SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
545 
546  if (Loc.getNode()) {
547  SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
548  } else {
549  // Record value as visited, but not spilled. This is case for allocas
550  // and constants. For this values we can avoid emitting spill load while
551  // visiting corresponding gc_relocate.
552  // Actually we do not need to record them in this map at all.
553  // We do this only to check that we are not relocating any unvisited
554  // value.
555  SpillMap.SlotMap[V] = None;
556 
557  // Default llvm mechanisms for exporting values which are used in
558  // different basic blocks does not work for gc relocates.
559  // Note that it would be incorrect to teach llvm that all relocates are
560  // uses of the corresponding values so that it would automatically
561  // export them. Relocates of the spilled values does not use original
562  // value.
563  if (Relocate->getParent() != StatepointInstr->getParent())
564  Builder.ExportFromCurrentBlock(V);
565  }
566  }
567 }
568 
571  // The basic scheme here is that information about both the original call and
572  // the safepoint is encoded in the CallInst. We create a temporary call and
573  // lower it, then reverse engineer the calling sequence.
574 
575  NumOfStatepoints++;
576  // Clear state
578 
579 #ifndef NDEBUG
580  // We schedule gc relocates before removeDuplicateGCPtrs since we _will_
581  // encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
582  for (auto *Reloc : SI.GCRelocates)
583  if (Reloc->getParent() == SI.StatepointInstr->getParent())
585 #endif
586 
587  // Remove any redundant llvm::Values which map to the same SDValue as another
588  // input. Also has the effect of removing duplicates in the original
589  // llvm::Value input list as well. This is a useful optimization for
590  // reducing the size of the StackMap section. It has no other impact.
591  removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
593  assert(SI.Bases.size() == SI.Ptrs.size() &&
594  SI.Ptrs.size() == SI.GCRelocates.size());
595 
596  // Lower statepoint vmstate and gcstate arguments
597  SmallVector<SDValue, 10> LoweredMetaArgs;
598  lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this);
599 
600  // Now that we've emitted the spills, we need to update the root so that the
601  // call sequence is ordered correctly.
602  SI.CLI.setChain(getRoot());
603 
604  // Get call node, we will replace it later with statepoint
605  SDValue ReturnVal;
606  SDNode *CallNode;
607  std::tie(ReturnVal, CallNode) =
608  lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
609 
610  // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
611  // nodes with all the appropriate arguments and return values.
612 
613  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
614  SDValue Chain = CallNode->getOperand(0);
615 
616  SDValue Glue;
617  bool CallHasIncomingGlue = CallNode->getGluedNode();
618  if (CallHasIncomingGlue) {
619  // Glue is always last operand
620  Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
621  }
622 
623  // Build the GC_TRANSITION_START node if necessary.
624  //
625  // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
626  // order in which they appear in the call to the statepoint intrinsic. If
627  // any of the operands is a pointer-typed, that operand is immediately
628  // followed by a SRCVALUE for the pointer that may be used during lowering
629  // (e.g. to form MachinePointerInfo values for loads/stores).
630  const bool IsGCTransition =
633  if (IsGCTransition) {
635 
636  // Add chain
637  TSOps.push_back(Chain);
638 
639  // Add GC transition arguments
640  for (const Value *V : SI.GCTransitionArgs) {
641  TSOps.push_back(getValue(V));
642  if (V->getType()->isPointerTy())
643  TSOps.push_back(DAG.getSrcValue(V));
644  }
645 
646  // Add glue if necessary
647  if (CallHasIncomingGlue)
648  TSOps.push_back(Glue);
649 
651 
652  SDValue GCTransitionStart =
653  DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
654 
655  Chain = GCTransitionStart.getValue(0);
656  Glue = GCTransitionStart.getValue(1);
657  }
658 
659  // TODO: Currently, all of these operands are being marked as read/write in
660  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
661  // and flags to be read-only.
663 
664  // Add the <id> and <numBytes> constants.
666  Ops.push_back(
668 
669  // Calculate and push starting position of vmstate arguments
670  // Get number of arguments incoming directly into call node
671  unsigned NumCallRegArgs =
672  CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
673  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
674 
675  // Add call target
676  SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
677  Ops.push_back(CallTarget);
678 
679  // Add call arguments
680  // Get position of register mask in the call
681  SDNode::op_iterator RegMaskIt;
682  if (CallHasIncomingGlue)
683  RegMaskIt = CallNode->op_end() - 2;
684  else
685  RegMaskIt = CallNode->op_end() - 1;
686  Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
687 
688  // Add a constant argument for the calling convention
689  pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
690 
691  // Add a constant argument for the flags
692  uint64_t Flags = SI.StatepointFlags;
693  assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
694  "Unknown flag used");
695  pushStackMapConstant(Ops, *this, Flags);
696 
697  // Insert all vmstate and gcstate arguments
698  Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
699 
700  // Add register mask from call node
701  Ops.push_back(*RegMaskIt);
702 
703  // Add chain
704  Ops.push_back(Chain);
705 
706  // Same for the glue, but we add it only if original call had it
707  if (Glue.getNode())
708  Ops.push_back(Glue);
709 
710  // Compute return values. Provide a glue output since we consume one as
711  // input. This allows someone else to chain off us as needed.
713 
714  SDNode *StatepointMCNode =
715  DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
716 
717  SDNode *SinkNode = StatepointMCNode;
718 
719  // Build the GC_TRANSITION_END node if necessary.
720  //
721  // See the comment above regarding GC_TRANSITION_START for the layout of
722  // the operands to the GC_TRANSITION_END node.
723  if (IsGCTransition) {
725 
726  // Add chain
727  TEOps.push_back(SDValue(StatepointMCNode, 0));
728 
729  // Add GC transition arguments
730  for (const Value *V : SI.GCTransitionArgs) {
731  TEOps.push_back(getValue(V));
732  if (V->getType()->isPointerTy())
733  TEOps.push_back(DAG.getSrcValue(V));
734  }
735 
736  // Add glue
737  TEOps.push_back(SDValue(StatepointMCNode, 1));
738 
740 
741  SDValue GCTransitionStart =
742  DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
743 
744  SinkNode = GCTransitionStart.getNode();
745  }
746 
747  // Replace original call
748  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
749  // Remove original call node
750  DAG.DeleteNode(CallNode);
751 
752  // DON'T set the root - under the assumption that it's already set past the
753  // inserted node we created.
754 
755  // TODO: A better future implementation would be to emit a single variable
756  // argument, variable return value STATEPOINT node here and then hookup the
757  // return value of each gc.relocate to the respective output of the
758  // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
759  // to actually be possible today.
760 
761  return ReturnVal;
762 }
763 
764 void
766  const BasicBlock *EHPadBB /*= nullptr*/) {
768  "anyregcc is not supported on statepoints!");
769 
770 #ifndef NDEBUG
771  // If this is a malformed statepoint, report it early to simplify debugging.
772  // This should catch any IR level mistake that's made when constructing or
773  // transforming statepoints.
774  ISP.verify();
775 
776  // Check that the associated GCStrategy expects to encounter statepoints.
778  "GCStrategy does not expect to encounter statepoints");
779 #endif
780 
781  SDValue ActualCallee;
782 
783  if (ISP.getNumPatchBytes() > 0) {
784  // If we've been asked to emit a nop sequence instead of a call instruction
785  // for this statepoint then don't lower the call target, but use a constant
786  // `null` instead. Not lowering the call target lets statepoint clients get
787  // away without providing a physical address for the symbolic call target at
788  // link time.
789 
790  const auto &TLI = DAG.getTargetLoweringInfo();
791  const auto &DL = DAG.getDataLayout();
792 
793  unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
794  ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
795  } else {
796  ActualCallee = getValue(ISP.getCalledValue());
797  }
798 
802  ISP.getNumCallArgs(), ActualCallee,
803  ISP.getActualReturnType(), false /* IsPatchPoint */);
804 
805  for (const GCRelocateInst *Relocate : ISP.getRelocates()) {
806  SI.GCRelocates.push_back(Relocate);
807  SI.Bases.push_back(Relocate->getBasePtr());
808  SI.Ptrs.push_back(Relocate->getDerivedPtr());
809  }
810 
812  SI.StatepointInstr = ISP.getInstruction();
813  SI.GCTransitionArgs =
815  SI.ID = ISP.getID();
817  SI.StatepointFlags = ISP.getFlags();
818  SI.NumPatchBytes = ISP.getNumPatchBytes();
819  SI.EHPadBB = EHPadBB;
820 
821  SDValue ReturnValue = LowerAsSTATEPOINT(SI);
822 
823  // Export the result value if needed
824  const GCResultInst *GCResult = ISP.getGCResult();
825  Type *RetTy = ISP.getActualReturnType();
826  if (!RetTy->isVoidTy() && GCResult) {
827  if (GCResult->getParent() != ISP.getCallSite().getParent()) {
828  // Result value will be used in a different basic block so we need to
829  // export it now. Default exporting mechanism will not work here because
830  // statepoint call has a different type than the actual call. It means
831  // that by default llvm will create export register of the wrong type
832  // (always i32 in our case). So instead we need to create export register
833  // with correct type manually.
834  // TODO: To eliminate this problem we can remove gc.result intrinsics
835  // completely and make statepoint call to return a tuple.
836  unsigned Reg = FuncInfo.CreateRegs(RetTy);
838  DAG.getDataLayout(), Reg, RetTy);
839  SDValue Chain = DAG.getEntryNode();
840 
841  RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
842  PendingExports.push_back(Chain);
844  } else {
845  // Result value will be used in a same basic block. Don't export it or
846  // perform any explicit register copies.
847  // We'll replace the actuall call node shortly. gc_result will grab
848  // this value.
849  setValue(ISP.getInstruction(), ReturnValue);
850  }
851  } else {
852  // The token value is never used from here on, just generate a poison value
854  }
855 }
856 
858  ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB,
859  bool VarArgDisallowed, bool ForceVoidReturnTy) {
861  unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin();
863  SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee,
864  ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(),
865  false);
866  if (!VarArgDisallowed)
867  SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg();
868 
869  auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt);
870 
872 
874  SI.ID = SD.StatepointID.getValueOr(DefaultID);
875  SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
876 
877  SI.DeoptState =
878  ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
879  SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
880  SI.EHPadBB = EHPadBB;
881 
882  // NB! The GC arguments are deliberately left empty.
883 
884  if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
885  const Instruction *Inst = CS.getInstruction();
886  ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal);
887  setValue(Inst, ReturnVal);
888  }
889 }
890 
892  ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) {
893  LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB,
894  /* VarArgDisallowed = */ false,
895  /* ForceVoidReturnTy = */ false);
896 }
897 
898 void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
899  // The result value of the gc_result is simply the result of the actual
900  // call. We've already emitted this, so just grab the value.
901  const Instruction *I = CI.getStatepoint();
902 
903  if (I->getParent() != CI.getParent()) {
904  // Statepoint is in different basic block so we should have stored call
905  // result in a virtual register.
906  // We can not use default getValue() functionality to copy value from this
907  // register because statepoint and actual call return types can be
908  // different, and getValue() will use CopyFromReg of the wrong type,
909  // which is always i32 in our case.
910  PointerType *CalleeType = cast<PointerType>(
912  Type *RetTy =
913  cast<FunctionType>(CalleeType->getElementType())->getReturnType();
914  SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
915 
916  assert(CopyFromReg.getNode());
917  setValue(&CI, CopyFromReg);
918  } else {
919  setValue(&CI, getValue(I));
920  }
921 }
922 
923 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
924 #ifndef NDEBUG
925  // Consistency check
926  // We skip this check for relocates not in the same basic block as their
927  // statepoint. It would be too expensive to preserve validation info through
928  // different basic blocks.
929  if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
931 
932  auto *Ty = Relocate.getType()->getScalarType();
933  if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
934  assert(*IsManaged && "Non gc managed pointer relocated!");
935 #endif
936 
937  const Value *DerivedPtr = Relocate.getDerivedPtr();
938  SDValue SD = getValue(DerivedPtr);
939 
940  auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
941  auto SlotIt = SpillMap.find(DerivedPtr);
942  assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
943  Optional<int> DerivedPtrLocation = SlotIt->second;
944 
945  // We didn't need to spill these special cases (constants and allocas).
946  // See the handling in spillIncomingValueForStatepoint for detail.
947  if (!DerivedPtrLocation) {
948  setValue(&Relocate, SD);
949  return;
950  }
951 
952  SDValue SpillSlot = DAG.getTargetFrameIndex(*DerivedPtrLocation,
953  SD.getValueType());
954 
955  // Be conservative: flush all pending loads
956  // TODO: Probably we can be less restrictive on this,
957  // it may allow more scheduling opportunities.
958  SDValue Chain = getRoot();
959 
960  SDValue SpillLoad =
961  DAG.getLoad(SpillSlot.getValueType(), getCurSDLoc(), Chain, SpillSlot,
963  *DerivedPtrLocation));
964 
965  // Again, be conservative, don't emit pending loads
966  DAG.setRoot(SpillLoad.getValue(1));
967 
968  assert(SpillLoad.getNode());
969  setValue(&Relocate, SpillLoad);
970 }
971 
973  const auto &TLI = DAG.getTargetLoweringInfo();
974  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
975  TLI.getPointerTy(DAG.getDataLayout()));
976 
977  // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
978  // call. We also do not lower the return value to any virtual register, and
979  // change the immediately following return to a trap instruction.
980  LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
981  /* VarArgDisallowed = */ true,
982  /* ForceVoidReturnTy = */ true);
983 }
984 
986  // We do not lower the return value from llvm.deoptimize to any virtual
987  // register, and change the immediately following return to a trap
988  // instruction.
990  DAG.setRoot(
992 }
MachineLoop * L
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
const NoneType None
Definition: None.h:23
const Instruction * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Definition: Statepoint.h:343
Represents calls to the gc.result intrinsic.
Definition: Statepoint.h:401
SDValue getValue(unsigned R) const
ValueTy * getCalledValue() const
Return the value actually being called or invoked.
Definition: Statepoint.h:136
LLVMContext * getContext() const
Definition: SelectionDAG.h:333
STATISTIC(NumFunctions,"Total number of functions")
bool hasValue() const
Definition: Optional.h:125
size_t i
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block...
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
void LowerDeoptimizeCall(const CallInst *CI)
TargetLowering::CallLoweringInfo CLI
Information regarding the underlying call instruction.
SmallVector< const GCRelocateInst *, 16 > GCRelocates
The set of gc.relocate calls associated with this gc.statepoint.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it's base class for read only access to a gc.statepoint.
Definition: Statepoint.h:300
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:329
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
BBTy * getParent() const
Get the basic block containing the call site.
Definition: CallSite.h:98
void LowerStatepoint(ImmutableStatepoint Statepoint, const BasicBlock *EHPadBB=nullptr)
This class represents a function call, abstracting a target machine's calling convention.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
void setValue(const Value *V, SDValue NewN)
CallSiteTy::arg_iterator vm_state_begin() const
Definition: Statepoint.h:229
unsigned getNumOperands() const
Return the number of values used by this operation.
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:387
Type * getElementType() const
Definition: DerivedTypes.h:462
void DeleteNode(SDNode *N)
Remove the specified node from the system.
op_iterator op_begin()
Definition: User.h:205
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling...
Keep track of frame indices allocated for statepoints as they could be used across basic block bounda...
SDValue getExternalSymbol(const char *Sym, EVT VT)
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
unsigned ID
The ID that the resulting STATEPOINT instruction has to report.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getRoot()
getRoot - Return the current virtual root of the Selection DAG, flushing any PendingLoad items...
Value * getDerivedPtr() const
Definition: Statepoint.h:394
struct fuzzer::@269 Flags
uint64_t StatepointFlags
Flags associated with the meta arguments being lowered.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:327
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
ArrayRef< const Use > GCTransitionArgs
The list of gc transition arguments present in the gc.statepoint being lowered.
CallLoweringInfo & setChain(SDValue InChain)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
Reg
All possible values of the reg field in the ModR/M byte.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
CallSiteTy getCallSite() const
Return the underlying CallSite.
Definition: Statepoint.h:110
void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - get or set the calling convention of the call.
Definition: CallSite.h:308
bool useStatepoints() const
Returns true if this strategy is expecting the use of gc.statepoints, and false otherwise.
Definition: GCStrategy.h:115
InstructionTy * getInstruction() const
Definition: Statepoint.h:140
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:535
This class represents a no-op cast from one type to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
virtual Optional< bool > isGCManagedPointer(const Type *Ty) const
If the type specified can be reliably distinguished, returns true for pointers to GC managed location...
Definition: GCStrategy.h:124
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:487
Type * getScalarType() const LLVM_READONLY
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.cpp:44
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:328
Class to represent pointers.
Definition: DerivedTypes.h:443
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:452
const GCResultInst * getGCResult() const
Get the experimental_gc_result call tied to this statepoint.
Definition: Statepoint.h:270
SDNode * getNode() const
get the SDNode which holds the desired result
CallSiteTy::arg_iterator gc_args_begin() const
Definition: Statepoint.h:245
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
const Instruction * StatepointInstr
The gc.statepoint instruction.
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Definition: CallSite.h:512
FunctionType * getFunctionType() const
Definition: CallSite.h:315
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:751
ArrayRef< const Use > DeoptState
The deoptimization state associated with this gc.statepoint call, if any.
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
static std::pair< SDValue, SDValue > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
CallSiteTy::arg_iterator gc_args_end() const
Definition: Statepoint.h:248
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
GCStrategy & getStrategy()
getStrategy - Return the GC strategy for the function.
Definition: GCMetadata.h:107
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
std::vector< const GCRelocateInst * > getRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition: Statepoint.h:414
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:154
uint32_t Offset
void clear()
Clear the memory usage of this object.
void verify()
Asserts if this statepoint is malformed.
Definition: Statepoint.h:281
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:676
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
DenseMap< const Instruction *, StatepointSpillMap > StatepointSpillMaps
Maps gc.statepoint instructions to their corresponding StatepointSpillMap instances.
EVT - Extended Value Type.
Definition: ValueTypes.h:31
Type * getType() const
getType - Return the type of the instruction that generated this call site
Definition: CallSite.h:258
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:213
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeSet AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:56
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:378
void setLocation(SDValue Val, SDValue Location)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
SmallBitVector & set()
AttributeSet getAttributes() const
getAttributes/setAttributes - get or set the parameter attributes of the call.
Definition: CallSite.h:325
IterTy arg_begin() const
Definition: CallSite.h:528
void clear()
Clear all bits.
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:757
InstrTy * getInstruction() const
Definition: CallSite.h:93
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SLI)
Lower SLI into a STATEPOINT instruction.
static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
CallSiteTy::arg_iterator vm_state_end() const
Definition: Statepoint.h:234
RegsForValue - This struct represents the registers (physical or virtual) that a particular set of va...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
uint64_t getID() const
Return the ID associated with this statepoint.
Definition: Statepoint.h:121
DenseMap< const Value *, const Value * > DuplicateMap
Maps llvm IR values to the values they were de-duplicated to.
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Type * getActualReturnType() const
Return the type of the value returned by the call underlying the statepoint.
Definition: Statepoint.h:161
int getNumCallArgs() const
Number of arguments to be passed to the actual callee.
Definition: Statepoint.h:168
unsigned CreateRegs(Type *Ty)
CreateRegs - Allocate the appropriate number of virtual registers of the correctly promoted or expand...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
size_t size() const
Returns the number of bits in this bitvector.
GCFunctionInfo * GFI
GFI - Garbage collection metadata for the function.
unsigned getNumArgOperands() const
Definition: CallSite.h:288
op_iterator op_begin() const
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:464
bool test(unsigned Idx) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Represents a use of a SDNode.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
GCStrategy describes a garbage collector algorithm's code generation requirements, and provides overridable hooks for those needs which cannot be abstractly described.
Definition: GCStrategy.h:78
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:536
ImmutableCallSite - establish a view to a call site for examination.
Definition: CallSite.h:665
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:256
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
#define I(x, y, z)
Definition: MD5.cpp:54
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
op_iterator op_end() const
SmallVector< const Value *, 16 > Bases
Bases[i] is the base pointer for Ptrs[i].
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:175
SDValue getCopyFromRegs(const Value *V, Type *Ty)
getCopyFromRegs - If there was virtual register allocated for the value V emit CopyFromReg of the spe...
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
bool isVarArg() const
Definition: DerivedTypes.h:122
unsigned NumPatchBytes
The number of patchable bytes the call needs to get lowered into.
void LowerCallSiteWithDeoptBundleImpl(ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
FunctionLoweringInfo & FuncInfo
FuncInfo - Information about the function as a whole.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const BasicBlock * EHPadBB
The exception handling unwind destination, in case this represents an invoke of gc.statepoint.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Represents calls to the gc.relocate intrinsic.
Definition: Statepoint.h:367
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
Definition: Value.h:71
void scheduleRelocCall(const CallInst &RelocCall)
Record the fact that we expect to encounter a given gc_relocate before the next statepoint.
ArrayRef< const Use > GCArgs
The full list of gc arguments to the gc.statepoint being lowered.
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:331
uint32_t getNumPatchBytes() const
Return the number of patchable bytes associated with this statepoint.
Definition: Statepoint.h:127
void relocCallVisited(const CallInst &RelocCall)
Remove this gc_relocate from the list we're expecting to see before the next statepoint.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
static void removeDuplicateGCPtrs(SmallVectorImpl< const Value * > &Bases, SmallVectorImpl< const Value * > &Ptrs, SmallVectorImpl< const GCRelocateInst * > &Relocs, SelectionDAGBuilder &Builder, FunctionLoweringInfo::StatepointSpillMap &SSM)
Remove any duplicate (as SDValues) from the derived pointer pairs.
int * Ptr
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:381
uint64_t getFlags() const
Definition: Statepoint.h:115
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
A bitmask that includes all valid flags.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:479
const BasicBlock * getParent() const
Definition: Instruction.h:62
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
This file describes how to lower LLVM code to machine code.
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:783