LLVM  9.0.0svn
StatepointLowering.cpp
Go to the documentation of this file.
1 //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file includes support code use by SelectionDAGBuilder when lowering a
10 // statepoint sequence in SelectionDAG IR.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "StatepointLowering.h"
15 #include "SelectionDAGBuilder.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/Statistic.h"
33 #include "llvm/CodeGen/StackMaps.h"
36 #include "llvm/IR/CallingConv.h"
37 #include "llvm/IR/DerivedTypes.h"
38 #include "llvm/IR/Instruction.h"
39 #include "llvm/IR/Instructions.h"
40 #include "llvm/IR/LLVMContext.h"
41 #include "llvm/IR/Statepoint.h"
42 #include "llvm/IR/Type.h"
43 #include "llvm/Support/Casting.h"
47 #include <cassert>
48 #include <cstddef>
49 #include <cstdint>
50 #include <iterator>
51 #include <tuple>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "statepoint-lowering"
57 
58 STATISTIC(NumSlotsAllocatedForStatepoints,
59  "Number of stack slots allocated for statepoints");
60 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
61 STATISTIC(StatepointMaxSlotsRequired,
62  "Maximum number of stack slots required for a singe statepoint");
63 
65  SelectionDAGBuilder &Builder, uint64_t Value) {
66  SDLoc L = Builder.getCurSDLoc();
67  Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
68  MVT::i64));
69  Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
70 }
71 
73  // Consistency check
74  assert(PendingGCRelocateCalls.empty() &&
75  "Trying to visit statepoint before finished processing previous one");
76  Locations.clear();
77  NextSlotToAllocate = 0;
78  // Need to resize this on each safepoint - we need the two to stay in sync and
79  // the clear patterns of a SelectionDAGBuilder have no relation to
80  // FunctionLoweringInfo. Also need to ensure used bits get cleared.
81  AllocatedStackSlots.clear();
82  AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
83 }
84 
86  Locations.clear();
87  AllocatedStackSlots.clear();
88  assert(PendingGCRelocateCalls.empty() &&
89  "cleared before statepoint sequence completed");
90 }
91 
92 SDValue
94  SelectionDAGBuilder &Builder) {
95  NumSlotsAllocatedForStatepoints++;
97 
98  unsigned SpillSize = ValueType.getStoreSize();
99  assert((SpillSize * 8) == ValueType.getSizeInBits() && "Size not in bytes?");
100 
101  // First look for a previously created stack slot which is not in
102  // use (accounting for the fact arbitrary slots may already be
103  // reserved), or to create a new stack slot and use it.
104 
105  const size_t NumSlots = AllocatedStackSlots.size();
106  assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
107 
108  assert(AllocatedStackSlots.size() ==
109  Builder.FuncInfo.StatepointStackSlots.size() &&
110  "Broken invariant");
111 
112  for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
113  if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
114  const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
115  if (MFI.getObjectSize(FI) == SpillSize) {
116  AllocatedStackSlots.set(NextSlotToAllocate);
117  // TODO: Is ValueType the right thing to use here?
118  return Builder.DAG.getFrameIndex(FI, ValueType);
119  }
120  }
121  }
122 
123  // Couldn't find a free slot, so create a new one:
124 
125  SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
126  const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
128 
130  AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
131  assert(AllocatedStackSlots.size() ==
132  Builder.FuncInfo.StatepointStackSlots.size() &&
133  "Broken invariant");
134 
135  StatepointMaxSlotsRequired.updateMax(
137 
138  return SpillSlot;
139 }
140 
141 /// Utility function for reservePreviousStackSlotForValue. Tries to find
142 /// stack slot index to which we have spilled value for previous statepoints.
143 /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
145  SelectionDAGBuilder &Builder,
146  int LookUpDepth) {
147  // Can not look any further - give up now
148  if (LookUpDepth <= 0)
149  return None;
150 
151  // Spill location is known for gc relocates
152  if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
153  const auto &SpillMap =
154  Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
155 
156  auto It = SpillMap.find(Relocate->getDerivedPtr());
157  if (It == SpillMap.end())
158  return None;
159 
160  return It->second;
161  }
162 
163  // Look through bitcast instructions.
164  if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
165  return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
166 
167  // Look through phi nodes
168  // All incoming values should have same known stack slot, otherwise result
169  // is unknown.
170  if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
171  Optional<int> MergedResult = None;
172 
173  for (auto &IncomingValue : Phi->incoming_values()) {
174  Optional<int> SpillSlot =
175  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
176  if (!SpillSlot.hasValue())
177  return None;
178 
179  if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
180  return None;
181 
182  MergedResult = SpillSlot;
183  }
184  return MergedResult;
185  }
186 
187  // TODO: We can do better for PHI nodes. In cases like this:
188  // ptr = phi(relocated_pointer, not_relocated_pointer)
189  // statepoint(ptr)
190  // We will return that stack slot for ptr is unknown. And later we might
191  // assign different stack slots for ptr and relocated_pointer. This limits
192  // llvm's ability to remove redundant stores.
193  // Unfortunately it's hard to accomplish in current infrastructure.
194  // We use this function to eliminate spill store completely, while
195  // in example we still need to emit store, but instead of any location
196  // we need to use special "preferred" location.
197 
198  // TODO: handle simple updates. If a value is modified and the original
199  // value is no longer live, it would be nice to put the modified value in the
200  // same slot. This allows folding of the memory accesses for some
201  // instructions types (like an increment).
202  // statepoint (i)
203  // i1 = i+1
204  // statepoint (i1)
205  // However we need to be careful for cases like this:
206  // statepoint(i)
207  // i1 = i+1
208  // statepoint(i, i1)
209  // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
210  // put handling of simple modifications in this function like it's done
211  // for bitcasts we might end up reserving i's slot for 'i+1' because order in
212  // which we visit values is unspecified.
213 
214  // Don't know any information about this instruction
215  return None;
216 }
217 
218 /// Try to find existing copies of the incoming values in stack slots used for
219 /// statepoint spilling. If we can find a spill slot for the incoming value,
220 /// mark that slot as allocated, and reuse the same slot for this safepoint.
221 /// This helps to avoid series of loads and stores that only serve to reshuffle
222 /// values on the stack between calls.
223 static void reservePreviousStackSlotForValue(const Value *IncomingValue,
224  SelectionDAGBuilder &Builder) {
225  SDValue Incoming = Builder.getValue(IncomingValue);
226 
227  if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
228  // We won't need to spill this, so no need to check for previously
229  // allocated stack slots
230  return;
231  }
232 
233  SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
234  if (OldLocation.getNode())
235  // Duplicates in input
236  return;
237 
238  const int LookUpDepth = 6;
240  findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
241  if (!Index.hasValue())
242  return;
243 
244  const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
245 
246  auto SlotIt = find(StatepointSlots, *Index);
247  assert(SlotIt != StatepointSlots.end() &&
248  "Value spilled to the unknown stack slot");
249 
250  // This is one of our dedicated lowering slots
251  const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
252  if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
253  // stack slot already assigned to someone else, can't use it!
254  // TODO: currently we reserve space for gc arguments after doing
255  // normal allocation for deopt arguments. We should reserve for
256  // _all_ deopt and gc arguments, then start allocating. This
257  // will prevent some moves being inserted when vm state changes,
258  // but gc state doesn't between two calls.
259  return;
260  }
261  // Reserve this stack slot
262  Builder.StatepointLowering.reserveStackSlot(Offset);
263 
264  // Cache this slot so we find it when going through the normal
265  // assignment loop.
266  SDValue Loc =
267  Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
268  Builder.StatepointLowering.setLocation(Incoming, Loc);
269 }
270 
271 /// Remove any duplicate (as SDValues) from the derived pointer pairs. This
272 /// is not required for correctness. It's purpose is to reduce the size of
273 /// StackMap section. It has no effect on the number of spill slots required
274 /// or the actual lowering.
275 static void
279  SelectionDAGBuilder &Builder,
282 
283  SmallVector<const Value *, 64> NewBases, NewPtrs;
285  for (size_t i = 0, e = Ptrs.size(); i < e; i++) {
286  SDValue SD = Builder.getValue(Ptrs[i]);
287  auto SeenIt = Seen.find(SD);
288 
289  if (SeenIt == Seen.end()) {
290  // Only add non-duplicates
291  NewBases.push_back(Bases[i]);
292  NewPtrs.push_back(Ptrs[i]);
293  NewRelocs.push_back(Relocs[i]);
294  Seen[SD] = Ptrs[i];
295  } else {
296  // Duplicate pointer found, note in SSM and move on:
297  SSM.DuplicateMap[Ptrs[i]] = SeenIt->second;
298  }
299  }
300  assert(Bases.size() >= NewBases.size());
301  assert(Ptrs.size() >= NewPtrs.size());
302  assert(Relocs.size() >= NewRelocs.size());
303  Bases = NewBases;
304  Ptrs = NewPtrs;
305  Relocs = NewRelocs;
306  assert(Ptrs.size() == Bases.size());
307  assert(Ptrs.size() == Relocs.size());
308 }
309 
310 /// Extract call from statepoint, lower it and return pointer to the
311 /// call node. Also update NodeMap so that getValue(statepoint) will
312 /// reference lowered call result
313 static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
315  SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
316  SDValue ReturnValue, CallEndVal;
317  std::tie(ReturnValue, CallEndVal) =
318  Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
319  SDNode *CallEnd = CallEndVal.getNode();
320 
321  // Get a call instruction from the call sequence chain. Tail calls are not
322  // allowed. The following code is essentially reverse engineering X86's
323  // LowerCallTo.
324  //
325  // We are expecting DAG to have the following form:
326  //
327  // ch = eh_label (only in case of invoke statepoint)
328  // ch, glue = callseq_start ch
329  // ch, glue = X86::Call ch, glue
330  // ch, glue = callseq_end ch, glue
331  // get_return_value ch, glue
332  //
333  // get_return_value can either be a sequence of CopyFromReg instructions
334  // to grab the return value from the return register(s), or it can be a LOAD
335  // to load a value returned by reference via a stack slot.
336 
337  bool HasDef = !SI.CLI.RetTy->isVoidTy();
338  if (HasDef) {
339  if (CallEnd->getOpcode() == ISD::LOAD)
340  CallEnd = CallEnd->getOperand(0).getNode();
341  else
342  while (CallEnd->getOpcode() == ISD::CopyFromReg)
343  CallEnd = CallEnd->getOperand(0).getNode();
344  }
345 
346  assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
347  return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
348 }
349 
350 /// Spill a value incoming to the statepoint. It might be either part of
351 /// vmstate
352 /// or gcstate. In both cases unconditionally spill it on the stack unless it
353 /// is a null constant. Return pair with first element being frame index
354 /// containing saved value and second element with outgoing chain from the
355 /// emitted store
356 static std::pair<SDValue, SDValue>
358  SelectionDAGBuilder &Builder) {
359  SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
360 
361  // Emit new store if we didn't do it for this ptr before
362  if (!Loc.getNode()) {
363  Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
364  Builder);
365  int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
366  // We use TargetFrameIndex so that isel will not select it into LEA
367  Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
368 
369 #ifndef NDEBUG
370  // Right now we always allocate spill slots that are of the same
371  // size as the value we're about to spill (the size of spillee can
372  // vary since we spill vectors of pointers too). At some point we
373  // can consider allowing spills of smaller values to larger slots
374  // (i.e. change the '==' in the assert below to a '>=').
376  assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
377  "Bad spill: stack slot does not match!");
378 #endif
379 
380  Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
382  Builder.DAG.getMachineFunction(), Index));
383 
384  Builder.StatepointLowering.setLocation(Incoming, Loc);
385  }
386 
387  assert(Loc.getNode());
388  return std::make_pair(Loc, Chain);
389 }
390 
391 /// Lower a single value incoming to a statepoint node. This value can be
392 /// either a deopt value or a gc value, the handling is the same. We special
393 /// case constants and allocas, then fall back to spilling if required.
394 static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
396  SelectionDAGBuilder &Builder) {
397  // Note: We know all of these spills are independent, but don't bother to
398  // exploit that chain wise. DAGCombine will happily do so as needed, so
399  // doing it here would be a small compile time win at most.
400  SDValue Chain = Builder.getRoot();
401 
402  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
403  // If the original value was a constant, make sure it gets recorded as
404  // such in the stackmap. This is required so that the consumer can
405  // parse any internal format to the deopt state. It also handles null
406  // pointers and other constant pointers in GC states. Note the constant
407  // vectors do not appear to actually hit this path and that anything larger
408  // than an i64 value (not type!) will fail asserts here.
409  pushStackMapConstant(Ops, Builder, C->getSExtValue());
410  } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
411  // This handles allocas as arguments to the statepoint (this is only
412  // really meaningful for a deopt value. For GC, we'd be trying to
413  // relocate the address of the alloca itself?)
414  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
415  "Incoming value is a frame index!");
416  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
417  Builder.getFrameIndexTy()));
418  } else if (LiveInOnly) {
419  // If this value is live in (not live-on-return, or live-through), we can
420  // treat it the same way patchpoint treats it's "live in" values. We'll
421  // end up folding some of these into stack references, but they'll be
422  // handled by the register allocator. Note that we do not have the notion
423  // of a late use so these values might be placed in registers which are
424  // clobbered by the call. This is fine for live-in.
425  Ops.push_back(Incoming);
426  } else {
427  // Otherwise, locate a spill slot and explicitly spill it so it
428  // can be found by the runtime later. We currently do not support
429  // tracking values through callee saved registers to their eventual
430  // spill location. This would be a useful optimization, but would
431  // need to be optional since it requires a lot of complexity on the
432  // runtime side which not all would support.
433  auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
434  Ops.push_back(Res.first);
435  Chain = Res.second;
436  }
437 
438  Builder.DAG.setRoot(Chain);
439 }
440 
441 /// Lower deopt state and gc pointer arguments of the statepoint. The actual
442 /// lowering is described in lowerIncomingStatepointValue. This function is
443 /// responsible for lowering everything in the right position and playing some
444 /// tricks to avoid redundant stack manipulation where possible. On
445 /// completion, 'Ops' will contain ready to use operands for machine code
446 /// statepoint. The chain nodes will have already been created and the DAG root
447 /// will be set to the last value spilled (if any were).
448 static void
451  SelectionDAGBuilder &Builder) {
452  // Lower the deopt and gc arguments for this statepoint. Layout will be:
453  // deopt argument length, deopt arguments.., gc arguments...
454 #ifndef NDEBUG
455  if (auto *GFI = Builder.GFI) {
456  // Check that each of the gc pointer and bases we've gotten out of the
457  // safepoint is something the strategy thinks might be a pointer (or vector
458  // of pointers) into the GC heap. This is basically just here to help catch
459  // errors during statepoint insertion. TODO: This should actually be in the
460  // Verifier, but we can't get to the GCStrategy from there (yet).
461  GCStrategy &S = GFI->getStrategy();
462  for (const Value *V : SI.Bases) {
463  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
464  if (Opt.hasValue()) {
465  assert(Opt.getValue() &&
466  "non gc managed base pointer found in statepoint");
467  }
468  }
469  for (const Value *V : SI.Ptrs) {
470  auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
471  if (Opt.hasValue()) {
472  assert(Opt.getValue() &&
473  "non gc managed derived pointer found in statepoint");
474  }
475  }
476  assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
477  } else {
478  assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
479  assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
480  }
481 #endif
482 
483  // Figure out what lowering strategy we're going to use for each part
484  // Note: Is is conservatively correct to lower both "live-in" and "live-out"
485  // as "live-through". A "live-through" variable is one which is "live-in",
486  // "live-out", and live throughout the lifetime of the call (i.e. we can find
487  // it from any PC within the transitive callee of the statepoint). In
488  // particular, if the callee spills callee preserved registers we may not
489  // be able to find a value placed in that register during the call. This is
490  // fine for live-out, but not for live-through. If we were willing to make
491  // assumptions about the code generator producing the callee, we could
492  // potentially allow live-through values in callee saved registers.
493  const bool LiveInDeopt =
495 
496  auto isGCValue =[&](const Value *V) {
497  return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
498  };
499 
500  // Before we actually start lowering (and allocating spill slots for values),
501  // reserve any stack slots which we judge to be profitable to reuse for a
502  // particular value. This is purely an optimization over the code below and
503  // doesn't change semantics at all. It is important for performance that we
504  // reserve slots for both deopt and gc values before lowering either.
505  for (const Value *V : SI.DeoptState) {
506  if (!LiveInDeopt || isGCValue(V))
508  }
509  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
510  reservePreviousStackSlotForValue(SI.Bases[i], Builder);
511  reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
512  }
513 
514  // First, prefix the list with the number of unique values to be
515  // lowered. Note that this is the number of *Values* not the
516  // number of SDValues required to lower them.
517  const int NumVMSArgs = SI.DeoptState.size();
518  pushStackMapConstant(Ops, Builder, NumVMSArgs);
519 
520  // The vm state arguments are lowered in an opaque manner. We do not know
521  // what type of values are contained within.
522  for (const Value *V : SI.DeoptState) {
523  SDValue Incoming;
524  // If this is a function argument at a static frame index, generate it as
525  // the frame index.
526  if (const Argument *Arg = dyn_cast<Argument>(V)) {
527  int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);
528  if (FI != INT_MAX)
529  Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());
530  }
531  if (!Incoming.getNode())
532  Incoming = Builder.getValue(V);
533  const bool LiveInValue = LiveInDeopt && !isGCValue(V);
534  lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder);
535  }
536 
537  // Finally, go ahead and lower all the gc arguments. There's no prefixed
538  // length for this one. After lowering, we'll have the base and pointer
539  // arrays interwoven with each (lowered) base pointer immediately followed by
540  // it's (lowered) derived pointer. i.e
541  // (base[0], ptr[0], base[1], ptr[1], ...)
542  for (unsigned i = 0; i < SI.Bases.size(); ++i) {
543  const Value *Base = SI.Bases[i];
544  lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
545  Ops, Builder);
546 
547  const Value *Ptr = SI.Ptrs[i];
548  lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
549  Ops, Builder);
550  }
551 
552  // If there are any explicit spill slots passed to the statepoint, record
553  // them, but otherwise do not do anything special. These are user provided
554  // allocas and give control over placement to the consumer. In this case,
555  // it is the contents of the slot which may get updated, not the pointer to
556  // the alloca
557  for (Value *V : SI.GCArgs) {
558  SDValue Incoming = Builder.getValue(V);
559  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
560  // This handles allocas as arguments to the statepoint
561  assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
562  "Incoming value is a frame index!");
563  Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
564  Builder.getFrameIndexTy()));
565  }
566  }
567 
568  // Record computed locations for all lowered values.
569  // This can not be embedded in lowering loops as we need to record *all*
570  // values, while previous loops account only values with unique SDValues.
571  const Instruction *StatepointInstr = SI.StatepointInstr;
572  auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
573 
574  for (const GCRelocateInst *Relocate : SI.GCRelocates) {
575  const Value *V = Relocate->getDerivedPtr();
576  SDValue SDV = Builder.getValue(V);
577  SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
578 
579  if (Loc.getNode()) {
580  SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
581  } else {
582  // Record value as visited, but not spilled. This is case for allocas
583  // and constants. For this values we can avoid emitting spill load while
584  // visiting corresponding gc_relocate.
585  // Actually we do not need to record them in this map at all.
586  // We do this only to check that we are not relocating any unvisited
587  // value.
588  SpillMap.SlotMap[V] = None;
589 
590  // Default llvm mechanisms for exporting values which are used in
591  // different basic blocks does not work for gc relocates.
592  // Note that it would be incorrect to teach llvm that all relocates are
593  // uses of the corresponding values so that it would automatically
594  // export them. Relocates of the spilled values does not use original
595  // value.
596  if (Relocate->getParent() != StatepointInstr->getParent())
597  Builder.ExportFromCurrentBlock(V);
598  }
599  }
600 }
601 
604  // The basic scheme here is that information about both the original call and
605  // the safepoint is encoded in the CallInst. We create a temporary call and
606  // lower it, then reverse engineer the calling sequence.
607 
608  NumOfStatepoints++;
609  // Clear state
610  StatepointLowering.startNewStatepoint(*this);
611 
612 #ifndef NDEBUG
613  // We schedule gc relocates before removeDuplicateGCPtrs since we _will_
614  // encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
615  for (auto *Reloc : SI.GCRelocates)
616  if (Reloc->getParent() == SI.StatepointInstr->getParent())
617  StatepointLowering.scheduleRelocCall(*Reloc);
618 #endif
619 
620  // Remove any redundant llvm::Values which map to the same SDValue as another
621  // input. Also has the effect of removing duplicates in the original
622  // llvm::Value input list as well. This is a useful optimization for
623  // reducing the size of the StackMap section. It has no other impact.
624  removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
625  FuncInfo.StatepointSpillMaps[SI.StatepointInstr]);
626  assert(SI.Bases.size() == SI.Ptrs.size() &&
627  SI.Ptrs.size() == SI.GCRelocates.size());
628 
629  // Lower statepoint vmstate and gcstate arguments
630  SmallVector<SDValue, 10> LoweredMetaArgs;
631  lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this);
632 
633  // Now that we've emitted the spills, we need to update the root so that the
634  // call sequence is ordered correctly.
635  SI.CLI.setChain(getRoot());
636 
637  // Get call node, we will replace it later with statepoint
638  SDValue ReturnVal;
639  SDNode *CallNode;
640  std::tie(ReturnVal, CallNode) =
641  lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
642 
643  // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
644  // nodes with all the appropriate arguments and return values.
645 
646  // Call Node: Chain, Target, {Args}, RegMask, [Glue]
647  SDValue Chain = CallNode->getOperand(0);
648 
649  SDValue Glue;
650  bool CallHasIncomingGlue = CallNode->getGluedNode();
651  if (CallHasIncomingGlue) {
652  // Glue is always last operand
653  Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
654  }
655 
656  // Build the GC_TRANSITION_START node if necessary.
657  //
658  // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
659  // order in which they appear in the call to the statepoint intrinsic. If
660  // any of the operands is a pointer-typed, that operand is immediately
661  // followed by a SRCVALUE for the pointer that may be used during lowering
662  // (e.g. to form MachinePointerInfo values for loads/stores).
663  const bool IsGCTransition =
666  if (IsGCTransition) {
668 
669  // Add chain
670  TSOps.push_back(Chain);
671 
672  // Add GC transition arguments
673  for (const Value *V : SI.GCTransitionArgs) {
674  TSOps.push_back(getValue(V));
675  if (V->getType()->isPointerTy())
676  TSOps.push_back(DAG.getSrcValue(V));
677  }
678 
679  // Add glue if necessary
680  if (CallHasIncomingGlue)
681  TSOps.push_back(Glue);
682 
683  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
684 
685  SDValue GCTransitionStart =
686  DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
687 
688  Chain = GCTransitionStart.getValue(0);
689  Glue = GCTransitionStart.getValue(1);
690  }
691 
692  // TODO: Currently, all of these operands are being marked as read/write in
693  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
694  // and flags to be read-only.
696 
697  // Add the <id> and <numBytes> constants.
698  Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
699  Ops.push_back(
700  DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
701 
702  // Calculate and push starting position of vmstate arguments
703  // Get number of arguments incoming directly into call node
704  unsigned NumCallRegArgs =
705  CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
706  Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
707 
708  // Add call target
709  SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
710  Ops.push_back(CallTarget);
711 
712  // Add call arguments
713  // Get position of register mask in the call
714  SDNode::op_iterator RegMaskIt;
715  if (CallHasIncomingGlue)
716  RegMaskIt = CallNode->op_end() - 2;
717  else
718  RegMaskIt = CallNode->op_end() - 1;
719  Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
720 
721  // Add a constant argument for the calling convention
722  pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
723 
724  // Add a constant argument for the flags
725  uint64_t Flags = SI.StatepointFlags;
726  assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
727  "Unknown flag used");
728  pushStackMapConstant(Ops, *this, Flags);
729 
730  // Insert all vmstate and gcstate arguments
731  Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
732 
733  // Add register mask from call node
734  Ops.push_back(*RegMaskIt);
735 
736  // Add chain
737  Ops.push_back(Chain);
738 
739  // Same for the glue, but we add it only if original call had it
740  if (Glue.getNode())
741  Ops.push_back(Glue);
742 
743  // Compute return values. Provide a glue output since we consume one as
744  // input. This allows someone else to chain off us as needed.
745  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
746 
747  SDNode *StatepointMCNode =
748  DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
749 
750  SDNode *SinkNode = StatepointMCNode;
751 
752  // Build the GC_TRANSITION_END node if necessary.
753  //
754  // See the comment above regarding GC_TRANSITION_START for the layout of
755  // the operands to the GC_TRANSITION_END node.
756  if (IsGCTransition) {
758 
759  // Add chain
760  TEOps.push_back(SDValue(StatepointMCNode, 0));
761 
762  // Add GC transition arguments
763  for (const Value *V : SI.GCTransitionArgs) {
764  TEOps.push_back(getValue(V));
765  if (V->getType()->isPointerTy())
766  TEOps.push_back(DAG.getSrcValue(V));
767  }
768 
769  // Add glue
770  TEOps.push_back(SDValue(StatepointMCNode, 1));
771 
772  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
773 
774  SDValue GCTransitionStart =
775  DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
776 
777  SinkNode = GCTransitionStart.getNode();
778  }
779 
780  // Replace original call
781  DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
782  // Remove original call node
783  DAG.DeleteNode(CallNode);
784 
785  // DON'T set the root - under the assumption that it's already set past the
786  // inserted node we created.
787 
788  // TODO: A better future implementation would be to emit a single variable
789  // argument, variable return value STATEPOINT node here and then hookup the
790  // return value of each gc.relocate to the respective output of the
791  // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
792  // to actually be possible today.
793 
794  return ReturnVal;
795 }
796 
797 void
799  const BasicBlock *EHPadBB /*= nullptr*/) {
801  "anyregcc is not supported on statepoints!");
802 
803 #ifndef NDEBUG
804  // If this is a malformed statepoint, report it early to simplify debugging.
805  // This should catch any IR level mistake that's made when constructing or
806  // transforming statepoints.
807  ISP.verify();
808 
809  // Check that the associated GCStrategy expects to encounter statepoints.
810  assert(GFI->getStrategy().useStatepoints() &&
811  "GCStrategy does not expect to encounter statepoints");
812 #endif
813 
814  SDValue ActualCallee;
815 
816  if (ISP.getNumPatchBytes() > 0) {
817  // If we've been asked to emit a nop sequence instead of a call instruction
818  // for this statepoint then don't lower the call target, but use a constant
819  // `null` instead. Not lowering the call target lets statepoint clients get
820  // away without providing a physical address for the symbolic call target at
821  // link time.
822 
823  const auto &TLI = DAG.getTargetLoweringInfo();
824  const auto &DL = DAG.getDataLayout();
825 
826  unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
827  ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
828  } else {
829  ActualCallee = getValue(ISP.getCalledValue());
830  }
831 
833  populateCallLoweringInfo(SI.CLI, ISP.getCall(),
835  ISP.getNumCallArgs(), ActualCallee,
836  ISP.getActualReturnType(), false /* IsPatchPoint */);
837 
838  for (const GCRelocateInst *Relocate : ISP.getRelocates()) {
839  SI.GCRelocates.push_back(Relocate);
840  SI.Bases.push_back(Relocate->getBasePtr());
841  SI.Ptrs.push_back(Relocate->getDerivedPtr());
842  }
843 
844  SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
845  SI.StatepointInstr = ISP.getInstruction();
846  SI.GCTransitionArgs =
848  SI.ID = ISP.getID();
849  SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end());
850  SI.StatepointFlags = ISP.getFlags();
851  SI.NumPatchBytes = ISP.getNumPatchBytes();
852  SI.EHPadBB = EHPadBB;
853 
854  SDValue ReturnValue = LowerAsSTATEPOINT(SI);
855 
856  // Export the result value if needed
857  const GCResultInst *GCResult = ISP.getGCResult();
858  Type *RetTy = ISP.getActualReturnType();
859  if (!RetTy->isVoidTy() && GCResult) {
860  if (GCResult->getParent() != ISP.getCall()->getParent()) {
861  // Result value will be used in a different basic block so we need to
862  // export it now. Default exporting mechanism will not work here because
863  // statepoint call has a different type than the actual call. It means
864  // that by default llvm will create export register of the wrong type
865  // (always i32 in our case). So instead we need to create export register
866  // with correct type manually.
867  // TODO: To eliminate this problem we can remove gc.result intrinsics
868  // completely and make statepoint call to return a tuple.
869  unsigned Reg = FuncInfo.CreateRegs(RetTy);
870  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
871  DAG.getDataLayout(), Reg, RetTy,
872  ISP.getCall()->getCallingConv());
873  SDValue Chain = DAG.getEntryNode();
874 
875  RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
876  PendingExports.push_back(Chain);
877  FuncInfo.ValueMap[ISP.getInstruction()] = Reg;
878  } else {
879  // Result value will be used in a same basic block. Don't export it or
880  // perform any explicit register copies.
881  // We'll replace the actuall call node shortly. gc_result will grab
882  // this value.
883  setValue(ISP.getInstruction(), ReturnValue);
884  }
885  } else {
886  // The token value is never used from here on, just generate a poison value
887  setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
888  }
889 }
890 
892  const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
893  bool VarArgDisallowed, bool ForceVoidReturnTy) {
895  unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
896  populateCallLoweringInfo(
897  SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee,
898  ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
899  false);
900  if (!VarArgDisallowed)
901  SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
902 
903  auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
904 
906 
908  SI.ID = SD.StatepointID.getValueOr(DefaultID);
909  SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
910 
911  SI.DeoptState =
912  ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
913  SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
914  SI.EHPadBB = EHPadBB;
915 
916  // NB! The GC arguments are deliberately left empty.
917 
918  if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
919  ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
920  setValue(Call, ReturnVal);
921  }
922 }
923 
925  const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
926  LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
927  /* VarArgDisallowed = */ false,
928  /* ForceVoidReturnTy = */ false);
929 }
930 
931 void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
932  // The result value of the gc_result is simply the result of the actual
933  // call. We've already emitted this, so just grab the value.
934  const Instruction *I = CI.getStatepoint();
935 
936  if (I->getParent() != CI.getParent()) {
937  // Statepoint is in different basic block so we should have stored call
938  // result in a virtual register.
939  // We can not use default getValue() functionality to copy value from this
940  // register because statepoint and actual call return types can be
941  // different, and getValue() will use CopyFromReg of the wrong type,
942  // which is always i32 in our case.
943  PointerType *CalleeType = cast<PointerType>(
945  Type *RetTy =
946  cast<FunctionType>(CalleeType->getElementType())->getReturnType();
947  SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
948 
949  assert(CopyFromReg.getNode());
950  setValue(&CI, CopyFromReg);
951  } else {
952  setValue(&CI, getValue(I));
953  }
954 }
955 
956 void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
957 #ifndef NDEBUG
958  // Consistency check
959  // We skip this check for relocates not in the same basic block as their
960  // statepoint. It would be too expensive to preserve validation info through
961  // different basic blocks.
962  if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
963  StatepointLowering.relocCallVisited(Relocate);
964 
965  auto *Ty = Relocate.getType()->getScalarType();
966  if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
967  assert(*IsManaged && "Non gc managed pointer relocated!");
968 #endif
969 
970  const Value *DerivedPtr = Relocate.getDerivedPtr();
971  SDValue SD = getValue(DerivedPtr);
972 
973  auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
974  auto SlotIt = SpillMap.find(DerivedPtr);
975  assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
976  Optional<int> DerivedPtrLocation = SlotIt->second;
977 
978  // We didn't need to spill these special cases (constants and allocas).
979  // See the handling in spillIncomingValueForStatepoint for detail.
980  if (!DerivedPtrLocation) {
981  setValue(&Relocate, SD);
982  return;
983  }
984 
985  SDValue SpillSlot =
986  DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
987 
988  // Note: We know all of these reloads are independent, but don't bother to
989  // exploit that chain wise. DAGCombine will happily do so as needed, so
990  // doing it here would be a small compile time win at most.
991  SDValue Chain = getRoot();
992 
993  SDValue SpillLoad =
994  DAG.getLoad(DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
995  Relocate.getType()),
996  getCurSDLoc(), Chain, SpillSlot,
997  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
998  *DerivedPtrLocation));
999 
1000  DAG.setRoot(SpillLoad.getValue(1));
1001 
1002  assert(SpillLoad.getNode());
1003  setValue(&Relocate, SpillLoad);
1004 }
1005 
1007  const auto &TLI = DAG.getTargetLoweringInfo();
1008  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1009  TLI.getPointerTy(DAG.getDataLayout()));
1010 
1011  // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1012  // call. We also do not lower the return value to any virtual register, and
1013  // change the immediately following return to a trap instruction.
1014  LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1015  /* VarArgDisallowed = */ true,
1016  /* ForceVoidReturnTy = */ true);
1017 }
1018 
1020  // We do not lower the return value from llvm.deoptimize to any virtual
1021  // register, and change the immediately following return to a trap
1022  // instruction.
1023  if (DAG.getTarget().Options.TrapUnreachable)
1024  DAG.setRoot(
1025  DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
1026 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
uint64_t CallInst * C
Represents calls to the gc.result intrinsic.
Definition: Statepoint.h:396
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
This class represents lattice values for constants.
Definition: AllocatorList.h:23
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn&#39;t known to be exported from the current basic block...
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
void LowerDeoptimizeCall(const CallInst *CI)
TargetLowering::CallLoweringInfo CLI
Information regarding the underlying call instruction.
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:60
SmallVector< const GCRelocateInst *, 16 > GCRelocates
The set of gc.relocate calls associated with this gc.statepoint.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it&#39;s base class for read only access to a gc.statepoint.
Definition: Statepoint.h:295
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
Type * getActualReturnType() const
Return the type of the value returned by the call underlying the statepoint.
Definition: Statepoint.h:160
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
STATISTIC(NumFunctions, "Total number of functions")
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1014
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:534
SDNode * getNode() const
get the SDNode which holds the desired result
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:465
CallBaseTy * getCall() const
Return the underlying call instruction.
Definition: Statepoint.h:110
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
op_iterator op_begin()
Definition: User.h:229
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling...
Keep track of frame indices allocated for statepoints as they could be used across basic block bounda...
const CallBase * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Definition: Statepoint.h:339
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool test(unsigned Idx) const
void verify()
Asserts if this statepoint is malformed.
Definition: Statepoint.h:276
unsigned ID
The ID that the resulting STATEPOINT instruction has to report.
SDValue getRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
InstructionTy * getInstruction() const
Definition: Statepoint.h:141
uint64_t StatepointFlags
Flags associated with the meta arguments being lowered.
Value * getDerivedPtr() const
Definition: Statepoint.h:390
ArrayRef< const Use > GCTransitionArgs
The list of gc transition arguments present in the gc.statepoint being lowered.
CallLoweringInfo & setChain(SDValue InChain)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
op_iterator op_end() const
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:303
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static void removeDuplicateGCPtrs(SmallVectorImpl< const Value *> &Bases, SmallVectorImpl< const Value *> &Ptrs, SmallVectorImpl< const GCRelocateInst *> &Relocs, SelectionDAGBuilder &Builder, FunctionLoweringInfo::StatepointSpillMap &SSM)
Remove any duplicate (as SDValues) from the derived pointer pairs.
const GCResultInst * getGCResult() const
Get the experimental_gc_result call tied to this statepoint.
Definition: Statepoint.h:265
arg_iterator gc_args_begin() const
Definition: Statepoint.h:244
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)
Lower SLI into a STATEPOINT instruction.
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:397
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:627
virtual Optional< bool > isGCManagedPointer(const Type *Ty) const
If the type specified can be reliably distinguished, returns true for pointers to GC managed location...
Definition: GCStrategy.h:99
bool isVarArg() const
Definition: DerivedTypes.h:122
This class represents a no-op cast from one type to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:575
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Class to represent pointers.
Definition: DerivedTypes.h:498
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:1716
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:445
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return &#39;this&#39;.
Definition: Type.h:303
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
uint64_t getID() const
Return the ID associated with this statepoint.
Definition: Statepoint.h:121
arg_iterator deopt_end() const
Definition: Statepoint.h:233
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
const Instruction * StatepointInstr
The gc.statepoint instruction.
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:858
ArrayRef< const Use > DeoptState
The deoptimization state associated with this gc.statepoint call, if any.
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
static std::pair< SDValue, SDValue > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
const SDValue & getOperand(unsigned Num) const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:223
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1058
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:160
void clear()
Clear the memory usage of this object.
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:771
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
int getNumCallArgs() const
Number of arguments to be passed to the actual callee.
Definition: Statepoint.h:167
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
DenseMap< const Instruction *, StatepointSpillMap > StatepointSpillMaps
Maps gc.statepoint instructions to their corresponding StatepointSpillMap instances.
Extended Value Type.
Definition: ValueTypes.h:33
size_t size() const
Definition: SmallVector.h:52
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1206
unsigned getNumOperands() const
Return the number of values used by this operation.
void setLocation(SDValue Val, SDValue Location)
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
SmallBitVector & set()
void clear()
Clear all bits.
arg_iterator gc_args_end() const
Definition: Statepoint.h:245
static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:839
DenseMap< const Value *, const Value * > DuplicateMap
Maps llvm IR values to the values they were de-duplicated to.
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
size_t size() const
Returns the number of bits in this bitvector.
Represents one node in the SelectionDAG.
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1280
GCFunctionInfo * GFI
Garbage collection metadata for the function.
amdgpu Simplify well known AMD library false FunctionCallee Callee
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:470
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1120
bool hasValue() const
Definition: Optional.h:259
Represents a use of a SDNode.
void LowerStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB=nullptr)
GCStrategy describes a garbage collector algorithm&#39;s code generation requirements, and provides overridable hooks for those needs which cannot be abstractly described.
Definition: GCStrategy.h:66
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1153
uint32_t getNumPatchBytes() const
Return the number of patchable bytes associated with this statepoint.
Definition: Statepoint.h:127
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1258
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:615
#define I(x, y, z)
Definition: MD5.cpp:58
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
SmallVector< const Value *, 16 > Bases
Bases[i] is the base pointer for Ptrs[i].
SDValue getValue(unsigned R) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:174
std::vector< const GCRelocateInst * > getRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition: Statepoint.h:410
unsigned NumPatchBytes
The number of patchable bytes the call needs to get lowered into.
uint64_t getFlags() const
Definition: Statepoint.h:115
ValueTy * getCalledValue() const
Return the value actually being called or invoked.
Definition: Statepoint.h:136
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const BasicBlock * EHPadBB
The exception handling unwind destination, in case this represents an invoke of gc.statepoint.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Represents calls to the gc.relocate intrinsic.
Definition: Statepoint.h:363
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
Definition: Value.h:72
ArrayRef< const Use > GCArgs
The full list of gc arguments to the gc.statepoint being lowered.
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
A bitmask that includes all valid flags.
Type * getElementType() const
Definition: DerivedTypes.h:517
arg_iterator deopt_begin() const
Definition: Statepoint.h:228
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
A discriminated union of two pointer types, with the discriminator in the low bit of the pointer...
Definition: PointerUnion.h:86
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1244