LLVM 19.0.0git
StatepointLowering.cpp
Go to the documentation of this file.
1//===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file includes support code use by SelectionDAGBuilder when lowering a
10// statepoint sequence in SelectionDAG IR.
11//
12//===----------------------------------------------------------------------===//
13
14#include "StatepointLowering.h"
15#include "SelectionDAGBuilder.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SetVector.h"
20#include "llvm/ADT/SmallSet.h"
22#include "llvm/ADT/Statistic.h"
36#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/GCStrategy.h"
39#include "llvm/IR/Instruction.h"
41#include "llvm/IR/LLVMContext.h"
42#include "llvm/IR/Statepoint.h"
43#include "llvm/IR/Type.h"
48#include <cassert>
49#include <cstddef>
50#include <cstdint>
51#include <iterator>
52#include <tuple>
53#include <utility>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "statepoint-lowering"
58
59STATISTIC(NumSlotsAllocatedForStatepoints,
60 "Number of stack slots allocated for statepoints");
61STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
62STATISTIC(StatepointMaxSlotsRequired,
63 "Maximum number of stack slots required for a singe statepoint");
64
66 "use-registers-for-deopt-values", cl::Hidden, cl::init(false),
67 cl::desc("Allow using registers for non pointer deopt args"));
68
70 "use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false),
71 cl::desc("Allow using registers for gc pointer in landing pad"));
72
74 "max-registers-for-gc-values", cl::Hidden, cl::init(0),
75 cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"));
76
78
81 SDLoc L = Builder.getCurSDLoc();
82 Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
83 MVT::i64));
84 Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
85}
86
88 // Consistency check
89 assert(PendingGCRelocateCalls.empty() &&
90 "Trying to visit statepoint before finished processing previous one");
91 Locations.clear();
92 NextSlotToAllocate = 0;
93 // Need to resize this on each safepoint - we need the two to stay in sync and
94 // the clear patterns of a SelectionDAGBuilder have no relation to
95 // FunctionLoweringInfo. Also need to ensure used bits get cleared.
96 AllocatedStackSlots.clear();
97 AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
98}
99
101 Locations.clear();
102 AllocatedStackSlots.clear();
103 assert(PendingGCRelocateCalls.empty() &&
104 "cleared before statepoint sequence completed");
105}
106
109 SelectionDAGBuilder &Builder) {
110 NumSlotsAllocatedForStatepoints++;
112
113 unsigned SpillSize = ValueType.getStoreSize();
114 assert((SpillSize * 8) ==
115 (-8u & (7 + ValueType.getSizeInBits())) && // Round up modulo 8.
116 "Size not in bytes?");
117
118 // First look for a previously created stack slot which is not in
119 // use (accounting for the fact arbitrary slots may already be
120 // reserved), or to create a new stack slot and use it.
121
122 const size_t NumSlots = AllocatedStackSlots.size();
123 assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
124
125 assert(AllocatedStackSlots.size() ==
127 "Broken invariant");
128
129 for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
130 if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
131 const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
132 if (MFI.getObjectSize(FI) == SpillSize) {
133 AllocatedStackSlots.set(NextSlotToAllocate);
134 // TODO: Is ValueType the right thing to use here?
135 return Builder.DAG.getFrameIndex(FI, ValueType);
136 }
137 }
138 }
139
140 // Couldn't find a free slot, so create a new one:
141
142 SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
143 const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
145
147 AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
148 assert(AllocatedStackSlots.size() ==
150 "Broken invariant");
151
152 StatepointMaxSlotsRequired.updateMax(
154
155 return SpillSlot;
156}
157
158/// Utility function for reservePreviousStackSlotForValue. Tries to find
159/// stack slot index to which we have spilled value for previous statepoints.
160/// LookUpDepth specifies maximum DFS depth this function is allowed to look.
161static std::optional<int> findPreviousSpillSlot(const Value *Val,
162 SelectionDAGBuilder &Builder,
163 int LookUpDepth) {
164 // Can not look any further - give up now
165 if (LookUpDepth <= 0)
166 return std::nullopt;
167
168 // Spill location is known for gc relocates
169 if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
170 const Value *Statepoint = Relocate->getStatepoint();
171 assert((isa<GCStatepointInst>(Statepoint) || isa<UndefValue>(Statepoint)) &&
172 "GetStatepoint must return one of two types");
173 if (isa<UndefValue>(Statepoint))
174 return std::nullopt;
175
176 const auto &RelocationMap = Builder.FuncInfo.StatepointRelocationMaps
177 [cast<GCStatepointInst>(Statepoint)];
178
179 auto It = RelocationMap.find(Relocate);
180 if (It == RelocationMap.end())
181 return std::nullopt;
182
183 auto &Record = It->second;
184 if (Record.type != RecordType::Spill)
185 return std::nullopt;
186
187 return Record.payload.FI;
188 }
189
190 // Look through bitcast instructions.
191 if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
192 return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
193
194 // Look through phi nodes
195 // All incoming values should have same known stack slot, otherwise result
196 // is unknown.
197 if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
198 std::optional<int> MergedResult;
199
200 for (const auto &IncomingValue : Phi->incoming_values()) {
201 std::optional<int> SpillSlot =
202 findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
203 if (!SpillSlot)
204 return std::nullopt;
205
206 if (MergedResult && *MergedResult != *SpillSlot)
207 return std::nullopt;
208
209 MergedResult = SpillSlot;
210 }
211 return MergedResult;
212 }
213
214 // TODO: We can do better for PHI nodes. In cases like this:
215 // ptr = phi(relocated_pointer, not_relocated_pointer)
216 // statepoint(ptr)
217 // We will return that stack slot for ptr is unknown. And later we might
218 // assign different stack slots for ptr and relocated_pointer. This limits
219 // llvm's ability to remove redundant stores.
220 // Unfortunately it's hard to accomplish in current infrastructure.
221 // We use this function to eliminate spill store completely, while
222 // in example we still need to emit store, but instead of any location
223 // we need to use special "preferred" location.
224
225 // TODO: handle simple updates. If a value is modified and the original
226 // value is no longer live, it would be nice to put the modified value in the
227 // same slot. This allows folding of the memory accesses for some
228 // instructions types (like an increment).
229 // statepoint (i)
230 // i1 = i+1
231 // statepoint (i1)
232 // However we need to be careful for cases like this:
233 // statepoint(i)
234 // i1 = i+1
235 // statepoint(i, i1)
236 // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
237 // put handling of simple modifications in this function like it's done
238 // for bitcasts we might end up reserving i's slot for 'i+1' because order in
239 // which we visit values is unspecified.
240
241 // Don't know any information about this instruction
242 return std::nullopt;
243}
244
245/// Return true if-and-only-if the given SDValue can be lowered as either a
246/// constant argument or a stack reference. The key point is that the value
247/// doesn't need to be spilled or tracked as a vreg use.
249 // We are making an unchecked assumption that the frame size <= 2^16 as that
250 // is the largest offset which can be encoded in the stackmap format.
251 if (isa<FrameIndexSDNode>(Incoming))
252 return true;
253
254 // The largest constant describeable in the StackMap format is 64 bits.
255 // Potential Optimization: Constants values are sign extended by consumer,
256 // and thus there are many constants of static type > 64 bits whose value
257 // happens to be sext(Con64) and could thus be lowered directly.
258 if (Incoming.getValueType().getSizeInBits() > 64)
259 return false;
260
261 return isIntOrFPConstant(Incoming) || Incoming.isUndef();
262}
263
264/// Try to find existing copies of the incoming values in stack slots used for
265/// statepoint spilling. If we can find a spill slot for the incoming value,
266/// mark that slot as allocated, and reuse the same slot for this safepoint.
267/// This helps to avoid series of loads and stores that only serve to reshuffle
268/// values on the stack between calls.
269static void reservePreviousStackSlotForValue(const Value *IncomingValue,
270 SelectionDAGBuilder &Builder) {
271 SDValue Incoming = Builder.getValue(IncomingValue);
272
273 // If we won't spill this, we don't need to check for previously allocated
274 // stack slots.
276 return;
277
278 SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
279 if (OldLocation.getNode())
280 // Duplicates in input
281 return;
282
283 const int LookUpDepth = 6;
284 std::optional<int> Index =
285 findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
286 if (!Index)
287 return;
288
289 const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
290
291 auto SlotIt = find(StatepointSlots, *Index);
292 assert(SlotIt != StatepointSlots.end() &&
293 "Value spilled to the unknown stack slot");
294
295 // This is one of our dedicated lowering slots
296 const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
298 // stack slot already assigned to someone else, can't use it!
299 // TODO: currently we reserve space for gc arguments after doing
300 // normal allocation for deopt arguments. We should reserve for
301 // _all_ deopt and gc arguments, then start allocating. This
302 // will prevent some moves being inserted when vm state changes,
303 // but gc state doesn't between two calls.
304 return;
305 }
306 // Reserve this stack slot
308
309 // Cache this slot so we find it when going through the normal
310 // assignment loop.
311 SDValue Loc =
312 Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
314}
315
316/// Extract call from statepoint, lower it and return pointer to the
317/// call node. Also update NodeMap so that getValue(statepoint) will
318/// reference lowered call result
319static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
321 SelectionDAGBuilder &Builder) {
322 SDValue ReturnValue, CallEndVal;
323 std::tie(ReturnValue, CallEndVal) =
324 Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
325 SDNode *CallEnd = CallEndVal.getNode();
326
327 // Get a call instruction from the call sequence chain. Tail calls are not
328 // allowed. The following code is essentially reverse engineering X86's
329 // LowerCallTo.
330 //
331 // We are expecting DAG to have the following form:
332 //
333 // ch = eh_label (only in case of invoke statepoint)
334 // ch, glue = callseq_start ch
335 // ch, glue = X86::Call ch, glue
336 // ch, glue = callseq_end ch, glue
337 // get_return_value ch, glue
338 //
339 // get_return_value can either be a sequence of CopyFromReg instructions
340 // to grab the return value from the return register(s), or it can be a LOAD
341 // to load a value returned by reference via a stack slot.
342
343 bool HasDef = !SI.CLI.RetTy->isVoidTy();
344 if (HasDef) {
345 if (CallEnd->getOpcode() == ISD::LOAD)
346 CallEnd = CallEnd->getOperand(0).getNode();
347 else
348 while (CallEnd->getOpcode() == ISD::CopyFromReg)
349 CallEnd = CallEnd->getOperand(0).getNode();
350 }
351
352 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
353 return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
354}
355
357 FrameIndexSDNode &FI) {
358 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI.getIndex());
359 auto MMOFlags = MachineMemOperand::MOStore |
361 auto &MFI = MF.getFrameInfo();
362 return MF.getMachineMemOperand(PtrInfo, MMOFlags,
363 MFI.getObjectSize(FI.getIndex()),
364 MFI.getObjectAlign(FI.getIndex()));
365}
366
367/// Spill a value incoming to the statepoint. It might be either part of
368/// vmstate
369/// or gcstate. In both cases unconditionally spill it on the stack unless it
370/// is a null constant. Return pair with first element being frame index
371/// containing saved value and second element with outgoing chain from the
372/// emitted store
373static std::tuple<SDValue, SDValue, MachineMemOperand*>
375 SelectionDAGBuilder &Builder) {
377 MachineMemOperand* MMO = nullptr;
378
379 // Emit new store if we didn't do it for this ptr before
380 if (!Loc.getNode()) {
381 Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
382 Builder);
383 int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
384 // We use TargetFrameIndex so that isel will not select it into LEA
385 Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
386
387 // Right now we always allocate spill slots that are of the same
388 // size as the value we're about to spill (the size of spillee can
389 // vary since we spill vectors of pointers too). At some point we
390 // can consider allowing spills of smaller values to larger slots
391 // (i.e. change the '==' in the assert below to a '>=').
393 assert((MFI.getObjectSize(Index) * 8) ==
394 (-8 & (7 + // Round up modulo 8.
395 (int64_t)Incoming.getValueSizeInBits())) &&
396 "Bad spill: stack slot does not match!");
397
398 // Note: Using the alignment of the spill slot (rather than the abi or
399 // preferred alignment) is required for correctness when dealing with spill
400 // slots with preferred alignments larger than frame alignment..
401 auto &MF = Builder.DAG.getMachineFunction();
402 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
403 auto *StoreMMO = MF.getMachineMemOperand(
405 MFI.getObjectAlign(Index));
406 Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
407 StoreMMO);
408
409 MMO = getMachineMemOperand(MF, *cast<FrameIndexSDNode>(Loc));
410
412 }
413
414 assert(Loc.getNode());
415 return std::make_tuple(Loc, Chain, MMO);
416}
417
418/// Lower a single value incoming to a statepoint node. This value can be
419/// either a deopt value or a gc value, the handling is the same. We special
420/// case constants and allocas, then fall back to spilling if required.
421static void
425 SelectionDAGBuilder &Builder) {
426
428 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
429 // This handles allocas as arguments to the statepoint (this is only
430 // really meaningful for a deopt value. For GC, we'd be trying to
431 // relocate the address of the alloca itself?)
432 assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
433 "Incoming value is a frame index!");
434 Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
435 Builder.getFrameIndexTy()));
436
437 auto &MF = Builder.DAG.getMachineFunction();
438 auto *MMO = getMachineMemOperand(MF, *FI);
439 MemRefs.push_back(MMO);
440 return;
441 }
442
443 assert(Incoming.getValueType().getSizeInBits() <= 64);
444
445 if (Incoming.isUndef()) {
446 // Put an easily recognized constant that's unlikely to be a valid
447 // value so that uses of undef by the consumer of the stackmap is
448 // easily recognized. This is legal since the compiler is always
449 // allowed to chose an arbitrary value for undef.
450 pushStackMapConstant(Ops, Builder, 0xFEFEFEFE);
451 return;
452 }
453
454 // If the original value was a constant, make sure it gets recorded as
455 // such in the stackmap. This is required so that the consumer can
456 // parse any internal format to the deopt state. It also handles null
457 // pointers and other constant pointers in GC states.
458 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
459 pushStackMapConstant(Ops, Builder, C->getSExtValue());
460 return;
461 } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Incoming)) {
462 pushStackMapConstant(Ops, Builder,
463 C->getValueAPF().bitcastToAPInt().getZExtValue());
464 return;
465 }
466
467 llvm_unreachable("unhandled direct lowering case");
468 }
469
470
471
472 if (!RequireSpillSlot) {
473 // If this value is live in (not live-on-return, or live-through), we can
474 // treat it the same way patchpoint treats it's "live in" values. We'll
475 // end up folding some of these into stack references, but they'll be
476 // handled by the register allocator. Note that we do not have the notion
477 // of a late use so these values might be placed in registers which are
478 // clobbered by the call. This is fine for live-in. For live-through
479 // fix-up pass should be executed to force spilling of such registers.
480 Ops.push_back(Incoming);
481 } else {
482 // Otherwise, locate a spill slot and explicitly spill it so it can be
483 // found by the runtime later. Note: We know all of these spills are
484 // independent, but don't bother to exploit that chain wise. DAGCombine
485 // will happily do so as needed, so doing it here would be a small compile
486 // time win at most.
487 SDValue Chain = Builder.getRoot();
488 auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
489 Ops.push_back(std::get<0>(Res));
490 if (auto *MMO = std::get<2>(Res))
491 MemRefs.push_back(MMO);
492 Chain = std::get<1>(Res);
493 Builder.DAG.setRoot(Chain);
494 }
495
496}
497
498/// Return true if value V represents the GC value. The behavior is conservative
499/// in case it is not sure that value is not GC the function returns true.
500static bool isGCValue(const Value *V, SelectionDAGBuilder &Builder) {
501 auto *Ty = V->getType();
502 if (!Ty->isPtrOrPtrVectorTy())
503 return false;
504 if (auto *GFI = Builder.GFI)
505 if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
506 return *IsManaged;
507 return true; // conservative
508}
509
510/// Lower deopt state and gc pointer arguments of the statepoint. The actual
511/// lowering is described in lowerIncomingStatepointValue. This function is
512/// responsible for lowering everything in the right position and playing some
513/// tricks to avoid redundant stack manipulation where possible. On
514/// completion, 'Ops' will contain ready to use operands for machine code
515/// statepoint. The chain nodes will have already been created and the DAG root
516/// will be set to the last value spilled (if any were).
517static void
521 DenseMap<SDValue, int> &LowerAsVReg,
523 SelectionDAGBuilder &Builder) {
524 // Lower the deopt and gc arguments for this statepoint. Layout will be:
525 // deopt argument length, deopt arguments.., gc arguments...
526
527 // Figure out what lowering strategy we're going to use for each part
528 // Note: It is conservatively correct to lower both "live-in" and "live-out"
529 // as "live-through". A "live-through" variable is one which is "live-in",
530 // "live-out", and live throughout the lifetime of the call (i.e. we can find
531 // it from any PC within the transitive callee of the statepoint). In
532 // particular, if the callee spills callee preserved registers we may not
533 // be able to find a value placed in that register during the call. This is
534 // fine for live-out, but not for live-through. If we were willing to make
535 // assumptions about the code generator producing the callee, we could
536 // potentially allow live-through values in callee saved registers.
537 const bool LiveInDeopt =
538 SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
539
540 // Decide which deriver pointers will go on VRegs
541 unsigned MaxVRegPtrs = MaxRegistersForGCPointers.getValue();
542
543 // Pointers used on exceptional path of invoke statepoint.
544 // We cannot assing them to VRegs.
545 SmallSet<SDValue, 8> LPadPointers;
547 if (const auto *StInvoke =
548 dyn_cast_or_null<InvokeInst>(SI.StatepointInstr)) {
549 LandingPadInst *LPI = StInvoke->getLandingPadInst();
550 for (const auto *Relocate : SI.GCRelocates)
551 if (Relocate->getOperand(0) == LPI) {
552 LPadPointers.insert(Builder.getValue(Relocate->getBasePtr()));
553 LPadPointers.insert(Builder.getValue(Relocate->getDerivedPtr()));
554 }
555 }
556
557 LLVM_DEBUG(dbgs() << "Deciding how to lower GC Pointers:\n");
558
559 // List of unique lowered GC Pointer values.
560 SmallSetVector<SDValue, 16> LoweredGCPtrs;
561 // Map lowered GC Pointer value to the index in above vector
562 DenseMap<SDValue, unsigned> GCPtrIndexMap;
563
564 unsigned CurNumVRegs = 0;
565
566 auto canPassGCPtrOnVReg = [&](SDValue SD) {
567 if (SD.getValueType().isVector())
568 return false;
569 if (LPadPointers.count(SD))
570 return false;
571 return !willLowerDirectly(SD);
572 };
573
574 auto processGCPtr = [&](const Value *V) {
575 SDValue PtrSD = Builder.getValue(V);
576 if (!LoweredGCPtrs.insert(PtrSD))
577 return; // skip duplicates
578 GCPtrIndexMap[PtrSD] = LoweredGCPtrs.size() - 1;
579
580 assert(!LowerAsVReg.count(PtrSD) && "must not have been seen");
581 if (LowerAsVReg.size() == MaxVRegPtrs)
582 return;
583 assert(V->getType()->isVectorTy() == PtrSD.getValueType().isVector() &&
584 "IR and SD types disagree");
585 if (!canPassGCPtrOnVReg(PtrSD)) {
586 LLVM_DEBUG(dbgs() << "direct/spill "; PtrSD.dump(&Builder.DAG));
587 return;
588 }
589 LLVM_DEBUG(dbgs() << "vreg "; PtrSD.dump(&Builder.DAG));
590 LowerAsVReg[PtrSD] = CurNumVRegs++;
591 };
592
593 // Process derived pointers first to give them more chance to go on VReg.
594 for (const Value *V : SI.Ptrs)
595 processGCPtr(V);
596 for (const Value *V : SI.Bases)
597 processGCPtr(V);
598
599 LLVM_DEBUG(dbgs() << LowerAsVReg.size() << " pointers will go in vregs\n");
600
601 auto requireSpillSlot = [&](const Value *V) {
603 Builder.getValue(V).getValueType()))
604 return true;
605 if (isGCValue(V, Builder))
606 return !LowerAsVReg.count(Builder.getValue(V));
607 return !(LiveInDeopt || UseRegistersForDeoptValues);
608 };
609
610 // Before we actually start lowering (and allocating spill slots for values),
611 // reserve any stack slots which we judge to be profitable to reuse for a
612 // particular value. This is purely an optimization over the code below and
613 // doesn't change semantics at all. It is important for performance that we
614 // reserve slots for both deopt and gc values before lowering either.
615 for (const Value *V : SI.DeoptState) {
616 if (requireSpillSlot(V))
618 }
619
620 for (const Value *V : SI.Ptrs) {
621 SDValue SDV = Builder.getValue(V);
622 if (!LowerAsVReg.count(SDV))
624 }
625
626 for (const Value *V : SI.Bases) {
627 SDValue SDV = Builder.getValue(V);
628 if (!LowerAsVReg.count(SDV))
630 }
631
632 // First, prefix the list with the number of unique values to be
633 // lowered. Note that this is the number of *Values* not the
634 // number of SDValues required to lower them.
635 const int NumVMSArgs = SI.DeoptState.size();
636 pushStackMapConstant(Ops, Builder, NumVMSArgs);
637
638 // The vm state arguments are lowered in an opaque manner. We do not know
639 // what type of values are contained within.
640 LLVM_DEBUG(dbgs() << "Lowering deopt state\n");
641 for (const Value *V : SI.DeoptState) {
643 // If this is a function argument at a static frame index, generate it as
644 // the frame index.
645 if (const Argument *Arg = dyn_cast<Argument>(V)) {
646 int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);
647 if (FI != INT_MAX)
648 Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());
649 }
650 if (!Incoming.getNode())
651 Incoming = Builder.getValue(V);
652 LLVM_DEBUG(dbgs() << "Value " << *V
653 << " requireSpillSlot = " << requireSpillSlot(V) << "\n");
654 lowerIncomingStatepointValue(Incoming, requireSpillSlot(V), Ops, MemRefs,
655 Builder);
656 }
657
658 // Finally, go ahead and lower all the gc arguments.
659 pushStackMapConstant(Ops, Builder, LoweredGCPtrs.size());
660 for (SDValue SDV : LoweredGCPtrs)
661 lowerIncomingStatepointValue(SDV, !LowerAsVReg.count(SDV), Ops, MemRefs,
662 Builder);
663
664 // Copy to out vector. LoweredGCPtrs will be empty after this point.
665 GCPtrs = LoweredGCPtrs.takeVector();
666
667 // If there are any explicit spill slots passed to the statepoint, record
668 // them, but otherwise do not do anything special. These are user provided
669 // allocas and give control over placement to the consumer. In this case,
670 // it is the contents of the slot which may get updated, not the pointer to
671 // the alloca
673 for (Value *V : SI.GCArgs) {
674 SDValue Incoming = Builder.getValue(V);
675 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
676 // This handles allocas as arguments to the statepoint
677 assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
678 "Incoming value is a frame index!");
679 Allocas.push_back(Builder.DAG.getTargetFrameIndex(
680 FI->getIndex(), Builder.getFrameIndexTy()));
681
682 auto &MF = Builder.DAG.getMachineFunction();
683 auto *MMO = getMachineMemOperand(MF, *FI);
684 MemRefs.push_back(MMO);
685 }
686 }
687 pushStackMapConstant(Ops, Builder, Allocas.size());
688 Ops.append(Allocas.begin(), Allocas.end());
689
690 // Now construct GC base/derived map;
691 pushStackMapConstant(Ops, Builder, SI.Ptrs.size());
692 SDLoc L = Builder.getCurSDLoc();
693 for (unsigned i = 0; i < SI.Ptrs.size(); ++i) {
694 SDValue Base = Builder.getValue(SI.Bases[i]);
695 assert(GCPtrIndexMap.count(Base) && "base not found in index map");
696 Ops.push_back(
697 Builder.DAG.getTargetConstant(GCPtrIndexMap[Base], L, MVT::i64));
698 SDValue Derived = Builder.getValue(SI.Ptrs[i]);
699 assert(GCPtrIndexMap.count(Derived) && "derived not found in index map");
700 Ops.push_back(
701 Builder.DAG.getTargetConstant(GCPtrIndexMap[Derived], L, MVT::i64));
702 }
703}
704
707 // The basic scheme here is that information about both the original call and
708 // the safepoint is encoded in the CallInst. We create a temporary call and
709 // lower it, then reverse engineer the calling sequence.
710
711 NumOfStatepoints++;
712 // Clear state
714 assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
715 assert((GFI || SI.Bases.empty()) &&
716 "No gc specified, so cannot relocate pointers!");
717
718 LLVM_DEBUG(if (SI.StatepointInstr) dbgs()
719 << "Lowering statepoint " << *SI.StatepointInstr << "\n");
720#ifndef NDEBUG
721 for (const auto *Reloc : SI.GCRelocates)
722 if (Reloc->getParent() == SI.StatepointInstr->getParent())
724#endif
725
726 // Lower statepoint vmstate and gcstate arguments
727
728 // All lowered meta args.
729 SmallVector<SDValue, 10> LoweredMetaArgs;
730 // Lowered GC pointers (subset of above).
731 SmallVector<SDValue, 16> LoweredGCArgs;
733 // Maps derived pointer SDValue to statepoint result of relocated pointer.
734 DenseMap<SDValue, int> LowerAsVReg;
735 lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, LoweredGCArgs, LowerAsVReg,
736 SI, *this);
737
738 // Now that we've emitted the spills, we need to update the root so that the
739 // call sequence is ordered correctly.
740 SI.CLI.setChain(getRoot());
741
742 // Get call node, we will replace it later with statepoint
743 SDValue ReturnVal;
744 SDNode *CallNode;
745 std::tie(ReturnVal, CallNode) = lowerCallFromStatepointLoweringInfo(SI, *this);
746
747 // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
748 // nodes with all the appropriate arguments and return values.
749
750 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
751 SDValue Chain = CallNode->getOperand(0);
752
753 SDValue Glue;
754 bool CallHasIncomingGlue = CallNode->getGluedNode();
755 if (CallHasIncomingGlue) {
756 // Glue is always last operand
757 Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
758 }
759
760 // Build the GC_TRANSITION_START node if necessary.
761 //
762 // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
763 // order in which they appear in the call to the statepoint intrinsic. If
764 // any of the operands is a pointer-typed, that operand is immediately
765 // followed by a SRCVALUE for the pointer that may be used during lowering
766 // (e.g. to form MachinePointerInfo values for loads/stores).
767 const bool IsGCTransition =
768 (SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) ==
770 if (IsGCTransition) {
772
773 // Add chain
774 TSOps.push_back(Chain);
775
776 // Add GC transition arguments
777 for (const Value *V : SI.GCTransitionArgs) {
778 TSOps.push_back(getValue(V));
779 if (V->getType()->isPointerTy())
780 TSOps.push_back(DAG.getSrcValue(V));
781 }
782
783 // Add glue if necessary
784 if (CallHasIncomingGlue)
785 TSOps.push_back(Glue);
786
787 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
788
789 SDValue GCTransitionStart =
791
792 Chain = GCTransitionStart.getValue(0);
793 Glue = GCTransitionStart.getValue(1);
794 }
795
796 // TODO: Currently, all of these operands are being marked as read/write in
797 // PrologEpilougeInserter.cpp, we should special case the VMState arguments
798 // and flags to be read-only.
800
801 // Add the <id> and <numBytes> constants.
802 Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
803 Ops.push_back(
804 DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
805
806 // Calculate and push starting position of vmstate arguments
807 // Get number of arguments incoming directly into call node
808 unsigned NumCallRegArgs =
809 CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
810 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
811
812 // Add call target
813 SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
814 Ops.push_back(CallTarget);
815
816 // Add call arguments
817 // Get position of register mask in the call
818 SDNode::op_iterator RegMaskIt;
819 if (CallHasIncomingGlue)
820 RegMaskIt = CallNode->op_end() - 2;
821 else
822 RegMaskIt = CallNode->op_end() - 1;
823 Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
824
825 // Add a constant argument for the calling convention
826 pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
827
828 // Add a constant argument for the flags
829 uint64_t Flags = SI.StatepointFlags;
830 assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
831 "Unknown flag used");
832 pushStackMapConstant(Ops, *this, Flags);
833
834 // Insert all vmstate and gcstate arguments
835 llvm::append_range(Ops, LoweredMetaArgs);
836
837 // Add register mask from call node
838 Ops.push_back(*RegMaskIt);
839
840 // Add chain
841 Ops.push_back(Chain);
842
843 // Same for the glue, but we add it only if original call had it
844 if (Glue.getNode())
845 Ops.push_back(Glue);
846
847 // Compute return values. Provide a glue output since we consume one as
848 // input. This allows someone else to chain off us as needed.
849 SmallVector<EVT, 8> NodeTys;
850 for (auto SD : LoweredGCArgs) {
851 if (!LowerAsVReg.count(SD))
852 continue;
853 NodeTys.push_back(SD.getValueType());
854 }
855 LLVM_DEBUG(dbgs() << "Statepoint has " << NodeTys.size() << " results\n");
856 assert(NodeTys.size() == LowerAsVReg.size() && "Inconsistent GC Ptr lowering");
857 NodeTys.push_back(MVT::Other);
858 NodeTys.push_back(MVT::Glue);
859
860 unsigned NumResults = NodeTys.size();
861 MachineSDNode *StatepointMCNode =
862 DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
863 DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
864
865 // For values lowered to tied-defs, create the virtual registers if used
866 // in other blocks. For local gc.relocate record appropriate statepoint
867 // result in StatepointLoweringState.
869 for (const auto *Relocate : SI.GCRelocates) {
870 Value *Derived = Relocate->getDerivedPtr();
871 SDValue SD = getValue(Derived);
872 if (!LowerAsVReg.count(SD))
873 continue;
874
875 SDValue Relocated = SDValue(StatepointMCNode, LowerAsVReg[SD]);
876
877 // Handle local relocate. Note that different relocates might
878 // map to the same SDValue.
879 if (SI.StatepointInstr->getParent() == Relocate->getParent()) {
881 if (Res)
882 assert(Res == Relocated);
883 else
884 StatepointLowering.setLocation(SD, Relocated);
885 continue;
886 }
887
888 // Handle multiple gc.relocates of the same input efficiently.
889 if (VirtRegs.count(SD))
890 continue;
891
892 auto *RetTy = Relocate->getType();
895 DAG.getDataLayout(), Reg, RetTy, std::nullopt);
896 SDValue Chain = DAG.getRoot();
897 RFV.getCopyToRegs(Relocated, DAG, getCurSDLoc(), Chain, nullptr);
898 PendingExports.push_back(Chain);
899
900 VirtRegs[SD] = Reg;
901 }
902
903 // Record for later use how each relocation was lowered. This is needed to
904 // allow later gc.relocates to mirror the lowering chosen.
905 const Instruction *StatepointInstr = SI.StatepointInstr;
906 auto &RelocationMap = FuncInfo.StatepointRelocationMaps[StatepointInstr];
907 for (const GCRelocateInst *Relocate : SI.GCRelocates) {
908 const Value *V = Relocate->getDerivedPtr();
909 SDValue SDV = getValue(V);
911
912 bool IsLocal = (Relocate->getParent() == StatepointInstr->getParent());
913
915 if (IsLocal && LowerAsVReg.count(SDV)) {
916 // Result is already stored in StatepointLowering
918 } else if (LowerAsVReg.count(SDV)) {
920 assert(VirtRegs.count(SDV));
921 Record.payload.Reg = VirtRegs[SDV];
922 } else if (Loc.getNode()) {
924 Record.payload.FI = cast<FrameIndexSDNode>(Loc)->getIndex();
925 } else {
927 // If we didn't relocate a value, we'll essentialy end up inserting an
928 // additional use of the original value when lowering the gc.relocate.
929 // We need to make sure the value is available at the new use, which
930 // might be in another block.
931 if (Relocate->getParent() != StatepointInstr->getParent())
933 }
934 RelocationMap[Relocate] = Record;
935 }
936
937
938
939 SDNode *SinkNode = StatepointMCNode;
940
941 // Build the GC_TRANSITION_END node if necessary.
942 //
943 // See the comment above regarding GC_TRANSITION_START for the layout of
944 // the operands to the GC_TRANSITION_END node.
945 if (IsGCTransition) {
947
948 // Add chain
949 TEOps.push_back(SDValue(StatepointMCNode, NumResults - 2));
950
951 // Add GC transition arguments
952 for (const Value *V : SI.GCTransitionArgs) {
953 TEOps.push_back(getValue(V));
954 if (V->getType()->isPointerTy())
955 TEOps.push_back(DAG.getSrcValue(V));
956 }
957
958 // Add glue
959 TEOps.push_back(SDValue(StatepointMCNode, NumResults - 1));
960
961 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
962
963 SDValue GCTransitionStart =
964 DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
965
966 SinkNode = GCTransitionStart.getNode();
967 }
968
969 // Replace original call
970 // Call: ch,glue = CALL ...
971 // Statepoint: [gc relocates],ch,glue = STATEPOINT ...
972 unsigned NumSinkValues = SinkNode->getNumValues();
973 SDValue StatepointValues[2] = {SDValue(SinkNode, NumSinkValues - 2),
974 SDValue(SinkNode, NumSinkValues - 1)};
975 DAG.ReplaceAllUsesWith(CallNode, StatepointValues);
976 // Remove original call node
977 DAG.DeleteNode(CallNode);
978
979 // Since we always emit CopyToRegs (even for local relocates), we must
980 // update root, so that they are emitted before any local uses.
981 (void)getControlRoot();
982
983 // TODO: A better future implementation would be to emit a single variable
984 // argument, variable return value STATEPOINT node here and then hookup the
985 // return value of each gc.relocate to the respective output of the
986 // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
987 // to actually be possible today.
988
989 return ReturnVal;
990}
991
992/// Return two gc.results if present. First result is a block local
993/// gc.result, second result is a non-block local gc.result. Corresponding
994/// entry will be nullptr if not present.
995static std::pair<const GCResultInst*, const GCResultInst*>
997 std::pair<const GCResultInst *, const GCResultInst*> Res(nullptr, nullptr);
998 for (const auto *U : S.users()) {
999 auto *GRI = dyn_cast<GCResultInst>(U);
1000 if (!GRI)
1001 continue;
1002 if (GRI->getParent() == S.getParent())
1003 Res.first = GRI;
1004 else
1005 Res.second = GRI;
1006 }
1007 return Res;
1008}
1009
1010void
1012 const BasicBlock *EHPadBB /*= nullptr*/) {
1013 assert(I.getCallingConv() != CallingConv::AnyReg &&
1014 "anyregcc is not supported on statepoints!");
1015
1016#ifndef NDEBUG
1017 // Check that the associated GCStrategy expects to encounter statepoints.
1019 "GCStrategy does not expect to encounter statepoints");
1020#endif
1021
1022 SDValue ActualCallee;
1023 SDValue Callee = getValue(I.getActualCalledOperand());
1024
1025 if (I.getNumPatchBytes() > 0) {
1026 // If we've been asked to emit a nop sequence instead of a call instruction
1027 // for this statepoint then don't lower the call target, but use a constant
1028 // `undef` instead. Not lowering the call target lets statepoint clients
1029 // get away without providing a physical address for the symbolic call
1030 // target at link time.
1031 ActualCallee = DAG.getUNDEF(Callee.getValueType());
1032 } else {
1033 ActualCallee = Callee;
1034 }
1035
1036 const auto GCResultLocality = getGCResultLocality(I);
1037 AttributeSet retAttrs;
1038 if (GCResultLocality.first)
1039 retAttrs = GCResultLocality.first->getAttributes().getRetAttrs();
1040
1043 I.getNumCallArgs(), ActualCallee,
1044 I.getActualReturnType(), retAttrs,
1045 /*IsPatchPoint=*/false);
1046
1047 // There may be duplication in the gc.relocate list; such as two copies of
1048 // each relocation on normal and exceptional path for an invoke. We only
1049 // need to spill once and record one copy in the stackmap, but we need to
1050 // reload once per gc.relocate. (Dedupping gc.relocates is trickier and best
1051 // handled as a CSE problem elsewhere.)
1052 // TODO: There a couple of major stackmap size optimizations we could do
1053 // here if we wished.
1054 // 1) If we've encountered a derived pair {B, D}, we don't need to actually
1055 // record {B,B} if it's seen later.
1056 // 2) Due to rematerialization, actual derived pointers are somewhat rare;
1057 // given that, we could change the format to record base pointer relocations
1058 // separately with half the space. This would require a format rev and a
1059 // fairly major rework of the STATEPOINT node though.
1061 for (const GCRelocateInst *Relocate : I.getGCRelocates()) {
1062 SI.GCRelocates.push_back(Relocate);
1063
1064 SDValue DerivedSD = getValue(Relocate->getDerivedPtr());
1065 if (Seen.insert(DerivedSD).second) {
1066 SI.Bases.push_back(Relocate->getBasePtr());
1067 SI.Ptrs.push_back(Relocate->getDerivedPtr());
1068 }
1069 }
1070
1071 // If we find a deopt value which isn't explicitly added, we need to
1072 // ensure it gets lowered such that gc cycles occurring before the
1073 // deoptimization event during the lifetime of the call don't invalidate
1074 // the pointer we're deopting with. Note that we assume that all
1075 // pointers passed to deopt are base pointers; relaxing that assumption
1076 // would require relatively large changes to how we represent relocations.
1077 for (Value *V : I.deopt_operands()) {
1078 if (!isGCValue(V, *this))
1079 continue;
1080 if (Seen.insert(getValue(V)).second) {
1081 SI.Bases.push_back(V);
1082 SI.Ptrs.push_back(V);
1083 }
1084 }
1085
1086 SI.GCArgs = ArrayRef<const Use>(I.gc_args_begin(), I.gc_args_end());
1087 SI.StatepointInstr = &I;
1088 SI.ID = I.getID();
1089
1090 SI.DeoptState = ArrayRef<const Use>(I.deopt_begin(), I.deopt_end());
1091 SI.GCTransitionArgs = ArrayRef<const Use>(I.gc_transition_args_begin(),
1092 I.gc_transition_args_end());
1093
1094 SI.StatepointFlags = I.getFlags();
1095 SI.NumPatchBytes = I.getNumPatchBytes();
1096 SI.EHPadBB = EHPadBB;
1097
1098 SDValue ReturnValue = LowerAsSTATEPOINT(SI);
1099
1100 // Export the result value if needed
1101 if (!GCResultLocality.first && !GCResultLocality.second) {
1102 // The return value is not needed, just generate a poison value.
1103 // Note: This covers the void return case.
1105 return;
1106 }
1107
1108 if (GCResultLocality.first) {
1109 // Result value will be used in a same basic block. Don't export it or
1110 // perform any explicit register copies. The gc_result will simply grab
1111 // this value.
1112 setValue(&I, ReturnValue);
1113 }
1114
1115 if (!GCResultLocality.second)
1116 return;
1117 // Result value will be used in a different basic block so we need to export
1118 // it now. Default exporting mechanism will not work here because statepoint
1119 // call has a different type than the actual call. It means that by default
1120 // llvm will create export register of the wrong type (always i32 in our
1121 // case). So instead we need to create export register with correct type
1122 // manually.
1123 // TODO: To eliminate this problem we can remove gc.result intrinsics
1124 // completely and make statepoint call to return a tuple.
1125 Type *RetTy = GCResultLocality.second->getType();
1128 DAG.getDataLayout(), Reg, RetTy,
1129 I.getCallingConv());
1130 SDValue Chain = DAG.getEntryNode();
1131
1132 RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
1133 PendingExports.push_back(Chain);
1134 FuncInfo.ValueMap[&I] = Reg;
1135}
1136
1138 const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
1139 bool VarArgDisallowed, bool ForceVoidReturnTy) {
1141 unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
1143 SI.CLI, Call, ArgBeginIndex, Call->arg_size(), Callee,
1144 ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
1145 Call->getAttributes().getRetAttrs(), /*IsPatchPoint=*/false);
1146 if (!VarArgDisallowed)
1147 SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
1148
1149 auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
1150
1152
1153 auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes());
1154 SI.ID = SD.StatepointID.value_or(DefaultID);
1155 SI.NumPatchBytes = SD.NumPatchBytes.value_or(0);
1156
1157 SI.DeoptState =
1158 ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
1159 SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
1160 SI.EHPadBB = EHPadBB;
1161
1162 // NB! The GC arguments are deliberately left empty.
1163
1164 LLVM_DEBUG(dbgs() << "Lowering call with deopt bundle " << *Call << "\n");
1165 if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
1166 ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
1167 setValue(Call, ReturnVal);
1168 }
1169}
1170
1172 const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
1173 LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
1174 /* VarArgDisallowed = */ false,
1175 /* ForceVoidReturnTy = */ false);
1176}
1177
1178void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
1179 // The result value of the gc_result is simply the result of the actual
1180 // call. We've already emitted this, so just grab the value.
1181 const Value *SI = CI.getStatepoint();
1182 assert((isa<GCStatepointInst>(SI) || isa<UndefValue>(SI)) &&
1183 "GetStatepoint must return one of two types");
1184 if (isa<UndefValue>(SI))
1185 return;
1186
1187 if (cast<GCStatepointInst>(SI)->getParent() == CI.getParent()) {
1188 setValue(&CI, getValue(SI));
1189 return;
1190 }
1191 // Statepoint is in different basic block so we should have stored call
1192 // result in a virtual register.
1193 // We can not use default getValue() functionality to copy value from this
1194 // register because statepoint and actual call return types can be
1195 // different, and getValue() will use CopyFromReg of the wrong type,
1196 // which is always i32 in our case.
1197 Type *RetTy = CI.getType();
1198 SDValue CopyFromReg = getCopyFromRegs(SI, RetTy);
1199
1200 assert(CopyFromReg.getNode());
1201 setValue(&CI, CopyFromReg);
1202}
1203
1204void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
1205 const Value *Statepoint = Relocate.getStatepoint();
1206#ifndef NDEBUG
1207 // Consistency check
1208 // We skip this check for relocates not in the same basic block as their
1209 // statepoint. It would be too expensive to preserve validation info through
1210 // different basic blocks.
1211 assert((isa<GCStatepointInst>(Statepoint) || isa<UndefValue>(Statepoint)) &&
1212 "GetStatepoint must return one of two types");
1213 if (isa<UndefValue>(Statepoint))
1214 return;
1215
1216 if (cast<GCStatepointInst>(Statepoint)->getParent() == Relocate.getParent())
1218#endif
1219
1220 const Value *DerivedPtr = Relocate.getDerivedPtr();
1221 auto &RelocationMap =
1222 FuncInfo.StatepointRelocationMaps[cast<GCStatepointInst>(Statepoint)];
1223 auto SlotIt = RelocationMap.find(&Relocate);
1224 assert(SlotIt != RelocationMap.end() && "Relocating not lowered gc value");
1225 const RecordType &Record = SlotIt->second;
1226
1227 // If relocation was done via virtual register..
1228 if (Record.type == RecordType::SDValueNode) {
1229 assert(cast<GCStatepointInst>(Statepoint)->getParent() ==
1230 Relocate.getParent() &&
1231 "Nonlocal gc.relocate mapped via SDValue");
1233 assert(SDV.getNode() && "empty SDValue");
1234 setValue(&Relocate, SDV);
1235 return;
1236 }
1237 if (Record.type == RecordType::VReg) {
1238 Register InReg = Record.payload.Reg;
1240 DAG.getDataLayout(), InReg, Relocate.getType(),
1241 std::nullopt); // This is not an ABI copy.
1242 // We generate copy to/from regs even for local uses, hence we must
1243 // chain with current root to ensure proper ordering of copies w.r.t.
1244 // statepoint.
1245 SDValue Chain = DAG.getRoot();
1246 SDValue Relocation = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
1247 Chain, nullptr, nullptr);
1248 setValue(&Relocate, Relocation);
1249 return;
1250 }
1251
1252 if (Record.type == RecordType::Spill) {
1253 unsigned Index = Record.payload.FI;
1255
1256 // All the reloads are independent and are reading memory only modified by
1257 // statepoints (i.e. no other aliasing stores); informing SelectionDAG of
1258 // this lets CSE kick in for free and allows reordering of
1259 // instructions if possible. The lowering for statepoint sets the root,
1260 // so this is ordering all reloads with the either
1261 // a) the statepoint node itself, or
1262 // b) the entry of the current block for an invoke statepoint.
1263 const SDValue Chain = DAG.getRoot(); // != Builder.getRoot()
1264
1265 auto &MF = DAG.getMachineFunction();
1266 auto &MFI = MF.getFrameInfo();
1267 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
1268 auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
1269 MFI.getObjectSize(Index),
1270 MFI.getObjectAlign(Index));
1271
1273 Relocate.getType());
1274
1275 SDValue SpillLoad =
1276 DAG.getLoad(LoadVT, getCurSDLoc(), Chain, SpillSlot, LoadMMO);
1277 PendingLoads.push_back(SpillLoad.getValue(1));
1278
1279 assert(SpillLoad.getNode());
1280 setValue(&Relocate, SpillLoad);
1281 return;
1282 }
1283
1285 SDValue SD = getValue(DerivedPtr);
1286
1287 if (SD.isUndef() && SD.getValueType().getSizeInBits() <= 64) {
1288 // Lowering relocate(undef) as arbitrary constant. Current constant value
1289 // is chosen such that it's unlikely to be a valid pointer.
1290 setValue(&Relocate, DAG.getConstant(0xFEFEFEFE, SDLoc(SD), MVT::i64));
1291 return;
1292 }
1293
1294 // We didn't need to spill these special cases (constants and allocas).
1295 // See the handling in spillIncomingValueForStatepoint for detail.
1296 setValue(&Relocate, SD);
1297}
1298
1300 const auto &TLI = DAG.getTargetLoweringInfo();
1301 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1302 TLI.getPointerTy(DAG.getDataLayout()));
1303
1304 // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1305 // call. We also do not lower the return value to any virtual register, and
1306 // change the immediately following return to a trap instruction.
1307 LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1308 /* VarArgDisallowed = */ true,
1309 /* ForceVoidReturnTy = */ true);
1310}
1311
1313 // We do not lower the return value from llvm.deoptimize to any virtual
1314 // register, and change the immediately following return to a trap
1315 // instruction.
1317 DAG.setRoot(
1318 DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
1319}
static const Function * getParent(const Value *V)
return RetTy
#define LLVM_DEBUG(X)
Definition: Debug.h:101
#define I(x, y, z)
Definition: MD5.cpp:58
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallSet class.
This file defines the SmallVector class.
static cl::opt< bool > UseRegistersForGCPointersInLandingPad("use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false), cl::desc("Allow using registers for gc pointer in landing pad"))
static void lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
static std::optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
static bool isGCValue(const Value *V, SelectionDAGBuilder &Builder)
Return true if value V represents the GC value.
static bool willLowerDirectly(SDValue Incoming)
Return true if-and-only-if the given SDValue can be lowered as either a constant argument or a stack ...
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SmallVectorImpl< SDValue > &GCPtrs, DenseMap< SDValue, int > &LowerAsVReg, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
static std::pair< const GCResultInst *, const GCResultInst * > getGCResultLocality(const GCStatepointInst &S)
Return two gc.results if present.
static cl::opt< bool > UseRegistersForDeoptValues("use-registers-for-deopt-values", cl::Hidden, cl::init(false), cl::desc("Allow using registers for non pointer deopt args"))
static cl::opt< unsigned > MaxRegistersForGCPointers("max-registers-for-gc-values", cl::Hidden, cl::init(0), cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"))
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling.
FunctionLoweringInfo::StatepointRelocationRecord RecordType
static MachineMemOperand * getMachineMemOperand(MachineFunction &MF, FrameIndexSDNode &FI)
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Extract call from statepoint, lower it and return pointer to the call node.
static std::tuple< SDValue, SDValue, MachineMemOperand * > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This file describes how to lower LLVM code to machine code.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1455
This class represents a function call, abstracting a target machine's calling convention.
unsigned size() const
Definition: DenseMap.h:99
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
Register CreateRegs(const Value *V)
DenseMap< const Instruction *, StatepointSpillMapTy > StatepointRelocationMaps
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
GCStrategy & getStrategy()
getStrategy - Return the GC strategy for the function.
Definition: GCMetadata.h:113
const Value * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Represents calls to the gc.relocate intrinsic.
Value * getDerivedPtr() const
Represents calls to the gc.result intrinsic.
Represents a gc.statepoint intrinsic call.
Definition: Statepoint.h:61
bool useStatepoints() const
Returns true if this strategy is expecting the use of gc.statepoints, and false otherwise.
Definition: GCStrategy.h:92
const BasicBlock * getParent() const
Definition: Instruction.h:151
The landingpad instruction holds all of the information necessary to generate correct exception handl...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
op_iterator op_end() const
op_iterator op_begin() const
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)
Lower SLI into a STATEPOINT instruction.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:727
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
Vector takeVector()
Clear the SetVector and return the underlying vector.
Definition: SetVector.h:87
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
SmallBitVector & set()
bool test(unsigned Idx) const
void clear()
Clear all bits.
size_type size() const
Returns the number of bits in this bitvector.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:818
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
void clear()
Clear the memory usage of this object.
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
void scheduleRelocCall(const GCRelocateInst &RelocCall)
Record the fact that we expect to encounter a given gc_relocate before the next statepoint.
void setLocation(SDValue Val, SDValue Location)
void relocCallVisited(const GCRelocateInst &RelocCall)
Remove this gc_relocate from the list we're expecting to see before the next statepoint.
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
TargetOptions Options
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
iterator_range< user_iterator > users()
Definition: Value.h:421
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition: CallingConv.h:60
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ GC_TRANSITION_START
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:1313
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1208
@ GC_TRANSITION_END
Definition: ISDOpcodes.h:1314
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1751
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2082
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:24
@ MaskAll
A bitmask that includes all valid flags.
@ DeoptLiveIn
Mark the deopt arguments associated with the statepoint as only being "live-in".
@ GCTransition
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:240