LLVM 18.0.0git
StatepointLowering.cpp
Go to the documentation of this file.
1//===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file includes support code use by SelectionDAGBuilder when lowering a
10// statepoint sequence in SelectionDAG IR.
11//
12//===----------------------------------------------------------------------===//
13
14#include "StatepointLowering.h"
15#include "SelectionDAGBuilder.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SetVector.h"
20#include "llvm/ADT/SmallSet.h"
22#include "llvm/ADT/Statistic.h"
36#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/GCStrategy.h"
39#include "llvm/IR/Instruction.h"
41#include "llvm/IR/LLVMContext.h"
42#include "llvm/IR/Statepoint.h"
43#include "llvm/IR/Type.h"
48#include <cassert>
49#include <cstddef>
50#include <cstdint>
51#include <iterator>
52#include <tuple>
53#include <utility>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "statepoint-lowering"
58
59STATISTIC(NumSlotsAllocatedForStatepoints,
60 "Number of stack slots allocated for statepoints");
61STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
62STATISTIC(StatepointMaxSlotsRequired,
63 "Maximum number of stack slots required for a singe statepoint");
64
66 "use-registers-for-deopt-values", cl::Hidden, cl::init(false),
67 cl::desc("Allow using registers for non pointer deopt args"));
68
70 "use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false),
71 cl::desc("Allow using registers for gc pointer in landing pad"));
72
74 "max-registers-for-gc-values", cl::Hidden, cl::init(0),
75 cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"));
76
78
81 SDLoc L = Builder.getCurSDLoc();
82 Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
83 MVT::i64));
84 Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
85}
86
88 // Consistency check
89 assert(PendingGCRelocateCalls.empty() &&
90 "Trying to visit statepoint before finished processing previous one");
91 Locations.clear();
92 NextSlotToAllocate = 0;
93 // Need to resize this on each safepoint - we need the two to stay in sync and
94 // the clear patterns of a SelectionDAGBuilder have no relation to
95 // FunctionLoweringInfo. Also need to ensure used bits get cleared.
96 AllocatedStackSlots.clear();
97 AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
98}
99
101 Locations.clear();
102 AllocatedStackSlots.clear();
103 assert(PendingGCRelocateCalls.empty() &&
104 "cleared before statepoint sequence completed");
105}
106
109 SelectionDAGBuilder &Builder) {
110 NumSlotsAllocatedForStatepoints++;
111 MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
112
113 unsigned SpillSize = ValueType.getStoreSize();
114 assert((SpillSize * 8) ==
115 (-8u & (7 + ValueType.getSizeInBits())) && // Round up modulo 8.
116 "Size not in bytes?");
117
118 // First look for a previously created stack slot which is not in
119 // use (accounting for the fact arbitrary slots may already be
120 // reserved), or to create a new stack slot and use it.
121
122 const size_t NumSlots = AllocatedStackSlots.size();
123 assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
124
125 assert(AllocatedStackSlots.size() ==
126 Builder.FuncInfo.StatepointStackSlots.size() &&
127 "Broken invariant");
128
129 for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
130 if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
131 const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
132 if (MFI.getObjectSize(FI) == SpillSize) {
133 AllocatedStackSlots.set(NextSlotToAllocate);
134 // TODO: Is ValueType the right thing to use here?
135 return Builder.DAG.getFrameIndex(FI, ValueType);
136 }
137 }
138 }
139
140 // Couldn't find a free slot, so create a new one:
141
142 SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
143 const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
145
146 Builder.FuncInfo.StatepointStackSlots.push_back(FI);
147 AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
148 assert(AllocatedStackSlots.size() ==
149 Builder.FuncInfo.StatepointStackSlots.size() &&
150 "Broken invariant");
151
152 StatepointMaxSlotsRequired.updateMax(
153 Builder.FuncInfo.StatepointStackSlots.size());
154
155 return SpillSlot;
156}
157
158/// Utility function for reservePreviousStackSlotForValue. Tries to find
159/// stack slot index to which we have spilled value for previous statepoints.
160/// LookUpDepth specifies maximum DFS depth this function is allowed to look.
161static std::optional<int> findPreviousSpillSlot(const Value *Val,
162 SelectionDAGBuilder &Builder,
163 int LookUpDepth) {
164 // Can not look any further - give up now
165 if (LookUpDepth <= 0)
166 return std::nullopt;
167
168 // Spill location is known for gc relocates
169 if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
170 const Value *Statepoint = Relocate->getStatepoint();
171 assert((isa<GCStatepointInst>(Statepoint) || isa<UndefValue>(Statepoint)) &&
172 "GetStatepoint must return one of two types");
173 if (isa<UndefValue>(Statepoint))
174 return std::nullopt;
175
176 const auto &RelocationMap = Builder.FuncInfo.StatepointRelocationMaps
177 [cast<GCStatepointInst>(Statepoint)];
178
179 auto It = RelocationMap.find(Relocate);
180 if (It == RelocationMap.end())
181 return std::nullopt;
182
183 auto &Record = It->second;
184 if (Record.type != RecordType::Spill)
185 return std::nullopt;
186
187 return Record.payload.FI;
188 }
189
190 // Look through bitcast instructions.
191 if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
192 return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
193
194 // Look through phi nodes
195 // All incoming values should have same known stack slot, otherwise result
196 // is unknown.
197 if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
198 std::optional<int> MergedResult;
199
200 for (const auto &IncomingValue : Phi->incoming_values()) {
201 std::optional<int> SpillSlot =
202 findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
203 if (!SpillSlot)
204 return std::nullopt;
205
206 if (MergedResult && *MergedResult != *SpillSlot)
207 return std::nullopt;
208
209 MergedResult = SpillSlot;
210 }
211 return MergedResult;
212 }
213
214 // TODO: We can do better for PHI nodes. In cases like this:
215 // ptr = phi(relocated_pointer, not_relocated_pointer)
216 // statepoint(ptr)
217 // We will return that stack slot for ptr is unknown. And later we might
218 // assign different stack slots for ptr and relocated_pointer. This limits
219 // llvm's ability to remove redundant stores.
220 // Unfortunately it's hard to accomplish in current infrastructure.
221 // We use this function to eliminate spill store completely, while
222 // in example we still need to emit store, but instead of any location
223 // we need to use special "preferred" location.
224
225 // TODO: handle simple updates. If a value is modified and the original
226 // value is no longer live, it would be nice to put the modified value in the
227 // same slot. This allows folding of the memory accesses for some
228 // instructions types (like an increment).
229 // statepoint (i)
230 // i1 = i+1
231 // statepoint (i1)
232 // However we need to be careful for cases like this:
233 // statepoint(i)
234 // i1 = i+1
235 // statepoint(i, i1)
236 // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
237 // put handling of simple modifications in this function like it's done
238 // for bitcasts we might end up reserving i's slot for 'i+1' because order in
239 // which we visit values is unspecified.
240
241 // Don't know any information about this instruction
242 return std::nullopt;
243}
244
245/// Return true if-and-only-if the given SDValue can be lowered as either a
246/// constant argument or a stack reference. The key point is that the value
247/// doesn't need to be spilled or tracked as a vreg use.
248static bool willLowerDirectly(SDValue Incoming) {
249 // We are making an unchecked assumption that the frame size <= 2^16 as that
250 // is the largest offset which can be encoded in the stackmap format.
251 if (isa<FrameIndexSDNode>(Incoming))
252 return true;
253
254 // The largest constant describeable in the StackMap format is 64 bits.
255 // Potential Optimization: Constants values are sign extended by consumer,
256 // and thus there are many constants of static type > 64 bits whose value
257 // happens to be sext(Con64) and could thus be lowered directly.
258 if (Incoming.getValueType().getSizeInBits() > 64)
259 return false;
260
261 return isIntOrFPConstant(Incoming) || Incoming.isUndef();
262}
263
264/// Try to find existing copies of the incoming values in stack slots used for
265/// statepoint spilling. If we can find a spill slot for the incoming value,
266/// mark that slot as allocated, and reuse the same slot for this safepoint.
267/// This helps to avoid series of loads and stores that only serve to reshuffle
268/// values on the stack between calls.
269static void reservePreviousStackSlotForValue(const Value *IncomingValue,
270 SelectionDAGBuilder &Builder) {
271 SDValue Incoming = Builder.getValue(IncomingValue);
272
273 // If we won't spill this, we don't need to check for previously allocated
274 // stack slots.
275 if (willLowerDirectly(Incoming))
276 return;
277
278 SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
279 if (OldLocation.getNode())
280 // Duplicates in input
281 return;
282
283 const int LookUpDepth = 6;
284 std::optional<int> Index =
285 findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
286 if (!Index)
287 return;
288
289 const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
290
291 auto SlotIt = find(StatepointSlots, *Index);
292 assert(SlotIt != StatepointSlots.end() &&
293 "Value spilled to the unknown stack slot");
294
295 // This is one of our dedicated lowering slots
296 const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
297 if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
298 // stack slot already assigned to someone else, can't use it!
299 // TODO: currently we reserve space for gc arguments after doing
300 // normal allocation for deopt arguments. We should reserve for
301 // _all_ deopt and gc arguments, then start allocating. This
302 // will prevent some moves being inserted when vm state changes,
303 // but gc state doesn't between two calls.
304 return;
305 }
306 // Reserve this stack slot
307 Builder.StatepointLowering.reserveStackSlot(Offset);
308
309 // Cache this slot so we find it when going through the normal
310 // assignment loop.
311 SDValue Loc =
312 Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
313 Builder.StatepointLowering.setLocation(Incoming, Loc);
314}
315
316/// Extract call from statepoint, lower it and return pointer to the
317/// call node. Also update NodeMap so that getValue(statepoint) will
318/// reference lowered call result
319static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
321 SelectionDAGBuilder &Builder) {
322 SDValue ReturnValue, CallEndVal;
323 std::tie(ReturnValue, CallEndVal) =
324 Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
325 SDNode *CallEnd = CallEndVal.getNode();
326
327 // Get a call instruction from the call sequence chain. Tail calls are not
328 // allowed. The following code is essentially reverse engineering X86's
329 // LowerCallTo.
330 //
331 // We are expecting DAG to have the following form:
332 //
333 // ch = eh_label (only in case of invoke statepoint)
334 // ch, glue = callseq_start ch
335 // ch, glue = X86::Call ch, glue
336 // ch, glue = callseq_end ch, glue
337 // get_return_value ch, glue
338 //
339 // get_return_value can either be a sequence of CopyFromReg instructions
340 // to grab the return value from the return register(s), or it can be a LOAD
341 // to load a value returned by reference via a stack slot.
342
343 bool HasDef = !SI.CLI.RetTy->isVoidTy();
344 if (HasDef) {
345 if (CallEnd->getOpcode() == ISD::LOAD)
346 CallEnd = CallEnd->getOperand(0).getNode();
347 else
348 while (CallEnd->getOpcode() == ISD::CopyFromReg)
349 CallEnd = CallEnd->getOperand(0).getNode();
350 }
351
352 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
353 return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
354}
355
357 FrameIndexSDNode &FI) {
358 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI.getIndex());
359 auto MMOFlags = MachineMemOperand::MOStore |
361 auto &MFI = MF.getFrameInfo();
362 return MF.getMachineMemOperand(PtrInfo, MMOFlags,
363 MFI.getObjectSize(FI.getIndex()),
364 MFI.getObjectAlign(FI.getIndex()));
365}
366
367/// Spill a value incoming to the statepoint. It might be either part of
368/// vmstate
369/// or gcstate. In both cases unconditionally spill it on the stack unless it
370/// is a null constant. Return pair with first element being frame index
371/// containing saved value and second element with outgoing chain from the
372/// emitted store
373static std::tuple<SDValue, SDValue, MachineMemOperand*>
375 SelectionDAGBuilder &Builder) {
376 SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
377 MachineMemOperand* MMO = nullptr;
378
379 // Emit new store if we didn't do it for this ptr before
380 if (!Loc.getNode()) {
381 Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
382 Builder);
383 int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
384 // We use TargetFrameIndex so that isel will not select it into LEA
385 Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
386
387 // Right now we always allocate spill slots that are of the same
388 // size as the value we're about to spill (the size of spillee can
389 // vary since we spill vectors of pointers too). At some point we
390 // can consider allowing spills of smaller values to larger slots
391 // (i.e. change the '==' in the assert below to a '>=').
392 MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
393 assert((MFI.getObjectSize(Index) * 8) ==
394 (-8 & (7 + // Round up modulo 8.
395 (int64_t)Incoming.getValueSizeInBits())) &&
396 "Bad spill: stack slot does not match!");
397
398 // Note: Using the alignment of the spill slot (rather than the abi or
399 // preferred alignment) is required for correctness when dealing with spill
400 // slots with preferred alignments larger than frame alignment..
401 auto &MF = Builder.DAG.getMachineFunction();
402 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
403 auto *StoreMMO = MF.getMachineMemOperand(
405 MFI.getObjectAlign(Index));
406 Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
407 StoreMMO);
408
409 MMO = getMachineMemOperand(MF, *cast<FrameIndexSDNode>(Loc));
410
411 Builder.StatepointLowering.setLocation(Incoming, Loc);
412 }
413
414 assert(Loc.getNode());
415 return std::make_tuple(Loc, Chain, MMO);
416}
417
418/// Lower a single value incoming to a statepoint node. This value can be
419/// either a deopt value or a gc value, the handling is the same. We special
420/// case constants and allocas, then fall back to spilling if required.
421static void
422lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot,
425 SelectionDAGBuilder &Builder) {
426
427 if (willLowerDirectly(Incoming)) {
428 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
429 // This handles allocas as arguments to the statepoint (this is only
430 // really meaningful for a deopt value. For GC, we'd be trying to
431 // relocate the address of the alloca itself?)
432 assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
433 "Incoming value is a frame index!");
434 Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
435 Builder.getFrameIndexTy()));
436
437 auto &MF = Builder.DAG.getMachineFunction();
438 auto *MMO = getMachineMemOperand(MF, *FI);
439 MemRefs.push_back(MMO);
440 return;
441 }
442
443 assert(Incoming.getValueType().getSizeInBits() <= 64);
444
445 if (Incoming.isUndef()) {
446 // Put an easily recognized constant that's unlikely to be a valid
447 // value so that uses of undef by the consumer of the stackmap is
448 // easily recognized. This is legal since the compiler is always
449 // allowed to chose an arbitrary value for undef.
450 pushStackMapConstant(Ops, Builder, 0xFEFEFEFE);
451 return;
452 }
453
454 // If the original value was a constant, make sure it gets recorded as
455 // such in the stackmap. This is required so that the consumer can
456 // parse any internal format to the deopt state. It also handles null
457 // pointers and other constant pointers in GC states.
458 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
459 pushStackMapConstant(Ops, Builder, C->getSExtValue());
460 return;
461 } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Incoming)) {
463 C->getValueAPF().bitcastToAPInt().getZExtValue());
464 return;
465 }
466
467 llvm_unreachable("unhandled direct lowering case");
468 }
469
470
471
472 if (!RequireSpillSlot) {
473 // If this value is live in (not live-on-return, or live-through), we can
474 // treat it the same way patchpoint treats it's "live in" values. We'll
475 // end up folding some of these into stack references, but they'll be
476 // handled by the register allocator. Note that we do not have the notion
477 // of a late use so these values might be placed in registers which are
478 // clobbered by the call. This is fine for live-in. For live-through
479 // fix-up pass should be executed to force spilling of such registers.
480 Ops.push_back(Incoming);
481 } else {
482 // Otherwise, locate a spill slot and explicitly spill it so it can be
483 // found by the runtime later. Note: We know all of these spills are
484 // independent, but don't bother to exploit that chain wise. DAGCombine
485 // will happily do so as needed, so doing it here would be a small compile
486 // time win at most.
487 SDValue Chain = Builder.getRoot();
488 auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
489 Ops.push_back(std::get<0>(Res));
490 if (auto *MMO = std::get<2>(Res))
491 MemRefs.push_back(MMO);
492 Chain = std::get<1>(Res);
493 Builder.DAG.setRoot(Chain);
494 }
495
496}
497
498/// Return true if value V represents the GC value. The behavior is conservative
499/// in case it is not sure that value is not GC the function returns true.
500static bool isGCValue(const Value *V, SelectionDAGBuilder &Builder) {
501 auto *Ty = V->getType();
502 if (!Ty->isPtrOrPtrVectorTy())
503 return false;
504 if (auto *GFI = Builder.GFI)
505 if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
506 return *IsManaged;
507 return true; // conservative
508}
509
510/// Lower deopt state and gc pointer arguments of the statepoint. The actual
511/// lowering is described in lowerIncomingStatepointValue. This function is
512/// responsible for lowering everything in the right position and playing some
513/// tricks to avoid redundant stack manipulation where possible. On
514/// completion, 'Ops' will contain ready to use operands for machine code
515/// statepoint. The chain nodes will have already been created and the DAG root
516/// will be set to the last value spilled (if any were).
517static void
521 DenseMap<SDValue, int> &LowerAsVReg,
523 SelectionDAGBuilder &Builder) {
524 // Lower the deopt and gc arguments for this statepoint. Layout will be:
525 // deopt argument length, deopt arguments.., gc arguments...
526
527 // Figure out what lowering strategy we're going to use for each part
528 // Note: It is conservatively correct to lower both "live-in" and "live-out"
529 // as "live-through". A "live-through" variable is one which is "live-in",
530 // "live-out", and live throughout the lifetime of the call (i.e. we can find
531 // it from any PC within the transitive callee of the statepoint). In
532 // particular, if the callee spills callee preserved registers we may not
533 // be able to find a value placed in that register during the call. This is
534 // fine for live-out, but not for live-through. If we were willing to make
535 // assumptions about the code generator producing the callee, we could
536 // potentially allow live-through values in callee saved registers.
537 const bool LiveInDeopt =
538 SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
539
540 // Decide which deriver pointers will go on VRegs
541 unsigned MaxVRegPtrs = MaxRegistersForGCPointers.getValue();
542
543 // Pointers used on exceptional path of invoke statepoint.
544 // We cannot assing them to VRegs.
545 SmallSet<SDValue, 8> LPadPointers;
547 if (const auto *StInvoke =
548 dyn_cast_or_null<InvokeInst>(SI.StatepointInstr)) {
549 LandingPadInst *LPI = StInvoke->getLandingPadInst();
550 for (const auto *Relocate : SI.GCRelocates)
551 if (Relocate->getOperand(0) == LPI) {
552 LPadPointers.insert(Builder.getValue(Relocate->getBasePtr()));
553 LPadPointers.insert(Builder.getValue(Relocate->getDerivedPtr()));
554 }
555 }
556
557 LLVM_DEBUG(dbgs() << "Deciding how to lower GC Pointers:\n");
558
559 // List of unique lowered GC Pointer values.
560 SmallSetVector<SDValue, 16> LoweredGCPtrs;
561 // Map lowered GC Pointer value to the index in above vector
562 DenseMap<SDValue, unsigned> GCPtrIndexMap;
563
564 unsigned CurNumVRegs = 0;
565
566 auto canPassGCPtrOnVReg = [&](SDValue SD) {
567 if (SD.getValueType().isVector())
568 return false;
569 if (LPadPointers.count(SD))
570 return false;
571 return !willLowerDirectly(SD);
572 };
573
574 auto processGCPtr = [&](const Value *V) {
575 SDValue PtrSD = Builder.getValue(V);
576 if (!LoweredGCPtrs.insert(PtrSD))
577 return; // skip duplicates
578 GCPtrIndexMap[PtrSD] = LoweredGCPtrs.size() - 1;
579
580 assert(!LowerAsVReg.count(PtrSD) && "must not have been seen");
581 if (LowerAsVReg.size() == MaxVRegPtrs)
582 return;
583 assert(V->getType()->isVectorTy() == PtrSD.getValueType().isVector() &&
584 "IR and SD types disagree");
585 if (!canPassGCPtrOnVReg(PtrSD)) {
586 LLVM_DEBUG(dbgs() << "direct/spill "; PtrSD.dump(&Builder.DAG));
587 return;
588 }
589 LLVM_DEBUG(dbgs() << "vreg "; PtrSD.dump(&Builder.DAG));
590 LowerAsVReg[PtrSD] = CurNumVRegs++;
591 };
592
593 // Process derived pointers first to give them more chance to go on VReg.
594 for (const Value *V : SI.Ptrs)
595 processGCPtr(V);
596 for (const Value *V : SI.Bases)
597 processGCPtr(V);
598
599 LLVM_DEBUG(dbgs() << LowerAsVReg.size() << " pointers will go in vregs\n");
600
601 auto requireSpillSlot = [&](const Value *V) {
602 if (!Builder.DAG.getTargetLoweringInfo().isTypeLegal(
603 Builder.getValue(V).getValueType()))
604 return true;
605 if (isGCValue(V, Builder))
606 return !LowerAsVReg.count(Builder.getValue(V));
607 return !(LiveInDeopt || UseRegistersForDeoptValues);
608 };
609
610 // Before we actually start lowering (and allocating spill slots for values),
611 // reserve any stack slots which we judge to be profitable to reuse for a
612 // particular value. This is purely an optimization over the code below and
613 // doesn't change semantics at all. It is important for performance that we
614 // reserve slots for both deopt and gc values before lowering either.
615 for (const Value *V : SI.DeoptState) {
616 if (requireSpillSlot(V))
618 }
619
620 for (const Value *V : SI.Ptrs) {
621 SDValue SDV = Builder.getValue(V);
622 if (!LowerAsVReg.count(SDV))
624 }
625
626 for (const Value *V : SI.Bases) {
627 SDValue SDV = Builder.getValue(V);
628 if (!LowerAsVReg.count(SDV))
630 }
631
632 // First, prefix the list with the number of unique values to be
633 // lowered. Note that this is the number of *Values* not the
634 // number of SDValues required to lower them.
635 const int NumVMSArgs = SI.DeoptState.size();
636 pushStackMapConstant(Ops, Builder, NumVMSArgs);
637
638 // The vm state arguments are lowered in an opaque manner. We do not know
639 // what type of values are contained within.
640 LLVM_DEBUG(dbgs() << "Lowering deopt state\n");
641 for (const Value *V : SI.DeoptState) {
642 SDValue Incoming;
643 // If this is a function argument at a static frame index, generate it as
644 // the frame index.
645 if (const Argument *Arg = dyn_cast<Argument>(V)) {
646 int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);
647 if (FI != INT_MAX)
648 Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());
649 }
650 if (!Incoming.getNode())
651 Incoming = Builder.getValue(V);
652 LLVM_DEBUG(dbgs() << "Value " << *V
653 << " requireSpillSlot = " << requireSpillSlot(V) << "\n");
654 lowerIncomingStatepointValue(Incoming, requireSpillSlot(V), Ops, MemRefs,
655 Builder);
656 }
657
658 // Finally, go ahead and lower all the gc arguments.
659 pushStackMapConstant(Ops, Builder, LoweredGCPtrs.size());
660 for (SDValue SDV : LoweredGCPtrs)
661 lowerIncomingStatepointValue(SDV, !LowerAsVReg.count(SDV), Ops, MemRefs,
662 Builder);
663
664 // Copy to out vector. LoweredGCPtrs will be empty after this point.
665 GCPtrs = LoweredGCPtrs.takeVector();
666
667 // If there are any explicit spill slots passed to the statepoint, record
668 // them, but otherwise do not do anything special. These are user provided
669 // allocas and give control over placement to the consumer. In this case,
670 // it is the contents of the slot which may get updated, not the pointer to
671 // the alloca
673 for (Value *V : SI.GCArgs) {
674 SDValue Incoming = Builder.getValue(V);
675 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
676 // This handles allocas as arguments to the statepoint
677 assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
678 "Incoming value is a frame index!");
679 Allocas.push_back(Builder.DAG.getTargetFrameIndex(
680 FI->getIndex(), Builder.getFrameIndexTy()));
681
682 auto &MF = Builder.DAG.getMachineFunction();
683 auto *MMO = getMachineMemOperand(MF, *FI);
684 MemRefs.push_back(MMO);
685 }
686 }
687 pushStackMapConstant(Ops, Builder, Allocas.size());
688 Ops.append(Allocas.begin(), Allocas.end());
689
690 // Now construct GC base/derived map;
691 pushStackMapConstant(Ops, Builder, SI.Ptrs.size());
692 SDLoc L = Builder.getCurSDLoc();
693 for (unsigned i = 0; i < SI.Ptrs.size(); ++i) {
694 SDValue Base = Builder.getValue(SI.Bases[i]);
695 assert(GCPtrIndexMap.count(Base) && "base not found in index map");
696 Ops.push_back(
697 Builder.DAG.getTargetConstant(GCPtrIndexMap[Base], L, MVT::i64));
698 SDValue Derived = Builder.getValue(SI.Ptrs[i]);
699 assert(GCPtrIndexMap.count(Derived) && "derived not found in index map");
700 Ops.push_back(
701 Builder.DAG.getTargetConstant(GCPtrIndexMap[Derived], L, MVT::i64));
702 }
703}
704
707 // The basic scheme here is that information about both the original call and
708 // the safepoint is encoded in the CallInst. We create a temporary call and
709 // lower it, then reverse engineer the calling sequence.
710
711 NumOfStatepoints++;
712 // Clear state
714 assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
715 assert((GFI || SI.Bases.empty()) &&
716 "No gc specified, so cannot relocate pointers!");
717
718 LLVM_DEBUG(if (SI.StatepointInstr) dbgs()
719 << "Lowering statepoint " << *SI.StatepointInstr << "\n");
720#ifndef NDEBUG
721 for (const auto *Reloc : SI.GCRelocates)
722 if (Reloc->getParent() == SI.StatepointInstr->getParent())
724#endif
725
726 // Lower statepoint vmstate and gcstate arguments
727
728 // All lowered meta args.
729 SmallVector<SDValue, 10> LoweredMetaArgs;
730 // Lowered GC pointers (subset of above).
731 SmallVector<SDValue, 16> LoweredGCArgs;
733 // Maps derived pointer SDValue to statepoint result of relocated pointer.
734 DenseMap<SDValue, int> LowerAsVReg;
735 lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, LoweredGCArgs, LowerAsVReg,
736 SI, *this);
737
738 // Now that we've emitted the spills, we need to update the root so that the
739 // call sequence is ordered correctly.
740 SI.CLI.setChain(getRoot());
741
742 // Get call node, we will replace it later with statepoint
743 SDValue ReturnVal;
744 SDNode *CallNode;
745 std::tie(ReturnVal, CallNode) = lowerCallFromStatepointLoweringInfo(SI, *this);
746
747 // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
748 // nodes with all the appropriate arguments and return values.
749
750 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
751 SDValue Chain = CallNode->getOperand(0);
752
753 SDValue Glue;
754 bool CallHasIncomingGlue = CallNode->getGluedNode();
755 if (CallHasIncomingGlue) {
756 // Glue is always last operand
757 Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
758 }
759
760 // Build the GC_TRANSITION_START node if necessary.
761 //
762 // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
763 // order in which they appear in the call to the statepoint intrinsic. If
764 // any of the operands is a pointer-typed, that operand is immediately
765 // followed by a SRCVALUE for the pointer that may be used during lowering
766 // (e.g. to form MachinePointerInfo values for loads/stores).
767 const bool IsGCTransition =
768 (SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) ==
770 if (IsGCTransition) {
772
773 // Add chain
774 TSOps.push_back(Chain);
775
776 // Add GC transition arguments
777 for (const Value *V : SI.GCTransitionArgs) {
778 TSOps.push_back(getValue(V));
779 if (V->getType()->isPointerTy())
780 TSOps.push_back(DAG.getSrcValue(V));
781 }
782
783 // Add glue if necessary
784 if (CallHasIncomingGlue)
785 TSOps.push_back(Glue);
786
787 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
788
789 SDValue GCTransitionStart =
791
792 Chain = GCTransitionStart.getValue(0);
793 Glue = GCTransitionStart.getValue(1);
794 }
795
796 // TODO: Currently, all of these operands are being marked as read/write in
797 // PrologEpilougeInserter.cpp, we should special case the VMState arguments
798 // and flags to be read-only.
800
801 // Add the <id> and <numBytes> constants.
802 Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
803 Ops.push_back(
804 DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
805
806 // Calculate and push starting position of vmstate arguments
807 // Get number of arguments incoming directly into call node
808 unsigned NumCallRegArgs =
809 CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
810 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
811
812 // Add call target
813 SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
814 Ops.push_back(CallTarget);
815
816 // Add call arguments
817 // Get position of register mask in the call
818 SDNode::op_iterator RegMaskIt;
819 if (CallHasIncomingGlue)
820 RegMaskIt = CallNode->op_end() - 2;
821 else
822 RegMaskIt = CallNode->op_end() - 1;
823 Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
824
825 // Add a constant argument for the calling convention
826 pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
827
828 // Add a constant argument for the flags
829 uint64_t Flags = SI.StatepointFlags;
830 assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
831 "Unknown flag used");
832 pushStackMapConstant(Ops, *this, Flags);
833
834 // Insert all vmstate and gcstate arguments
835 llvm::append_range(Ops, LoweredMetaArgs);
836
837 // Add register mask from call node
838 Ops.push_back(*RegMaskIt);
839
840 // Add chain
841 Ops.push_back(Chain);
842
843 // Same for the glue, but we add it only if original call had it
844 if (Glue.getNode())
845 Ops.push_back(Glue);
846
847 // Compute return values. Provide a glue output since we consume one as
848 // input. This allows someone else to chain off us as needed.
849 SmallVector<EVT, 8> NodeTys;
850 for (auto SD : LoweredGCArgs) {
851 if (!LowerAsVReg.count(SD))
852 continue;
853 NodeTys.push_back(SD.getValueType());
854 }
855 LLVM_DEBUG(dbgs() << "Statepoint has " << NodeTys.size() << " results\n");
856 assert(NodeTys.size() == LowerAsVReg.size() && "Inconsistent GC Ptr lowering");
857 NodeTys.push_back(MVT::Other);
858 NodeTys.push_back(MVT::Glue);
859
860 unsigned NumResults = NodeTys.size();
861 MachineSDNode *StatepointMCNode =
862 DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
863 DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
864
865 // For values lowered to tied-defs, create the virtual registers if used
866 // in other blocks. For local gc.relocate record appropriate statepoint
867 // result in StatepointLoweringState.
869 for (const auto *Relocate : SI.GCRelocates) {
870 Value *Derived = Relocate->getDerivedPtr();
871 SDValue SD = getValue(Derived);
872 if (!LowerAsVReg.count(SD))
873 continue;
874
875 SDValue Relocated = SDValue(StatepointMCNode, LowerAsVReg[SD]);
876
877 // Handle local relocate. Note that different relocates might
878 // map to the same SDValue.
879 if (SI.StatepointInstr->getParent() == Relocate->getParent()) {
881 if (Res)
882 assert(Res == Relocated);
883 else
884 StatepointLowering.setLocation(SD, Relocated);
885 continue;
886 }
887
888 // Handle multiple gc.relocates of the same input efficiently.
889 if (VirtRegs.count(SD))
890 continue;
891
892 auto *RetTy = Relocate->getType();
895 DAG.getDataLayout(), Reg, RetTy, std::nullopt);
896 SDValue Chain = DAG.getRoot();
897 RFV.getCopyToRegs(Relocated, DAG, getCurSDLoc(), Chain, nullptr);
898 PendingExports.push_back(Chain);
899
900 VirtRegs[SD] = Reg;
901 }
902
903 // Record for later use how each relocation was lowered. This is needed to
904 // allow later gc.relocates to mirror the lowering chosen.
905 const Instruction *StatepointInstr = SI.StatepointInstr;
906 auto &RelocationMap = FuncInfo.StatepointRelocationMaps[StatepointInstr];
907 for (const GCRelocateInst *Relocate : SI.GCRelocates) {
908 const Value *V = Relocate->getDerivedPtr();
909 SDValue SDV = getValue(V);
911
912 bool IsLocal = (Relocate->getParent() == StatepointInstr->getParent());
913
915 if (IsLocal && LowerAsVReg.count(SDV)) {
916 // Result is already stored in StatepointLowering
918 } else if (LowerAsVReg.count(SDV)) {
920 assert(VirtRegs.count(SDV));
921 Record.payload.Reg = VirtRegs[SDV];
922 } else if (Loc.getNode()) {
924 Record.payload.FI = cast<FrameIndexSDNode>(Loc)->getIndex();
925 } else {
927 // If we didn't relocate a value, we'll essentialy end up inserting an
928 // additional use of the original value when lowering the gc.relocate.
929 // We need to make sure the value is available at the new use, which
930 // might be in another block.
931 if (Relocate->getParent() != StatepointInstr->getParent())
933 }
934 RelocationMap[Relocate] = Record;
935 }
936
937
938
939 SDNode *SinkNode = StatepointMCNode;
940
941 // Build the GC_TRANSITION_END node if necessary.
942 //
943 // See the comment above regarding GC_TRANSITION_START for the layout of
944 // the operands to the GC_TRANSITION_END node.
945 if (IsGCTransition) {
947
948 // Add chain
949 TEOps.push_back(SDValue(StatepointMCNode, NumResults - 2));
950
951 // Add GC transition arguments
952 for (const Value *V : SI.GCTransitionArgs) {
953 TEOps.push_back(getValue(V));
954 if (V->getType()->isPointerTy())
955 TEOps.push_back(DAG.getSrcValue(V));
956 }
957
958 // Add glue
959 TEOps.push_back(SDValue(StatepointMCNode, NumResults - 1));
960
961 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
962
963 SDValue GCTransitionStart =
964 DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
965
966 SinkNode = GCTransitionStart.getNode();
967 }
968
969 // Replace original call
970 // Call: ch,glue = CALL ...
971 // Statepoint: [gc relocates],ch,glue = STATEPOINT ...
972 unsigned NumSinkValues = SinkNode->getNumValues();
973 SDValue StatepointValues[2] = {SDValue(SinkNode, NumSinkValues - 2),
974 SDValue(SinkNode, NumSinkValues - 1)};
975 DAG.ReplaceAllUsesWith(CallNode, StatepointValues);
976 // Remove original call node
977 DAG.DeleteNode(CallNode);
978
979 // Since we always emit CopyToRegs (even for local relocates), we must
980 // update root, so that they are emitted before any local uses.
981 (void)getControlRoot();
982
983 // TODO: A better future implementation would be to emit a single variable
984 // argument, variable return value STATEPOINT node here and then hookup the
985 // return value of each gc.relocate to the respective output of the
986 // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
987 // to actually be possible today.
988
989 return ReturnVal;
990}
991
992/// Return two gc.results if present. First result is a block local
993/// gc.result, second result is a non-block local gc.result. Corresponding
994/// entry will be nullptr if not present.
995static std::pair<const GCResultInst*, const GCResultInst*>
997 std::pair<const GCResultInst *, const GCResultInst*> Res(nullptr, nullptr);
998 for (const auto *U : S.users()) {
999 auto *GRI = dyn_cast<GCResultInst>(U);
1000 if (!GRI)
1001 continue;
1002 if (GRI->getParent() == S.getParent())
1003 Res.first = GRI;
1004 else
1005 Res.second = GRI;
1006 }
1007 return Res;
1008}
1009
1010void
1012 const BasicBlock *EHPadBB /*= nullptr*/) {
1013 assert(I.getCallingConv() != CallingConv::AnyReg &&
1014 "anyregcc is not supported on statepoints!");
1015
1016#ifndef NDEBUG
1017 // Check that the associated GCStrategy expects to encounter statepoints.
1019 "GCStrategy does not expect to encounter statepoints");
1020#endif
1021
1022 SDValue ActualCallee;
1023 SDValue Callee = getValue(I.getActualCalledOperand());
1024
1025 if (I.getNumPatchBytes() > 0) {
1026 // If we've been asked to emit a nop sequence instead of a call instruction
1027 // for this statepoint then don't lower the call target, but use a constant
1028 // `undef` instead. Not lowering the call target lets statepoint clients
1029 // get away without providing a physical address for the symbolic call
1030 // target at link time.
1031 ActualCallee = DAG.getUNDEF(Callee.getValueType());
1032 } else {
1033 ActualCallee = Callee;
1034 }
1035
1038 I.getNumCallArgs(), ActualCallee,
1039 I.getActualReturnType(), false /* IsPatchPoint */);
1040
1041 // There may be duplication in the gc.relocate list; such as two copies of
1042 // each relocation on normal and exceptional path for an invoke. We only
1043 // need to spill once and record one copy in the stackmap, but we need to
1044 // reload once per gc.relocate. (Dedupping gc.relocates is trickier and best
1045 // handled as a CSE problem elsewhere.)
1046 // TODO: There a couple of major stackmap size optimizations we could do
1047 // here if we wished.
1048 // 1) If we've encountered a derived pair {B, D}, we don't need to actually
1049 // record {B,B} if it's seen later.
1050 // 2) Due to rematerialization, actual derived pointers are somewhat rare;
1051 // given that, we could change the format to record base pointer relocations
1052 // separately with half the space. This would require a format rev and a
1053 // fairly major rework of the STATEPOINT node though.
1055 for (const GCRelocateInst *Relocate : I.getGCRelocates()) {
1056 SI.GCRelocates.push_back(Relocate);
1057
1058 SDValue DerivedSD = getValue(Relocate->getDerivedPtr());
1059 if (Seen.insert(DerivedSD).second) {
1060 SI.Bases.push_back(Relocate->getBasePtr());
1061 SI.Ptrs.push_back(Relocate->getDerivedPtr());
1062 }
1063 }
1064
1065 // If we find a deopt value which isn't explicitly added, we need to
1066 // ensure it gets lowered such that gc cycles occurring before the
1067 // deoptimization event during the lifetime of the call don't invalidate
1068 // the pointer we're deopting with. Note that we assume that all
1069 // pointers passed to deopt are base pointers; relaxing that assumption
1070 // would require relatively large changes to how we represent relocations.
1071 for (Value *V : I.deopt_operands()) {
1072 if (!isGCValue(V, *this))
1073 continue;
1074 if (Seen.insert(getValue(V)).second) {
1075 SI.Bases.push_back(V);
1076 SI.Ptrs.push_back(V);
1077 }
1078 }
1079
1080 SI.GCArgs = ArrayRef<const Use>(I.gc_args_begin(), I.gc_args_end());
1081 SI.StatepointInstr = &I;
1082 SI.ID = I.getID();
1083
1084 SI.DeoptState = ArrayRef<const Use>(I.deopt_begin(), I.deopt_end());
1085 SI.GCTransitionArgs = ArrayRef<const Use>(I.gc_transition_args_begin(),
1086 I.gc_transition_args_end());
1087
1088 SI.StatepointFlags = I.getFlags();
1089 SI.NumPatchBytes = I.getNumPatchBytes();
1090 SI.EHPadBB = EHPadBB;
1091
1092 SDValue ReturnValue = LowerAsSTATEPOINT(SI);
1093
1094 // Export the result value if needed
1095 const auto GCResultLocality = getGCResultLocality(I);
1096
1097 if (!GCResultLocality.first && !GCResultLocality.second) {
1098 // The return value is not needed, just generate a poison value.
1099 // Note: This covers the void return case.
1101 return;
1102 }
1103
1104 if (GCResultLocality.first) {
1105 // Result value will be used in a same basic block. Don't export it or
1106 // perform any explicit register copies. The gc_result will simply grab
1107 // this value.
1108 setValue(&I, ReturnValue);
1109 }
1110
1111 if (!GCResultLocality.second)
1112 return;
1113 // Result value will be used in a different basic block so we need to export
1114 // it now. Default exporting mechanism will not work here because statepoint
1115 // call has a different type than the actual call. It means that by default
1116 // llvm will create export register of the wrong type (always i32 in our
1117 // case). So instead we need to create export register with correct type
1118 // manually.
1119 // TODO: To eliminate this problem we can remove gc.result intrinsics
1120 // completely and make statepoint call to return a tuple.
1121 Type *RetTy = GCResultLocality.second->getType();
1124 DAG.getDataLayout(), Reg, RetTy,
1125 I.getCallingConv());
1126 SDValue Chain = DAG.getEntryNode();
1127
1128 RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
1129 PendingExports.push_back(Chain);
1130 FuncInfo.ValueMap[&I] = Reg;
1131}
1132
1134 const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
1135 bool VarArgDisallowed, bool ForceVoidReturnTy) {
1137 unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
1139 SI.CLI, Call, ArgBeginIndex, Call->arg_size(), Callee,
1140 ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
1141 false);
1142 if (!VarArgDisallowed)
1143 SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
1144
1145 auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
1146
1148
1149 auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes());
1150 SI.ID = SD.StatepointID.value_or(DefaultID);
1151 SI.NumPatchBytes = SD.NumPatchBytes.value_or(0);
1152
1153 SI.DeoptState =
1154 ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
1155 SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
1156 SI.EHPadBB = EHPadBB;
1157
1158 // NB! The GC arguments are deliberately left empty.
1159
1160 LLVM_DEBUG(dbgs() << "Lowering call with deopt bundle " << *Call << "\n");
1161 if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
1162 ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
1163 setValue(Call, ReturnVal);
1164 }
1165}
1166
1168 const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
1169 LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
1170 /* VarArgDisallowed = */ false,
1171 /* ForceVoidReturnTy = */ false);
1172}
1173
1174void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
1175 // The result value of the gc_result is simply the result of the actual
1176 // call. We've already emitted this, so just grab the value.
1177 const Value *SI = CI.getStatepoint();
1178 assert((isa<GCStatepointInst>(SI) || isa<UndefValue>(SI)) &&
1179 "GetStatepoint must return one of two types");
1180 if (isa<UndefValue>(SI))
1181 return;
1182
1183 if (cast<GCStatepointInst>(SI)->getParent() == CI.getParent()) {
1184 setValue(&CI, getValue(SI));
1185 return;
1186 }
1187 // Statepoint is in different basic block so we should have stored call
1188 // result in a virtual register.
1189 // We can not use default getValue() functionality to copy value from this
1190 // register because statepoint and actual call return types can be
1191 // different, and getValue() will use CopyFromReg of the wrong type,
1192 // which is always i32 in our case.
1193 Type *RetTy = CI.getType();
1194 SDValue CopyFromReg = getCopyFromRegs(SI, RetTy);
1195
1196 assert(CopyFromReg.getNode());
1197 setValue(&CI, CopyFromReg);
1198}
1199
1200void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
1201 const Value *Statepoint = Relocate.getStatepoint();
1202#ifndef NDEBUG
1203 // Consistency check
1204 // We skip this check for relocates not in the same basic block as their
1205 // statepoint. It would be too expensive to preserve validation info through
1206 // different basic blocks.
1207 assert((isa<GCStatepointInst>(Statepoint) || isa<UndefValue>(Statepoint)) &&
1208 "GetStatepoint must return one of two types");
1209 if (isa<UndefValue>(Statepoint))
1210 return;
1211
1212 if (cast<GCStatepointInst>(Statepoint)->getParent() == Relocate.getParent())
1214#endif
1215
1216 const Value *DerivedPtr = Relocate.getDerivedPtr();
1217 auto &RelocationMap =
1218 FuncInfo.StatepointRelocationMaps[cast<GCStatepointInst>(Statepoint)];
1219 auto SlotIt = RelocationMap.find(&Relocate);
1220 assert(SlotIt != RelocationMap.end() && "Relocating not lowered gc value");
1221 const RecordType &Record = SlotIt->second;
1222
1223 // If relocation was done via virtual register..
1224 if (Record.type == RecordType::SDValueNode) {
1225 assert(cast<GCStatepointInst>(Statepoint)->getParent() ==
1226 Relocate.getParent() &&
1227 "Nonlocal gc.relocate mapped via SDValue");
1229 assert(SDV.getNode() && "empty SDValue");
1230 setValue(&Relocate, SDV);
1231 return;
1232 }
1233 if (Record.type == RecordType::VReg) {
1234 Register InReg = Record.payload.Reg;
1236 DAG.getDataLayout(), InReg, Relocate.getType(),
1237 std::nullopt); // This is not an ABI copy.
1238 // We generate copy to/from regs even for local uses, hence we must
1239 // chain with current root to ensure proper ordering of copies w.r.t.
1240 // statepoint.
1241 SDValue Chain = DAG.getRoot();
1242 SDValue Relocation = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
1243 Chain, nullptr, nullptr);
1244 setValue(&Relocate, Relocation);
1245 return;
1246 }
1247
1248 if (Record.type == RecordType::Spill) {
1249 unsigned Index = Record.payload.FI;
1251
1252 // All the reloads are independent and are reading memory only modified by
1253 // statepoints (i.e. no other aliasing stores); informing SelectionDAG of
1254 // this lets CSE kick in for free and allows reordering of
1255 // instructions if possible. The lowering for statepoint sets the root,
1256 // so this is ordering all reloads with the either
1257 // a) the statepoint node itself, or
1258 // b) the entry of the current block for an invoke statepoint.
1259 const SDValue Chain = DAG.getRoot(); // != Builder.getRoot()
1260
1261 auto &MF = DAG.getMachineFunction();
1262 auto &MFI = MF.getFrameInfo();
1263 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
1264 auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
1265 MFI.getObjectSize(Index),
1266 MFI.getObjectAlign(Index));
1267
1269 Relocate.getType());
1270
1271 SDValue SpillLoad =
1272 DAG.getLoad(LoadVT, getCurSDLoc(), Chain, SpillSlot, LoadMMO);
1273 PendingLoads.push_back(SpillLoad.getValue(1));
1274
1275 assert(SpillLoad.getNode());
1276 setValue(&Relocate, SpillLoad);
1277 return;
1278 }
1279
1281 SDValue SD = getValue(DerivedPtr);
1282
1283 if (SD.isUndef() && SD.getValueType().getSizeInBits() <= 64) {
1284 // Lowering relocate(undef) as arbitrary constant. Current constant value
1285 // is chosen such that it's unlikely to be a valid pointer.
1286 setValue(&Relocate, DAG.getTargetConstant(0xFEFEFEFE, SDLoc(SD), MVT::i64));
1287 return;
1288 }
1289
1290 // We didn't need to spill these special cases (constants and allocas).
1291 // See the handling in spillIncomingValueForStatepoint for detail.
1292 setValue(&Relocate, SD);
1293}
1294
1296 const auto &TLI = DAG.getTargetLoweringInfo();
1297 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1298 TLI.getPointerTy(DAG.getDataLayout()));
1299
1300 // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1301 // call. We also do not lower the return value to any virtual register, and
1302 // change the immediately following return to a trap instruction.
1303 LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1304 /* VarArgDisallowed = */ true,
1305 /* ForceVoidReturnTy = */ true);
1306}
1307
1309 // We do not lower the return value from llvm.deoptimize to any virtual
1310 // register, and change the immediately following return to a trap
1311 // instruction.
1313 DAG.setRoot(
1314 DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
1315}
assume Assume Builder
static const Function * getParent(const Value *V)
return RetTy
#define LLVM_DEBUG(X)
Definition: Debug.h:101
#define I(x, y, z)
Definition: MD5.cpp:58
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallSet class.
This file defines the SmallVector class.
cl::opt< bool > UseRegistersForGCPointersInLandingPad("use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false), cl::desc("Allow using registers for gc pointer in landing pad"))
static void lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
static std::optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
static bool isGCValue(const Value *V, SelectionDAGBuilder &Builder)
Return true if value V represents the GC value.
static bool willLowerDirectly(SDValue Incoming)
Return true if-and-only-if the given SDValue can be lowered as either a constant argument or a stack ...
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SmallVectorImpl< SDValue > &GCPtrs, DenseMap< SDValue, int > &LowerAsVReg, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
static std::pair< const GCResultInst *, const GCResultInst * > getGCResultLocality(const GCStatepointInst &S)
Return two gc.results if present.
cl::opt< bool > UseRegistersForDeoptValues("use-registers-for-deopt-values", cl::Hidden, cl::init(false), cl::desc("Allow using registers for non pointer deopt args"))
cl::opt< unsigned > MaxRegistersForGCPointers("max-registers-for-gc-values", cl::Hidden, cl::init(0), cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"))
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling.
FunctionLoweringInfo::StatepointRelocationRecord RecordType
static MachineMemOperand * getMachineMemOperand(MachineFunction &MF, FrameIndexSDNode &FI)
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Extract call from statepoint, lower it and return pointer to the call node.
static std::tuple< SDValue, SDValue, MachineMemOperand * > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This file describes how to lower LLVM code to machine code.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1190
This class represents a function call, abstracting a target machine's calling convention.
unsigned size() const
Definition: DenseMap.h:99
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
Register CreateRegs(const Value *V)
DenseMap< const Instruction *, StatepointSpillMapTy > StatepointRelocationMaps
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
GCStrategy & getStrategy()
getStrategy - Return the GC strategy for the function.
Definition: GCMetadata.h:108
const Value * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Represents calls to the gc.relocate intrinsic.
Value * getDerivedPtr() const
Represents calls to the gc.result intrinsic.
Represents a gc.statepoint intrinsic call.
Definition: Statepoint.h:61
bool useStatepoints() const
Returns true if this strategy is expecting the use of gc.statepoints, and false otherwise.
Definition: GCStrategy.h:92
const BasicBlock * getParent() const
Definition: Instruction.h:90
The landingpad instruction holds all of the information necessary to generate correct exception handl...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
op_iterator op_end() const
op_iterator op_begin() const
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)
Lower SLI into a STATEPOINT instruction.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:725
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:674
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
Vector takeVector()
Clear the SetVector and return the underlying vector.
Definition: SetVector.h:87
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
SmallBitVector & set()
bool test(unsigned Idx) const
void clear()
Clear all bits.
size_type size() const
Returns the number of bits in this bitvector.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:809
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
void clear()
Clear the memory usage of this object.
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
void scheduleRelocCall(const GCRelocateInst &RelocCall)
Record the fact that we expect to encounter a given gc_relocate before the next statepoint.
void setLocation(SDValue Val, SDValue Location)
void relocCallVisited(const GCRelocateInst &RelocCall)
Remove this gc_relocate from the list we're expecting to see before the next statepoint.
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
TargetOptions Options
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
iterator_range< user_iterator > users()
Definition: Value.h:421
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ AnyReg
Used for dynamic register based calls (e.g.
Definition: CallingConv.h:60
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1026
@ GC_TRANSITION_START
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:1302
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1197
@ GC_TRANSITION_END
Definition: ISDOpcodes.h:1303
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:440
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1747
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:2037
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:24
@ MaskAll
A bitmask that includes all valid flags.
@ DeoptLiveIn
Mark the deopt arguments associated with the statepoint as only being "live-in".
@ GCTransition
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:240