LLVM 17.0.0git
StatepointLowering.cpp
Go to the documentation of this file.
1//===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file includes support code use by SelectionDAGBuilder when lowering a
10// statepoint sequence in SelectionDAG IR.
11//
12//===----------------------------------------------------------------------===//
13
14#include "StatepointLowering.h"
15#include "SelectionDAGBuilder.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SetVector.h"
20#include "llvm/ADT/SmallSet.h"
22#include "llvm/ADT/Statistic.h"
35#include "llvm/IR/CallingConv.h"
37#include "llvm/IR/GCStrategy.h"
38#include "llvm/IR/Instruction.h"
40#include "llvm/IR/LLVMContext.h"
41#include "llvm/IR/Statepoint.h"
42#include "llvm/IR/Type.h"
48#include <cassert>
49#include <cstddef>
50#include <cstdint>
51#include <iterator>
52#include <tuple>
53#include <utility>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "statepoint-lowering"
58
59STATISTIC(NumSlotsAllocatedForStatepoints,
60 "Number of stack slots allocated for statepoints");
61STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
62STATISTIC(StatepointMaxSlotsRequired,
63 "Maximum number of stack slots required for a singe statepoint");
64
66 "use-registers-for-deopt-values", cl::Hidden, cl::init(false),
67 cl::desc("Allow using registers for non pointer deopt args"));
68
70 "use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false),
71 cl::desc("Allow using registers for gc pointer in landing pad"));
72
74 "max-registers-for-gc-values", cl::Hidden, cl::init(0),
75 cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"));
76
78
81 SDLoc L = Builder.getCurSDLoc();
82 Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
83 MVT::i64));
84 Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
85}
86
88 // Consistency check
89 assert(PendingGCRelocateCalls.empty() &&
90 "Trying to visit statepoint before finished processing previous one");
91 Locations.clear();
92 NextSlotToAllocate = 0;
93 // Need to resize this on each safepoint - we need the two to stay in sync and
94 // the clear patterns of a SelectionDAGBuilder have no relation to
95 // FunctionLoweringInfo. Also need to ensure used bits get cleared.
96 AllocatedStackSlots.clear();
97 AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
98}
99
101 Locations.clear();
102 AllocatedStackSlots.clear();
103 assert(PendingGCRelocateCalls.empty() &&
104 "cleared before statepoint sequence completed");
105}
106
109 SelectionDAGBuilder &Builder) {
110 NumSlotsAllocatedForStatepoints++;
111 MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
112
113 unsigned SpillSize = ValueType.getStoreSize();
114 assert((SpillSize * 8) ==
115 (-8u & (7 + ValueType.getSizeInBits())) && // Round up modulo 8.
116 "Size not in bytes?");
117
118 // First look for a previously created stack slot which is not in
119 // use (accounting for the fact arbitrary slots may already be
120 // reserved), or to create a new stack slot and use it.
121
122 const size_t NumSlots = AllocatedStackSlots.size();
123 assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
124
125 assert(AllocatedStackSlots.size() ==
126 Builder.FuncInfo.StatepointStackSlots.size() &&
127 "Broken invariant");
128
129 for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
130 if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
131 const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
132 if (MFI.getObjectSize(FI) == SpillSize) {
133 AllocatedStackSlots.set(NextSlotToAllocate);
134 // TODO: Is ValueType the right thing to use here?
135 return Builder.DAG.getFrameIndex(FI, ValueType);
136 }
137 }
138 }
139
140 // Couldn't find a free slot, so create a new one:
141
142 SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
143 const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
145
146 Builder.FuncInfo.StatepointStackSlots.push_back(FI);
147 AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
148 assert(AllocatedStackSlots.size() ==
149 Builder.FuncInfo.StatepointStackSlots.size() &&
150 "Broken invariant");
151
152 StatepointMaxSlotsRequired.updateMax(
153 Builder.FuncInfo.StatepointStackSlots.size());
154
155 return SpillSlot;
156}
157
158/// Utility function for reservePreviousStackSlotForValue. Tries to find
159/// stack slot index to which we have spilled value for previous statepoints.
160/// LookUpDepth specifies maximum DFS depth this function is allowed to look.
161static std::optional<int> findPreviousSpillSlot(const Value *Val,
162 SelectionDAGBuilder &Builder,
163 int LookUpDepth) {
164 // Can not look any further - give up now
165 if (LookUpDepth <= 0)
166 return std::nullopt;
167
168 // Spill location is known for gc relocates
169 if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
170 const Value *Statepoint = Relocate->getStatepoint();
171 assert((isa<GCStatepointInst>(Statepoint) || isa<UndefValue>(Statepoint)) &&
172 "GetStatepoint must return one of two types");
173 if (isa<UndefValue>(Statepoint))
174 return std::nullopt;
175
176 const auto &RelocationMap = Builder.FuncInfo.StatepointRelocationMaps
177 [cast<GCStatepointInst>(Statepoint)];
178
179 auto It = RelocationMap.find(Relocate);
180 if (It == RelocationMap.end())
181 return std::nullopt;
182
183 auto &Record = It->second;
184 if (Record.type != RecordType::Spill)
185 return std::nullopt;
186
187 return Record.payload.FI;
188 }
189
190 // Look through bitcast instructions.
191 if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
192 return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
193
194 // Look through phi nodes
195 // All incoming values should have same known stack slot, otherwise result
196 // is unknown.
197 if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
198 std::optional<int> MergedResult;
199
200 for (const auto &IncomingValue : Phi->incoming_values()) {
201 std::optional<int> SpillSlot =
202 findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
203 if (!SpillSlot)
204 return std::nullopt;
205
206 if (MergedResult && *MergedResult != *SpillSlot)
207 return std::nullopt;
208
209 MergedResult = SpillSlot;
210 }
211 return MergedResult;
212 }
213
214 // TODO: We can do better for PHI nodes. In cases like this:
215 // ptr = phi(relocated_pointer, not_relocated_pointer)
216 // statepoint(ptr)
217 // We will return that stack slot for ptr is unknown. And later we might
218 // assign different stack slots for ptr and relocated_pointer. This limits
219 // llvm's ability to remove redundant stores.
220 // Unfortunately it's hard to accomplish in current infrastructure.
221 // We use this function to eliminate spill store completely, while
222 // in example we still need to emit store, but instead of any location
223 // we need to use special "preferred" location.
224
225 // TODO: handle simple updates. If a value is modified and the original
226 // value is no longer live, it would be nice to put the modified value in the
227 // same slot. This allows folding of the memory accesses for some
228 // instructions types (like an increment).
229 // statepoint (i)
230 // i1 = i+1
231 // statepoint (i1)
232 // However we need to be careful for cases like this:
233 // statepoint(i)
234 // i1 = i+1
235 // statepoint(i, i1)
236 // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
237 // put handling of simple modifications in this function like it's done
238 // for bitcasts we might end up reserving i's slot for 'i+1' because order in
239 // which we visit values is unspecified.
240
241 // Don't know any information about this instruction
242 return std::nullopt;
243}
244
245/// Return true if-and-only-if the given SDValue can be lowered as either a
246/// constant argument or a stack reference. The key point is that the value
247/// doesn't need to be spilled or tracked as a vreg use.
248static bool willLowerDirectly(SDValue Incoming) {
249 // We are making an unchecked assumption that the frame size <= 2^16 as that
250 // is the largest offset which can be encoded in the stackmap format.
251 if (isa<FrameIndexSDNode>(Incoming))
252 return true;
253
254 // The largest constant describeable in the StackMap format is 64 bits.
255 // Potential Optimization: Constants values are sign extended by consumer,
256 // and thus there are many constants of static type > 64 bits whose value
257 // happens to be sext(Con64) and could thus be lowered directly.
258 if (Incoming.getValueType().getSizeInBits() > 64)
259 return false;
260
261 return (isa<ConstantSDNode>(Incoming) || isa<ConstantFPSDNode>(Incoming) ||
262 Incoming.isUndef());
263}
264
265/// Try to find existing copies of the incoming values in stack slots used for
266/// statepoint spilling. If we can find a spill slot for the incoming value,
267/// mark that slot as allocated, and reuse the same slot for this safepoint.
268/// This helps to avoid series of loads and stores that only serve to reshuffle
269/// values on the stack between calls.
270static void reservePreviousStackSlotForValue(const Value *IncomingValue,
271 SelectionDAGBuilder &Builder) {
272 SDValue Incoming = Builder.getValue(IncomingValue);
273
274 // If we won't spill this, we don't need to check for previously allocated
275 // stack slots.
276 if (willLowerDirectly(Incoming))
277 return;
278
279 SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
280 if (OldLocation.getNode())
281 // Duplicates in input
282 return;
283
284 const int LookUpDepth = 6;
285 std::optional<int> Index =
286 findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
287 if (!Index)
288 return;
289
290 const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
291
292 auto SlotIt = find(StatepointSlots, *Index);
293 assert(SlotIt != StatepointSlots.end() &&
294 "Value spilled to the unknown stack slot");
295
296 // This is one of our dedicated lowering slots
297 const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
298 if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
299 // stack slot already assigned to someone else, can't use it!
300 // TODO: currently we reserve space for gc arguments after doing
301 // normal allocation for deopt arguments. We should reserve for
302 // _all_ deopt and gc arguments, then start allocating. This
303 // will prevent some moves being inserted when vm state changes,
304 // but gc state doesn't between two calls.
305 return;
306 }
307 // Reserve this stack slot
308 Builder.StatepointLowering.reserveStackSlot(Offset);
309
310 // Cache this slot so we find it when going through the normal
311 // assignment loop.
312 SDValue Loc =
313 Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
314 Builder.StatepointLowering.setLocation(Incoming, Loc);
315}
316
317/// Extract call from statepoint, lower it and return pointer to the
318/// call node. Also update NodeMap so that getValue(statepoint) will
319/// reference lowered call result
320static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
322 SelectionDAGBuilder &Builder) {
323 SDValue ReturnValue, CallEndVal;
324 std::tie(ReturnValue, CallEndVal) =
325 Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
326 SDNode *CallEnd = CallEndVal.getNode();
327
328 // Get a call instruction from the call sequence chain. Tail calls are not
329 // allowed. The following code is essentially reverse engineering X86's
330 // LowerCallTo.
331 //
332 // We are expecting DAG to have the following form:
333 //
334 // ch = eh_label (only in case of invoke statepoint)
335 // ch, glue = callseq_start ch
336 // ch, glue = X86::Call ch, glue
337 // ch, glue = callseq_end ch, glue
338 // get_return_value ch, glue
339 //
340 // get_return_value can either be a sequence of CopyFromReg instructions
341 // to grab the return value from the return register(s), or it can be a LOAD
342 // to load a value returned by reference via a stack slot.
343
344 bool HasDef = !SI.CLI.RetTy->isVoidTy();
345 if (HasDef) {
346 if (CallEnd->getOpcode() == ISD::LOAD)
347 CallEnd = CallEnd->getOperand(0).getNode();
348 else
349 while (CallEnd->getOpcode() == ISD::CopyFromReg)
350 CallEnd = CallEnd->getOperand(0).getNode();
351 }
352
353 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
354 return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
355}
356
358 FrameIndexSDNode &FI) {
359 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI.getIndex());
360 auto MMOFlags = MachineMemOperand::MOStore |
362 auto &MFI = MF.getFrameInfo();
363 return MF.getMachineMemOperand(PtrInfo, MMOFlags,
364 MFI.getObjectSize(FI.getIndex()),
365 MFI.getObjectAlign(FI.getIndex()));
366}
367
368/// Spill a value incoming to the statepoint. It might be either part of
369/// vmstate
370/// or gcstate. In both cases unconditionally spill it on the stack unless it
371/// is a null constant. Return pair with first element being frame index
372/// containing saved value and second element with outgoing chain from the
373/// emitted store
374static std::tuple<SDValue, SDValue, MachineMemOperand*>
376 SelectionDAGBuilder &Builder) {
377 SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
378 MachineMemOperand* MMO = nullptr;
379
380 // Emit new store if we didn't do it for this ptr before
381 if (!Loc.getNode()) {
382 Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
383 Builder);
384 int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
385 // We use TargetFrameIndex so that isel will not select it into LEA
386 Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
387
388 // Right now we always allocate spill slots that are of the same
389 // size as the value we're about to spill (the size of spillee can
390 // vary since we spill vectors of pointers too). At some point we
391 // can consider allowing spills of smaller values to larger slots
392 // (i.e. change the '==' in the assert below to a '>=').
393 MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
394 assert((MFI.getObjectSize(Index) * 8) ==
395 (-8 & (7 + // Round up modulo 8.
396 (int64_t)Incoming.getValueSizeInBits())) &&
397 "Bad spill: stack slot does not match!");
398
399 // Note: Using the alignment of the spill slot (rather than the abi or
400 // preferred alignment) is required for correctness when dealing with spill
401 // slots with preferred alignments larger than frame alignment..
402 auto &MF = Builder.DAG.getMachineFunction();
403 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
404 auto *StoreMMO = MF.getMachineMemOperand(
406 MFI.getObjectAlign(Index));
407 Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
408 StoreMMO);
409
410 MMO = getMachineMemOperand(MF, *cast<FrameIndexSDNode>(Loc));
411
412 Builder.StatepointLowering.setLocation(Incoming, Loc);
413 }
414
415 assert(Loc.getNode());
416 return std::make_tuple(Loc, Chain, MMO);
417}
418
419/// Lower a single value incoming to a statepoint node. This value can be
420/// either a deopt value or a gc value, the handling is the same. We special
421/// case constants and allocas, then fall back to spilling if required.
422static void
423lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot,
426 SelectionDAGBuilder &Builder) {
427
428 if (willLowerDirectly(Incoming)) {
429 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
430 // This handles allocas as arguments to the statepoint (this is only
431 // really meaningful for a deopt value. For GC, we'd be trying to
432 // relocate the address of the alloca itself?)
433 assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
434 "Incoming value is a frame index!");
435 Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
436 Builder.getFrameIndexTy()));
437
438 auto &MF = Builder.DAG.getMachineFunction();
439 auto *MMO = getMachineMemOperand(MF, *FI);
440 MemRefs.push_back(MMO);
441 return;
442 }
443
444 assert(Incoming.getValueType().getSizeInBits() <= 64);
445
446 if (Incoming.isUndef()) {
447 // Put an easily recognized constant that's unlikely to be a valid
448 // value so that uses of undef by the consumer of the stackmap is
449 // easily recognized. This is legal since the compiler is always
450 // allowed to chose an arbitrary value for undef.
451 pushStackMapConstant(Ops, Builder, 0xFEFEFEFE);
452 return;
453 }
454
455 // If the original value was a constant, make sure it gets recorded as
456 // such in the stackmap. This is required so that the consumer can
457 // parse any internal format to the deopt state. It also handles null
458 // pointers and other constant pointers in GC states.
459 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
460 pushStackMapConstant(Ops, Builder, C->getSExtValue());
461 return;
462 } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Incoming)) {
464 C->getValueAPF().bitcastToAPInt().getZExtValue());
465 return;
466 }
467
468 llvm_unreachable("unhandled direct lowering case");
469 }
470
471
472
473 if (!RequireSpillSlot) {
474 // If this value is live in (not live-on-return, or live-through), we can
475 // treat it the same way patchpoint treats it's "live in" values. We'll
476 // end up folding some of these into stack references, but they'll be
477 // handled by the register allocator. Note that we do not have the notion
478 // of a late use so these values might be placed in registers which are
479 // clobbered by the call. This is fine for live-in. For live-through
480 // fix-up pass should be executed to force spilling of such registers.
481 Ops.push_back(Incoming);
482 } else {
483 // Otherwise, locate a spill slot and explicitly spill it so it can be
484 // found by the runtime later. Note: We know all of these spills are
485 // independent, but don't bother to exploit that chain wise. DAGCombine
486 // will happily do so as needed, so doing it here would be a small compile
487 // time win at most.
488 SDValue Chain = Builder.getRoot();
489 auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
490 Ops.push_back(std::get<0>(Res));
491 if (auto *MMO = std::get<2>(Res))
492 MemRefs.push_back(MMO);
493 Chain = std::get<1>(Res);;
494 Builder.DAG.setRoot(Chain);
495 }
496
497}
498
499/// Return true if value V represents the GC value. The behavior is conservative
500/// in case it is not sure that value is not GC the function returns true.
501static bool isGCValue(const Value *V, SelectionDAGBuilder &Builder) {
502 auto *Ty = V->getType();
503 if (!Ty->isPtrOrPtrVectorTy())
504 return false;
505 if (auto *GFI = Builder.GFI)
506 if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
507 return *IsManaged;
508 return true; // conservative
509}
510
511/// Lower deopt state and gc pointer arguments of the statepoint. The actual
512/// lowering is described in lowerIncomingStatepointValue. This function is
513/// responsible for lowering everything in the right position and playing some
514/// tricks to avoid redundant stack manipulation where possible. On
515/// completion, 'Ops' will contain ready to use operands for machine code
516/// statepoint. The chain nodes will have already been created and the DAG root
517/// will be set to the last value spilled (if any were).
518static void
522 DenseMap<SDValue, int> &LowerAsVReg,
524 SelectionDAGBuilder &Builder) {
525 // Lower the deopt and gc arguments for this statepoint. Layout will be:
526 // deopt argument length, deopt arguments.., gc arguments...
527
528 // Figure out what lowering strategy we're going to use for each part
529 // Note: Is is conservatively correct to lower both "live-in" and "live-out"
530 // as "live-through". A "live-through" variable is one which is "live-in",
531 // "live-out", and live throughout the lifetime of the call (i.e. we can find
532 // it from any PC within the transitive callee of the statepoint). In
533 // particular, if the callee spills callee preserved registers we may not
534 // be able to find a value placed in that register during the call. This is
535 // fine for live-out, but not for live-through. If we were willing to make
536 // assumptions about the code generator producing the callee, we could
537 // potentially allow live-through values in callee saved registers.
538 const bool LiveInDeopt =
539 SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
540
541 // Decide which deriver pointers will go on VRegs
542 unsigned MaxVRegPtrs = MaxRegistersForGCPointers.getValue();
543
544 // Pointers used on exceptional path of invoke statepoint.
545 // We cannot assing them to VRegs.
546 SmallSet<SDValue, 8> LPadPointers;
548 if (const auto *StInvoke =
549 dyn_cast_or_null<InvokeInst>(SI.StatepointInstr)) {
550 LandingPadInst *LPI = StInvoke->getLandingPadInst();
551 for (const auto *Relocate : SI.GCRelocates)
552 if (Relocate->getOperand(0) == LPI) {
553 LPadPointers.insert(Builder.getValue(Relocate->getBasePtr()));
554 LPadPointers.insert(Builder.getValue(Relocate->getDerivedPtr()));
555 }
556 }
557
558 LLVM_DEBUG(dbgs() << "Deciding how to lower GC Pointers:\n");
559
560 // List of unique lowered GC Pointer values.
561 SmallSetVector<SDValue, 16> LoweredGCPtrs;
562 // Map lowered GC Pointer value to the index in above vector
563 DenseMap<SDValue, unsigned> GCPtrIndexMap;
564
565 unsigned CurNumVRegs = 0;
566
567 auto canPassGCPtrOnVReg = [&](SDValue SD) {
568 if (SD.getValueType().isVector())
569 return false;
570 if (LPadPointers.count(SD))
571 return false;
572 return !willLowerDirectly(SD);
573 };
574
575 auto processGCPtr = [&](const Value *V) {
576 SDValue PtrSD = Builder.getValue(V);
577 if (!LoweredGCPtrs.insert(PtrSD))
578 return; // skip duplicates
579 GCPtrIndexMap[PtrSD] = LoweredGCPtrs.size() - 1;
580
581 assert(!LowerAsVReg.count(PtrSD) && "must not have been seen");
582 if (LowerAsVReg.size() == MaxVRegPtrs)
583 return;
584 assert(V->getType()->isVectorTy() == PtrSD.getValueType().isVector() &&
585 "IR and SD types disagree");
586 if (!canPassGCPtrOnVReg(PtrSD)) {
587 LLVM_DEBUG(dbgs() << "direct/spill "; PtrSD.dump(&Builder.DAG));
588 return;
589 }
590 LLVM_DEBUG(dbgs() << "vreg "; PtrSD.dump(&Builder.DAG));
591 LowerAsVReg[PtrSD] = CurNumVRegs++;
592 };
593
594 // Process derived pointers first to give them more chance to go on VReg.
595 for (const Value *V : SI.Ptrs)
596 processGCPtr(V);
597 for (const Value *V : SI.Bases)
598 processGCPtr(V);
599
600 LLVM_DEBUG(dbgs() << LowerAsVReg.size() << " pointers will go in vregs\n");
601
602 auto requireSpillSlot = [&](const Value *V) {
603 if (!Builder.DAG.getTargetLoweringInfo().isTypeLegal(
604 Builder.getValue(V).getValueType()))
605 return true;
606 if (isGCValue(V, Builder))
607 return !LowerAsVReg.count(Builder.getValue(V));
608 return !(LiveInDeopt || UseRegistersForDeoptValues);
609 };
610
611 // Before we actually start lowering (and allocating spill slots for values),
612 // reserve any stack slots which we judge to be profitable to reuse for a
613 // particular value. This is purely an optimization over the code below and
614 // doesn't change semantics at all. It is important for performance that we
615 // reserve slots for both deopt and gc values before lowering either.
616 for (const Value *V : SI.DeoptState) {
617 if (requireSpillSlot(V))
619 }
620
621 for (const Value *V : SI.Ptrs) {
622 SDValue SDV = Builder.getValue(V);
623 if (!LowerAsVReg.count(SDV))
625 }
626
627 for (const Value *V : SI.Bases) {
628 SDValue SDV = Builder.getValue(V);
629 if (!LowerAsVReg.count(SDV))
631 }
632
633 // First, prefix the list with the number of unique values to be
634 // lowered. Note that this is the number of *Values* not the
635 // number of SDValues required to lower them.
636 const int NumVMSArgs = SI.DeoptState.size();
637 pushStackMapConstant(Ops, Builder, NumVMSArgs);
638
639 // The vm state arguments are lowered in an opaque manner. We do not know
640 // what type of values are contained within.
641 LLVM_DEBUG(dbgs() << "Lowering deopt state\n");
642 for (const Value *V : SI.DeoptState) {
643 SDValue Incoming;
644 // If this is a function argument at a static frame index, generate it as
645 // the frame index.
646 if (const Argument *Arg = dyn_cast<Argument>(V)) {
647 int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);
648 if (FI != INT_MAX)
649 Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());
650 }
651 if (!Incoming.getNode())
652 Incoming = Builder.getValue(V);
653 LLVM_DEBUG(dbgs() << "Value " << *V
654 << " requireSpillSlot = " << requireSpillSlot(V) << "\n");
655 lowerIncomingStatepointValue(Incoming, requireSpillSlot(V), Ops, MemRefs,
656 Builder);
657 }
658
659 // Finally, go ahead and lower all the gc arguments.
660 pushStackMapConstant(Ops, Builder, LoweredGCPtrs.size());
661 for (SDValue SDV : LoweredGCPtrs)
662 lowerIncomingStatepointValue(SDV, !LowerAsVReg.count(SDV), Ops, MemRefs,
663 Builder);
664
665 // Copy to out vector. LoweredGCPtrs will be empty after this point.
666 GCPtrs = LoweredGCPtrs.takeVector();
667
668 // If there are any explicit spill slots passed to the statepoint, record
669 // them, but otherwise do not do anything special. These are user provided
670 // allocas and give control over placement to the consumer. In this case,
671 // it is the contents of the slot which may get updated, not the pointer to
672 // the alloca
674 for (Value *V : SI.GCArgs) {
675 SDValue Incoming = Builder.getValue(V);
676 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
677 // This handles allocas as arguments to the statepoint
678 assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
679 "Incoming value is a frame index!");
680 Allocas.push_back(Builder.DAG.getTargetFrameIndex(
681 FI->getIndex(), Builder.getFrameIndexTy()));
682
683 auto &MF = Builder.DAG.getMachineFunction();
684 auto *MMO = getMachineMemOperand(MF, *FI);
685 MemRefs.push_back(MMO);
686 }
687 }
688 pushStackMapConstant(Ops, Builder, Allocas.size());
689 Ops.append(Allocas.begin(), Allocas.end());
690
691 // Now construct GC base/derived map;
692 pushStackMapConstant(Ops, Builder, SI.Ptrs.size());
693 SDLoc L = Builder.getCurSDLoc();
694 for (unsigned i = 0; i < SI.Ptrs.size(); ++i) {
695 SDValue Base = Builder.getValue(SI.Bases[i]);
696 assert(GCPtrIndexMap.count(Base) && "base not found in index map");
697 Ops.push_back(
698 Builder.DAG.getTargetConstant(GCPtrIndexMap[Base], L, MVT::i64));
699 SDValue Derived = Builder.getValue(SI.Ptrs[i]);
700 assert(GCPtrIndexMap.count(Derived) && "derived not found in index map");
701 Ops.push_back(
702 Builder.DAG.getTargetConstant(GCPtrIndexMap[Derived], L, MVT::i64));
703 }
704}
705
708 // The basic scheme here is that information about both the original call and
709 // the safepoint is encoded in the CallInst. We create a temporary call and
710 // lower it, then reverse engineer the calling sequence.
711
712 NumOfStatepoints++;
713 // Clear state
715 assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
716 assert((GFI || SI.Bases.empty()) &&
717 "No gc specified, so cannot relocate pointers!");
718
719 LLVM_DEBUG(dbgs() << "Lowering statepoint " << *SI.StatepointInstr << "\n");
720#ifndef NDEBUG
721 for (const auto *Reloc : SI.GCRelocates)
722 if (Reloc->getParent() == SI.StatepointInstr->getParent())
724#endif
725
726 // Lower statepoint vmstate and gcstate arguments
727
728 // All lowered meta args.
729 SmallVector<SDValue, 10> LoweredMetaArgs;
730 // Lowered GC pointers (subset of above).
731 SmallVector<SDValue, 16> LoweredGCArgs;
733 // Maps derived pointer SDValue to statepoint result of relocated pointer.
734 DenseMap<SDValue, int> LowerAsVReg;
735 lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, LoweredGCArgs, LowerAsVReg,
736 SI, *this);
737
738 // Now that we've emitted the spills, we need to update the root so that the
739 // call sequence is ordered correctly.
740 SI.CLI.setChain(getRoot());
741
742 // Get call node, we will replace it later with statepoint
743 SDValue ReturnVal;
744 SDNode *CallNode;
745 std::tie(ReturnVal, CallNode) = lowerCallFromStatepointLoweringInfo(SI, *this);
746
747 // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
748 // nodes with all the appropriate arguments and return values.
749
750 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
751 SDValue Chain = CallNode->getOperand(0);
752
753 SDValue Glue;
754 bool CallHasIncomingGlue = CallNode->getGluedNode();
755 if (CallHasIncomingGlue) {
756 // Glue is always last operand
757 Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
758 }
759
760 // Build the GC_TRANSITION_START node if necessary.
761 //
762 // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
763 // order in which they appear in the call to the statepoint intrinsic. If
764 // any of the operands is a pointer-typed, that operand is immediately
765 // followed by a SRCVALUE for the pointer that may be used during lowering
766 // (e.g. to form MachinePointerInfo values for loads/stores).
767 const bool IsGCTransition =
768 (SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) ==
770 if (IsGCTransition) {
772
773 // Add chain
774 TSOps.push_back(Chain);
775
776 // Add GC transition arguments
777 for (const Value *V : SI.GCTransitionArgs) {
778 TSOps.push_back(getValue(V));
779 if (V->getType()->isPointerTy())
780 TSOps.push_back(DAG.getSrcValue(V));
781 }
782
783 // Add glue if necessary
784 if (CallHasIncomingGlue)
785 TSOps.push_back(Glue);
786
788
789 SDValue GCTransitionStart =
791
792 Chain = GCTransitionStart.getValue(0);
793 Glue = GCTransitionStart.getValue(1);
794 }
795
796 // TODO: Currently, all of these operands are being marked as read/write in
797 // PrologEpilougeInserter.cpp, we should special case the VMState arguments
798 // and flags to be read-only.
800
801 // Add the <id> and <numBytes> constants.
803 Ops.push_back(
804 DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
805
806 // Calculate and push starting position of vmstate arguments
807 // Get number of arguments incoming directly into call node
808 unsigned NumCallRegArgs =
809 CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
810 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
811
812 // Add call target
813 SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
814 Ops.push_back(CallTarget);
815
816 // Add call arguments
817 // Get position of register mask in the call
818 SDNode::op_iterator RegMaskIt;
819 if (CallHasIncomingGlue)
820 RegMaskIt = CallNode->op_end() - 2;
821 else
822 RegMaskIt = CallNode->op_end() - 1;
823 Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
824
825 // Add a constant argument for the calling convention
826 pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
827
828 // Add a constant argument for the flags
829 uint64_t Flags = SI.StatepointFlags;
831 "Unknown flag used");
832 pushStackMapConstant(Ops, *this, Flags);
833
834 // Insert all vmstate and gcstate arguments
835 llvm::append_range(Ops, LoweredMetaArgs);
836
837 // Add register mask from call node
838 Ops.push_back(*RegMaskIt);
839
840 // Add chain
841 Ops.push_back(Chain);
842
843 // Same for the glue, but we add it only if original call had it
844 if (Glue.getNode())
845 Ops.push_back(Glue);
846
847 // Compute return values. Provide a glue output since we consume one as
848 // input. This allows someone else to chain off us as needed.
849 SmallVector<EVT, 8> NodeTys;
850 for (auto SD : LoweredGCArgs) {
851 if (!LowerAsVReg.count(SD))
852 continue;
853 NodeTys.push_back(SD.getValueType());
854 }
855 LLVM_DEBUG(dbgs() << "Statepoint has " << NodeTys.size() << " results\n");
856 assert(NodeTys.size() == LowerAsVReg.size() && "Inconsistent GC Ptr lowering");
857 NodeTys.push_back(MVT::Other);
858 NodeTys.push_back(MVT::Glue);
859
860 unsigned NumResults = NodeTys.size();
861 MachineSDNode *StatepointMCNode =
862 DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
863 DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
864
865 // For values lowered to tied-defs, create the virtual registers if used
866 // in other blocks. For local gc.relocate record appropriate statepoint
867 // result in StatepointLoweringState.
869 for (const auto *Relocate : SI.GCRelocates) {
870 Value *Derived = Relocate->getDerivedPtr();
871 SDValue SD = getValue(Derived);
872 if (!LowerAsVReg.count(SD))
873 continue;
874
875 SDValue Relocated = SDValue(StatepointMCNode, LowerAsVReg[SD]);
876
877 // Handle local relocate. Note that different relocates might
878 // map to the same SDValue.
879 if (SI.StatepointInstr->getParent() == Relocate->getParent()) {
881 if (Res)
882 assert(Res == Relocated);
883 else
884 StatepointLowering.setLocation(SD, Relocated);
885 continue;
886 }
887
888 // Handle multiple gc.relocates of the same input efficiently.
889 if (VirtRegs.count(SD))
890 continue;
891
892 auto *RetTy = Relocate->getType();
893 Register Reg = FuncInfo.CreateRegs(RetTy);
895 DAG.getDataLayout(), Reg, RetTy, std::nullopt);
896 SDValue Chain = DAG.getRoot();
897 RFV.getCopyToRegs(Relocated, DAG, getCurSDLoc(), Chain, nullptr);
898 PendingExports.push_back(Chain);
899
900 VirtRegs[SD] = Reg;
901 }
902
903 // Record for later use how each relocation was lowered. This is needed to
904 // allow later gc.relocates to mirror the lowering chosen.
905 const Instruction *StatepointInstr = SI.StatepointInstr;
906 auto &RelocationMap = FuncInfo.StatepointRelocationMaps[StatepointInstr];
907 for (const GCRelocateInst *Relocate : SI.GCRelocates) {
908 const Value *V = Relocate->getDerivedPtr();
909 SDValue SDV = getValue(V);
911
912 bool IsLocal = (Relocate->getParent() == StatepointInstr->getParent());
913
915 if (IsLocal && LowerAsVReg.count(SDV)) {
916 // Result is already stored in StatepointLowering
918 } else if (LowerAsVReg.count(SDV)) {
920 assert(VirtRegs.count(SDV));
921 Record.payload.Reg = VirtRegs[SDV];
922 } else if (Loc.getNode()) {
924 Record.payload.FI = cast<FrameIndexSDNode>(Loc)->getIndex();
925 } else {
927 // If we didn't relocate a value, we'll essentialy end up inserting an
928 // additional use of the original value when lowering the gc.relocate.
929 // We need to make sure the value is available at the new use, which
930 // might be in another block.
931 if (Relocate->getParent() != StatepointInstr->getParent())
933 }
934 RelocationMap[Relocate] = Record;
935 }
936
937
938
939 SDNode *SinkNode = StatepointMCNode;
940
941 // Build the GC_TRANSITION_END node if necessary.
942 //
943 // See the comment above regarding GC_TRANSITION_START for the layout of
944 // the operands to the GC_TRANSITION_END node.
945 if (IsGCTransition) {
947
948 // Add chain
949 TEOps.push_back(SDValue(StatepointMCNode, NumResults - 2));
950
951 // Add GC transition arguments
952 for (const Value *V : SI.GCTransitionArgs) {
953 TEOps.push_back(getValue(V));
954 if (V->getType()->isPointerTy())
955 TEOps.push_back(DAG.getSrcValue(V));
956 }
957
958 // Add glue
959 TEOps.push_back(SDValue(StatepointMCNode, NumResults - 1));
960
962
963 SDValue GCTransitionStart =
964 DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
965
966 SinkNode = GCTransitionStart.getNode();
967 }
968
969 // Replace original call
970 // Call: ch,glue = CALL ...
971 // Statepoint: [gc relocates],ch,glue = STATEPOINT ...
972 unsigned NumSinkValues = SinkNode->getNumValues();
973 SDValue StatepointValues[2] = {SDValue(SinkNode, NumSinkValues - 2),
974 SDValue(SinkNode, NumSinkValues - 1)};
975 DAG.ReplaceAllUsesWith(CallNode, StatepointValues);
976 // Remove original call node
977 DAG.DeleteNode(CallNode);
978
979 // Since we always emit CopyToRegs (even for local relocates), we must
980 // update root, so that they are emitted before any local uses.
981 (void)getControlRoot();
982
983 // TODO: A better future implementation would be to emit a single variable
984 // argument, variable return value STATEPOINT node here and then hookup the
985 // return value of each gc.relocate to the respective output of the
986 // previously emitted STATEPOINT value. Unfortunately, this doesn't appear
987 // to actually be possible today.
988
989 return ReturnVal;
990}
991
992/// Return two gc.results if present. First result is a block local
993/// gc.result, second result is a non-block local gc.result. Corresponding
994/// entry will be nullptr if not present.
995static std::pair<const GCResultInst*, const GCResultInst*>
997 std::pair<const GCResultInst *, const GCResultInst*> Res(nullptr, nullptr);
998 for (const auto *U : S.users()) {
999 auto *GRI = dyn_cast<GCResultInst>(U);
1000 if (!GRI)
1001 continue;
1002 if (GRI->getParent() == S.getParent())
1003 Res.first = GRI;
1004 else
1005 Res.second = GRI;
1006 }
1007 return Res;
1008}
1009
1010void
1012 const BasicBlock *EHPadBB /*= nullptr*/) {
1013 assert(I.getCallingConv() != CallingConv::AnyReg &&
1014 "anyregcc is not supported on statepoints!");
1015
1016#ifndef NDEBUG
1017 // Check that the associated GCStrategy expects to encounter statepoints.
1019 "GCStrategy does not expect to encounter statepoints");
1020#endif
1021
1022 SDValue ActualCallee;
1023 SDValue Callee = getValue(I.getActualCalledOperand());
1024
1025 if (I.getNumPatchBytes() > 0) {
1026 // If we've been asked to emit a nop sequence instead of a call instruction
1027 // for this statepoint then don't lower the call target, but use a constant
1028 // `undef` instead. Not lowering the call target lets statepoint clients
1029 // get away without providing a physical address for the symbolic call
1030 // target at link time.
1031 ActualCallee = DAG.getUNDEF(Callee.getValueType());
1032 } else {
1033 ActualCallee = Callee;
1034 }
1035
1038 I.getNumCallArgs(), ActualCallee,
1039 I.getActualReturnType(), false /* IsPatchPoint */);
1040
1041 // There may be duplication in the gc.relocate list; such as two copies of
1042 // each relocation on normal and exceptional path for an invoke. We only
1043 // need to spill once and record one copy in the stackmap, but we need to
1044 // reload once per gc.relocate. (Dedupping gc.relocates is trickier and best
1045 // handled as a CSE problem elsewhere.)
1046 // TODO: There a couple of major stackmap size optimizations we could do
1047 // here if we wished.
1048 // 1) If we've encountered a derived pair {B, D}, we don't need to actually
1049 // record {B,B} if it's seen later.
1050 // 2) Due to rematerialization, actual derived pointers are somewhat rare;
1051 // given that, we could change the format to record base pointer relocations
1052 // separately with half the space. This would require a format rev and a
1053 // fairly major rework of the STATEPOINT node though.
1055 for (const GCRelocateInst *Relocate : I.getGCRelocates()) {
1056 SI.GCRelocates.push_back(Relocate);
1057
1058 SDValue DerivedSD = getValue(Relocate->getDerivedPtr());
1059 if (Seen.insert(DerivedSD).second) {
1060 SI.Bases.push_back(Relocate->getBasePtr());
1061 SI.Ptrs.push_back(Relocate->getDerivedPtr());
1062 }
1063 }
1064
1065 // If we find a deopt value which isn't explicitly added, we need to
1066 // ensure it gets lowered such that gc cycles occurring before the
1067 // deoptimization event during the lifetime of the call don't invalidate
1068 // the pointer we're deopting with. Note that we assume that all
1069 // pointers passed to deopt are base pointers; relaxing that assumption
1070 // would require relatively large changes to how we represent relocations.
1071 for (Value *V : I.deopt_operands()) {
1072 if (!isGCValue(V, *this))
1073 continue;
1074 if (Seen.insert(getValue(V)).second) {
1075 SI.Bases.push_back(V);
1076 SI.Ptrs.push_back(V);
1077 }
1078 }
1079
1080 SI.GCArgs = ArrayRef<const Use>(I.gc_args_begin(), I.gc_args_end());
1081 SI.StatepointInstr = &I;
1082 SI.ID = I.getID();
1083
1084 SI.DeoptState = ArrayRef<const Use>(I.deopt_begin(), I.deopt_end());
1085 SI.GCTransitionArgs = ArrayRef<const Use>(I.gc_transition_args_begin(),
1086 I.gc_transition_args_end());
1087
1088 SI.StatepointFlags = I.getFlags();
1089 SI.NumPatchBytes = I.getNumPatchBytes();
1090 SI.EHPadBB = EHPadBB;
1091
1092 SDValue ReturnValue = LowerAsSTATEPOINT(SI);
1093
1094 // Export the result value if needed
1095 const auto GCResultLocality = getGCResultLocality(I);
1096
1097 if (!GCResultLocality.first && !GCResultLocality.second) {
1098 // The return value is not needed, just generate a poison value.
1099 // Note: This covers the void return case.
1101 return;
1102 }
1103
1104 if (GCResultLocality.first) {
1105 // Result value will be used in a same basic block. Don't export it or
1106 // perform any explicit register copies. The gc_result will simply grab
1107 // this value.
1108 setValue(&I, ReturnValue);
1109 }
1110
1111 if (!GCResultLocality.second)
1112 return;
1113 // Result value will be used in a different basic block so we need to export
1114 // it now. Default exporting mechanism will not work here because statepoint
1115 // call has a different type than the actual call. It means that by default
1116 // llvm will create export register of the wrong type (always i32 in our
1117 // case). So instead we need to create export register with correct type
1118 // manually.
1119 // TODO: To eliminate this problem we can remove gc.result intrinsics
1120 // completely and make statepoint call to return a tuple.
1121 Type *RetTy = GCResultLocality.second->getType();
1122 Register Reg = FuncInfo.CreateRegs(RetTy);
1124 DAG.getDataLayout(), Reg, RetTy,
1125 I.getCallingConv());
1126 SDValue Chain = DAG.getEntryNode();
1127
1128 RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
1129 PendingExports.push_back(Chain);
1130 FuncInfo.ValueMap[&I] = Reg;
1131}
1132
1134 const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
1135 bool VarArgDisallowed, bool ForceVoidReturnTy) {
1137 unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
1139 SI.CLI, Call, ArgBeginIndex, Call->arg_size(), Callee,
1140 ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
1141 false);
1142 if (!VarArgDisallowed)
1143 SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
1144
1145 auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
1146
1148
1149 auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes());
1150 SI.ID = SD.StatepointID.value_or(DefaultID);
1151 SI.NumPatchBytes = SD.NumPatchBytes.value_or(0);
1152
1153 SI.DeoptState =
1154 ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
1155 SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
1156 SI.EHPadBB = EHPadBB;
1157
1158 // NB! The GC arguments are deliberately left empty.
1159
1160 if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
1161 ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
1162 setValue(Call, ReturnVal);
1163 }
1164}
1165
1167 const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
1169 /* VarArgDisallowed = */ false,
1170 /* ForceVoidReturnTy = */ false);
1171}
1172
1173void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
1174 // The result value of the gc_result is simply the result of the actual
1175 // call. We've already emitted this, so just grab the value.
1176 const Value *SI = CI.getStatepoint();
1177 assert((isa<GCStatepointInst>(SI) || isa<UndefValue>(SI)) &&
1178 "GetStatepoint must return one of two types");
1179 if (isa<UndefValue>(SI))
1180 return;
1181
1182 if (cast<GCStatepointInst>(SI)->getParent() == CI.getParent()) {
1183 setValue(&CI, getValue(SI));
1184 return;
1185 }
1186 // Statepoint is in different basic block so we should have stored call
1187 // result in a virtual register.
1188 // We can not use default getValue() functionality to copy value from this
1189 // register because statepoint and actual call return types can be
1190 // different, and getValue() will use CopyFromReg of the wrong type,
1191 // which is always i32 in our case.
1192 Type *RetTy = CI.getType();
1193 SDValue CopyFromReg = getCopyFromRegs(SI, RetTy);
1194
1195 assert(CopyFromReg.getNode());
1196 setValue(&CI, CopyFromReg);
1197}
1198
1199void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
1200 const Value *Statepoint = Relocate.getStatepoint();
1201#ifndef NDEBUG
1202 // Consistency check
1203 // We skip this check for relocates not in the same basic block as their
1204 // statepoint. It would be too expensive to preserve validation info through
1205 // different basic blocks.
1206 assert((isa<GCStatepointInst>(Statepoint) || isa<UndefValue>(Statepoint)) &&
1207 "GetStatepoint must return one of two types");
1208 if (isa<UndefValue>(Statepoint))
1209 return;
1210
1211 if (cast<GCStatepointInst>(Statepoint)->getParent() == Relocate.getParent())
1213#endif
1214
1215 const Value *DerivedPtr = Relocate.getDerivedPtr();
1216 auto &RelocationMap =
1217 FuncInfo.StatepointRelocationMaps[cast<GCStatepointInst>(Statepoint)];
1218 auto SlotIt = RelocationMap.find(&Relocate);
1219 assert(SlotIt != RelocationMap.end() && "Relocating not lowered gc value");
1220 const RecordType &Record = SlotIt->second;
1221
1222 // If relocation was done via virtual register..
1223 if (Record.type == RecordType::SDValueNode) {
1224 assert(cast<GCStatepointInst>(Statepoint)->getParent() ==
1225 Relocate.getParent() &&
1226 "Nonlocal gc.relocate mapped via SDValue");
1228 assert(SDV.getNode() && "empty SDValue");
1229 setValue(&Relocate, SDV);
1230 return;
1231 }
1232 if (Record.type == RecordType::VReg) {
1233 Register InReg = Record.payload.Reg;
1235 DAG.getDataLayout(), InReg, Relocate.getType(),
1236 std::nullopt); // This is not an ABI copy.
1237 // We generate copy to/from regs even for local uses, hence we must
1238 // chain with current root to ensure proper ordering of copies w.r.t.
1239 // statepoint.
1240 SDValue Chain = DAG.getRoot();
1241 SDValue Relocation = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
1242 Chain, nullptr, nullptr);
1243 setValue(&Relocate, Relocation);
1244 return;
1245 }
1246
1247 if (Record.type == RecordType::Spill) {
1248 unsigned Index = Record.payload.FI;
1250
1251 // All the reloads are independent and are reading memory only modified by
1252 // statepoints (i.e. no other aliasing stores); informing SelectionDAG of
1253 // this this let's CSE kick in for free and allows reordering of
1254 // instructions if possible. The lowering for statepoint sets the root,
1255 // so this is ordering all reloads with the either
1256 // a) the statepoint node itself, or
1257 // b) the entry of the current block for an invoke statepoint.
1258 const SDValue Chain = DAG.getRoot(); // != Builder.getRoot()
1259
1260 auto &MF = DAG.getMachineFunction();
1261 auto &MFI = MF.getFrameInfo();
1262 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index);
1263 auto *LoadMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
1264 MFI.getObjectSize(Index),
1265 MFI.getObjectAlign(Index));
1266
1268 Relocate.getType());
1269
1270 SDValue SpillLoad =
1271 DAG.getLoad(LoadVT, getCurSDLoc(), Chain, SpillSlot, LoadMMO);
1272 PendingLoads.push_back(SpillLoad.getValue(1));
1273
1274 assert(SpillLoad.getNode());
1275 setValue(&Relocate, SpillLoad);
1276 return;
1277 }
1278
1280 SDValue SD = getValue(DerivedPtr);
1281
1282 if (SD.isUndef() && SD.getValueType().getSizeInBits() <= 64) {
1283 // Lowering relocate(undef) as arbitrary constant. Current constant value
1284 // is chosen such that it's unlikely to be a valid pointer.
1285 setValue(&Relocate, DAG.getTargetConstant(0xFEFEFEFE, SDLoc(SD), MVT::i64));
1286 return;
1287 }
1288
1289 // We didn't need to spill these special cases (constants and allocas).
1290 // See the handling in spillIncomingValueForStatepoint for detail.
1291 setValue(&Relocate, SD);
1292}
1293
1295 const auto &TLI = DAG.getTargetLoweringInfo();
1296 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1297 TLI.getPointerTy(DAG.getDataLayout()));
1298
1299 // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
1300 // call. We also do not lower the return value to any virtual register, and
1301 // change the immediately following return to a trap instruction.
1302 LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
1303 /* VarArgDisallowed = */ true,
1304 /* ForceVoidReturnTy = */ true);
1305}
1306
1308 // We do not lower the return value from llvm.deoptimize to any virtual
1309 // register, and change the immediately following return to a trap
1310 // instruction.
1312 DAG.setRoot(
1314}
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
assume Assume Builder
static const Function * getParent(const Value *V)
return RetTy
#define LLVM_DEBUG(X)
Definition: Debug.h:101
#define I(x, y, z)
Definition: MD5.cpp:58
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
@ SI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallSet class.
This file defines the SmallVector class.
cl::opt< bool > UseRegistersForGCPointersInLandingPad("use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false), cl::desc("Allow using registers for gc pointer in landing pad"))
static void lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
static std::optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
static bool isGCValue(const Value *V, SelectionDAGBuilder &Builder)
Return true if value V represents the GC value.
static bool willLowerDirectly(SDValue Incoming)
Return true if-and-only-if the given SDValue can be lowered as either a constant argument or a stack ...
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SmallVectorImpl< SDValue > &GCPtrs, DenseMap< SDValue, int > &LowerAsVReg, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
static std::pair< const GCResultInst *, const GCResultInst * > getGCResultLocality(const GCStatepointInst &S)
Return two gc.results if present.
cl::opt< bool > UseRegistersForDeoptValues("use-registers-for-deopt-values", cl::Hidden, cl::init(false), cl::desc("Allow using registers for non pointer deopt args"))
cl::opt< unsigned > MaxRegistersForGCPointers("max-registers-for-gc-values", cl::Hidden, cl::init(0), cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"))
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling.
FunctionLoweringInfo::StatepointRelocationRecord RecordType
static MachineMemOperand * getMachineMemOperand(MachineFunction &MF, FrameIndexSDNode &FI)
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Extract call from statepoint, lower it and return pointer to the call node.
static std::tuple< SDValue, SDValue, MachineMemOperand * > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
This file describes how to lower LLVM code to machine code.
@ Flags
Definition: TextStubV5.cpp:93
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1186
This class represents a function call, abstracting a target machine's calling convention.
unsigned size() const
Definition: DenseMap.h:99
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
GCStrategy & getStrategy()
getStrategy - Return the GC strategy for the function.
Definition: GCMetadata.h:108
const Value * getStatepoint() const
The statepoint with which this gc.relocate is associated.
Represents calls to the gc.relocate intrinsic.
Value * getDerivedPtr() const
Represents calls to the gc.result intrinsic.
Represents a gc.statepoint intrinsic call.
Definition: Statepoint.h:61
bool useStatepoints() const
Returns true if this strategy is expecting the use of gc.statepoints, and false otherwise.
Definition: GCStrategy.h:92
const BasicBlock * getParent() const
Definition: Instruction.h:90
The landingpad instruction holds all of the information necessary to generate correct exception handl...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
op_iterator op_end() const
op_iterator op_begin() const
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void setValue(const Value *V, SDValue NewN)
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)
Lower SLI into a STATEPOINT instruction.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:726
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:675
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:77
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
Vector takeVector()
Clear the SetVector and return the underlying vector.
Definition: SetVector.h:66
SmallBitVector & set()
bool test(unsigned Idx) const
void clear()
Clear all bits.
size_type size() const
Returns the number of bits in this bitvector.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:301
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:177
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:809
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
void clear()
Clear the memory usage of this object.
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
void scheduleRelocCall(const GCRelocateInst &RelocCall)
Record the fact that we expect to encounter a given gc_relocate before the next statepoint.
void setLocation(SDValue Val, SDValue Location)
void relocCallVisited(const GCRelocateInst &RelocCall)
Remove this gc_relocate from the list we're expecting to see before the next statepoint.
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
TargetOptions Options
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
iterator_range< user_iterator > users()
Definition: Value.h:421
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ AnyReg
Used for dynamic register based calls (e.g.
Definition: CallingConv.h:60
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:978
@ GC_TRANSITION_START
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:1250
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:208
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1145
@ GC_TRANSITION_END
Definition: ISDOpcodes.h:1251
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1839
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:2129
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:24
@ MaskAll
A bitmask that includes all valid flags.
@ DeoptLiveIn
Mark the deopt arguments associated with the statepoint as only being "live-in".
@ GCTransition
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Flag, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
static const uint64_t DeoptBundleStatepointID
Definition: Statepoint.h:240