LLVM  16.0.0git
FixupStatepointCallerSaved.cpp
Go to the documentation of this file.
1 //===-- FixupStatepointCallerSaved.cpp - Fixup caller saved registers ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// Statepoint instruction in deopt parameters contains values which are
11 /// meaningful to the runtime and should be able to be read at the moment the
12 /// call returns. So we can say that we need to encode the fact that these
13 /// values are "late read" by runtime. If we could express this notion for
14 /// register allocator it would produce the right form for us.
15 /// The need to fixup (i.e this pass) is specifically handling the fact that
16 /// we cannot describe such a late read for the register allocator.
17 /// Register allocator may put the value on a register clobbered by the call.
18 /// This pass forces the spill of such registers and replaces corresponding
19 /// statepoint operands to added spill slots.
20 ///
21 //===----------------------------------------------------------------------===//
22 
23 #include "llvm/ADT/SmallSet.h"
24 #include "llvm/ADT/Statistic.h"
27 #include "llvm/CodeGen/StackMaps.h"
29 #include "llvm/IR/Statepoint.h"
30 #include "llvm/InitializePasses.h"
31 #include "llvm/Support/Debug.h"
32 
33 using namespace llvm;
34 
35 #define DEBUG_TYPE "fixup-statepoint-caller-saved"
36 STATISTIC(NumSpilledRegisters, "Number of spilled register");
37 STATISTIC(NumSpillSlotsAllocated, "Number of spill slots allocated");
38 STATISTIC(NumSpillSlotsExtended, "Number of spill slots extended");
39 
41  "fixup-scs-extend-slot-size", cl::Hidden, cl::init(false),
42  cl::desc("Allow spill in spill slot of greater size than register size"),
43  cl::Hidden);
44 
46  "fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false),
47  cl::desc("Allow passing GC Pointer arguments in callee saved registers"));
48 
50  "fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true),
51  cl::desc("Enable simple copy propagation during register reloading"));
52 
53 // This is purely debugging option.
54 // It may be handy for investigating statepoint spilling issues.
56  "fixup-max-csr-statepoints", cl::Hidden,
57  cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"));
58 
59 namespace {
60 
61 class FixupStatepointCallerSaved : public MachineFunctionPass {
62 public:
63  static char ID;
64 
65  FixupStatepointCallerSaved() : MachineFunctionPass(ID) {
67  }
68 
69  void getAnalysisUsage(AnalysisUsage &AU) const override {
70  AU.setPreservesCFG();
72  }
73 
74  StringRef getPassName() const override {
75  return "Fixup Statepoint Caller Saved";
76  }
77 
78  bool runOnMachineFunction(MachineFunction &MF) override;
79 };
80 
81 } // End anonymous namespace.
82 
85 
86 INITIALIZE_PASS_BEGIN(FixupStatepointCallerSaved, DEBUG_TYPE,
87  "Fixup Statepoint Caller Saved", false, false)
88 INITIALIZE_PASS_END(FixupStatepointCallerSaved, DEBUG_TYPE,
89  "Fixup Statepoint Caller Saved", false, false)
90 
91 // Utility function to get size of the register.
94  return TRI.getSpillSize(*RC);
95 }
96 
97 // Try to eliminate redundant copy to register which we're going to
98 // spill, i.e. try to change:
99 // X = COPY Y
100 // SPILL X
101 // to
102 // SPILL Y
103 // If there are no uses of X between copy and STATEPOINT, that COPY
104 // may be eliminated.
105 // Reg - register we're about to spill
106 // RI - On entry points to statepoint.
107 // On successful copy propagation set to new spill point.
108 // IsKill - set to true if COPY is Kill (there are no uses of Y)
109 // Returns either found source copy register or original one.
112  bool &IsKill, const TargetInstrInfo &TII,
113  const TargetRegisterInfo &TRI) {
114  // First check if statepoint itself uses Reg in non-meta operands.
115  int Idx = RI->findRegisterUseOperandIdx(Reg, false, &TRI);
116  if (Idx >= 0 && (unsigned)Idx < StatepointOpers(&*RI).getNumDeoptArgsIdx()) {
117  IsKill = false;
118  return Reg;
119  }
120 
121  if (!EnableCopyProp)
122  return Reg;
123 
124  MachineBasicBlock *MBB = RI->getParent();
126  MachineInstr *Def = nullptr, *Use = nullptr;
127  for (auto It = ++(RI.getReverse()); It != E; ++It) {
128  if (It->readsRegister(Reg, &TRI) && !Use)
129  Use = &*It;
130  if (It->modifiesRegister(Reg, &TRI)) {
131  Def = &*It;
132  break;
133  }
134  }
135 
136  if (!Def)
137  return Reg;
138 
139  auto DestSrc = TII.isCopyInstr(*Def);
140  if (!DestSrc || DestSrc->Destination->getReg() != Reg)
141  return Reg;
142 
143  Register SrcReg = DestSrc->Source->getReg();
144 
145  if (getRegisterSize(TRI, Reg) != getRegisterSize(TRI, SrcReg))
146  return Reg;
147 
148  LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation "
149  << printReg(Reg, &TRI) << " -> " << printReg(SrcReg, &TRI)
150  << "\n");
151 
152  // Insert spill immediately after Def
154  IsKill = DestSrc->Source->isKill();
155 
156  if (!Use) {
157  // There are no uses of original register between COPY and STATEPOINT.
158  // There can't be any after STATEPOINT, so we can eliminate Def.
159  LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def);
160  Def->eraseFromParent();
161  } else if (IsKill) {
162  // COPY will remain in place, spill will be inserted *after* it, so it is
163  // not a kill of source anymore.
164  const_cast<MachineOperand *>(DestSrc->Source)->setIsKill(false);
165  }
166 
167  return SrcReg;
168 }
169 
170 namespace {
171 // Pair {Register, FrameIndex}
172 using RegSlotPair = std::pair<Register, int>;
173 
174 // Keeps track of what reloads were inserted in MBB.
175 class RegReloadCache {
176  using ReloadSet = SmallSet<RegSlotPair, 8>;
178 
179 public:
180  RegReloadCache() = default;
181 
182  // Record reload of Reg from FI in block MBB
183  void recordReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
184  RegSlotPair RSP(Reg, FI);
185  auto Res = Reloads[MBB].insert(RSP);
186  (void)Res;
187  assert(Res.second && "reload already exists");
188  }
189 
190  // Does basic block MBB contains reload of Reg from FI?
191  bool hasReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
192  RegSlotPair RSP(Reg, FI);
193  return Reloads.count(MBB) && Reloads[MBB].count(RSP);
194  }
195 };
196 
197 // Cache used frame indexes during statepoint re-write to re-use them in
198 // processing next statepoint instruction.
199 // Two strategies. One is to preserve the size of spill slot while another one
200 // extends the size of spill slots to reduce the number of them, causing
201 // the less total frame size. But unspill will have "implicit" any extend.
202 class FrameIndexesCache {
203 private:
204  struct FrameIndexesPerSize {
205  // List of used frame indexes during processing previous statepoints.
206  SmallVector<int, 8> Slots;
207  // Current index of un-used yet frame index.
208  unsigned Index = 0;
209  };
210  MachineFrameInfo &MFI;
211  const TargetRegisterInfo &TRI;
212  // Map size to list of frame indexes of this size. If the mode is
213  // FixupSCSExtendSlotSize then the key 0 is used to keep all frame indexes.
214  // If the size of required spill slot is greater than in a cache then the
215  // size will be increased.
217 
218  // Keeps track of slots reserved for the shared landing pad processing.
219  // Initialized from GlobalIndices for the current EHPad.
220  SmallSet<int, 8> ReservedSlots;
221 
222  // Landing pad can be destination of several statepoints. Every register
223  // defined by such statepoints must be spilled to the same stack slot.
224  // This map keeps that information.
226  GlobalIndices;
227 
228  FrameIndexesPerSize &getCacheBucket(unsigned Size) {
229  // In FixupSCSExtendSlotSize mode the bucket with 0 index is used
230  // for all sizes.
231  return Cache[FixupSCSExtendSlotSize ? 0 : Size];
232  }
233 
234 public:
235  FrameIndexesCache(MachineFrameInfo &MFI, const TargetRegisterInfo &TRI)
236  : MFI(MFI), TRI(TRI) {}
237  // Reset the current state of used frame indexes. After invocation of
238  // this function all frame indexes are available for allocation with
239  // the exception of slots reserved for landing pad processing (if any).
240  void reset(const MachineBasicBlock *EHPad) {
241  for (auto &It : Cache)
242  It.second.Index = 0;
243 
244  ReservedSlots.clear();
245  if (EHPad && GlobalIndices.count(EHPad))
246  for (auto &RSP : GlobalIndices[EHPad])
247  ReservedSlots.insert(RSP.second);
248  }
249 
250  // Get frame index to spill the register.
251  int getFrameIndex(Register Reg, MachineBasicBlock *EHPad) {
252  // Check if slot for Reg is already reserved at EHPad.
253  auto It = GlobalIndices.find(EHPad);
254  if (It != GlobalIndices.end()) {
255  auto &Vec = It->second;
256  auto Idx = llvm::find_if(
257  Vec, [Reg](RegSlotPair &RSP) { return Reg == RSP.first; });
258  if (Idx != Vec.end()) {
259  int FI = Idx->second;
260  LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register "
261  << printReg(Reg, &TRI) << " at "
262  << printMBBReference(*EHPad) << "\n");
263  assert(ReservedSlots.count(FI) && "using unreserved slot");
264  return FI;
265  }
266  }
267 
268  unsigned Size = getRegisterSize(TRI, Reg);
269  FrameIndexesPerSize &Line = getCacheBucket(Size);
270  while (Line.Index < Line.Slots.size()) {
271  int FI = Line.Slots[Line.Index++];
272  if (ReservedSlots.count(FI))
273  continue;
274  // If all sizes are kept together we probably need to extend the
275  // spill slot size.
276  if (MFI.getObjectSize(FI) < Size) {
277  MFI.setObjectSize(FI, Size);
278  MFI.setObjectAlignment(FI, Align(Size));
279  NumSpillSlotsExtended++;
280  }
281  return FI;
282  }
283  int FI = MFI.CreateSpillStackObject(Size, Align(Size));
284  NumSpillSlotsAllocated++;
285  Line.Slots.push_back(FI);
286  ++Line.Index;
287 
288  // Remember assignment {Reg, FI} for EHPad
289  if (EHPad) {
290  GlobalIndices[EHPad].push_back(std::make_pair(Reg, FI));
291  LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling reg "
292  << printReg(Reg, &TRI) << " at landing pad "
293  << printMBBReference(*EHPad) << "\n");
294  }
295 
296  return FI;
297  }
298 
299  // Sort all registers to spill in descendent order. In the
300  // FixupSCSExtendSlotSize mode it will minimize the total frame size.
301  // In non FixupSCSExtendSlotSize mode we can skip this step.
302  void sortRegisters(SmallVectorImpl<Register> &Regs) {
304  return;
305  llvm::sort(Regs, [&](Register &A, Register &B) {
306  return getRegisterSize(TRI, A) > getRegisterSize(TRI, B);
307  });
308  }
309 };
310 
311 // Describes the state of the current processing statepoint instruction.
312 class StatepointState {
313 private:
314  // statepoint instruction.
315  MachineInstr &MI;
316  MachineFunction &MF;
317  // If non-null then statepoint is invoke, and this points to the landing pad.
318  MachineBasicBlock *EHPad;
319  const TargetRegisterInfo &TRI;
320  const TargetInstrInfo &TII;
321  MachineFrameInfo &MFI;
322  // Mask with callee saved registers.
323  const uint32_t *Mask;
324  // Cache of frame indexes used on previous instruction processing.
325  FrameIndexesCache &CacheFI;
326  bool AllowGCPtrInCSR;
327  // Operands with physical registers requiring spilling.
328  SmallVector<unsigned, 8> OpsToSpill;
329  // Set of register to spill.
330  SmallVector<Register, 8> RegsToSpill;
331  // Set of registers to reload after statepoint.
332  SmallVector<Register, 8> RegsToReload;
333  // Map Register to Frame Slot index.
334  DenseMap<Register, int> RegToSlotIdx;
335 
336 public:
337  StatepointState(MachineInstr &MI, const uint32_t *Mask,
338  FrameIndexesCache &CacheFI, bool AllowGCPtrInCSR)
339  : MI(MI), MF(*MI.getMF()), TRI(*MF.getSubtarget().getRegisterInfo()),
340  TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),
341  Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {
342 
343  // Find statepoint's landing pad, if any.
344  EHPad = nullptr;
345  MachineBasicBlock *MBB = MI.getParent();
346  // Invoke statepoint must be last one in block.
347  bool Last = std::none_of(++MI.getIterator(), MBB->end().getInstrIterator(),
348  [](MachineInstr &I) {
349  return I.getOpcode() == TargetOpcode::STATEPOINT;
350  });
351 
352  if (!Last)
353  return;
354 
355  auto IsEHPad = [](MachineBasicBlock *B) { return B->isEHPad(); };
356 
357  assert(llvm::count_if(MBB->successors(), IsEHPad) < 2 && "multiple EHPads");
358 
359  auto It = llvm::find_if(MBB->successors(), IsEHPad);
360  if (It != MBB->succ_end())
361  EHPad = *It;
362  }
363 
364  MachineBasicBlock *getEHPad() const { return EHPad; }
365 
366  // Return true if register is callee saved.
367  bool isCalleeSaved(Register Reg) { return (Mask[Reg / 32] >> Reg % 32) & 1; }
368 
369  // Iterates over statepoint meta args to find caller saver registers.
370  // Also cache the size of found registers.
371  // Returns true if caller save registers found.
372  bool findRegistersToSpill() {
373  SmallSet<Register, 8> GCRegs;
374  // All GC pointer operands assigned to registers produce new value.
375  // Since they're tied to their defs, it is enough to collect def registers.
376  for (const auto &Def : MI.defs())
377  GCRegs.insert(Def.getReg());
378 
379  SmallSet<Register, 8> VisitedRegs;
380  for (unsigned Idx = StatepointOpers(&MI).getVarIdx(),
381  EndIdx = MI.getNumOperands();
382  Idx < EndIdx; ++Idx) {
383  MachineOperand &MO = MI.getOperand(Idx);
384  // Leave `undef` operands as is, StackMaps will rewrite them
385  // into a constant.
386  if (!MO.isReg() || MO.isImplicit() || MO.isUndef())
387  continue;
388  Register Reg = MO.getReg();
389  assert(Reg.isPhysical() && "Only physical regs are expected");
390 
391  if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !is_contained(GCRegs, Reg)))
392  continue;
393 
394  LLVM_DEBUG(dbgs() << "Will spill " << printReg(Reg, &TRI) << " at index "
395  << Idx << "\n");
396 
397  if (VisitedRegs.insert(Reg).second)
398  RegsToSpill.push_back(Reg);
399  OpsToSpill.push_back(Idx);
400  }
401  CacheFI.sortRegisters(RegsToSpill);
402  return !RegsToSpill.empty();
403  }
404 
405  // Spill all caller saved registers right before statepoint instruction.
406  // Remember frame index where register is spilled.
407  void spillRegisters() {
408  for (Register Reg : RegsToSpill) {
409  int FI = CacheFI.getFrameIndex(Reg, EHPad);
411 
412  NumSpilledRegisters++;
413  RegToSlotIdx[Reg] = FI;
414 
415  LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, &TRI) << " to FI " << FI
416  << "\n");
417 
418  // Perform trivial copy propagation
419  bool IsKill = true;
420  MachineBasicBlock::iterator InsertBefore(MI);
421  Reg = performCopyPropagation(Reg, InsertBefore, IsKill, TII, TRI);
422 
423  LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore);
424  TII.storeRegToStackSlot(*MI.getParent(), InsertBefore, Reg, IsKill, FI,
425  RC, &TRI);
426  }
427  }
428 
429  void insertReloadBefore(unsigned Reg, MachineBasicBlock::iterator It,
432  int FI = RegToSlotIdx[Reg];
433  if (It != MBB->end()) {
434  TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI);
435  return;
436  }
437 
438  // To insert reload at the end of MBB, insert it before last instruction
439  // and then swap them.
440  assert(!MBB->empty() && "Empty block");
441  --It;
442  TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, &TRI);
443  MachineInstr *Reload = It->getPrevNode();
444  int Dummy = 0;
445  (void)Dummy;
446  assert(TII.isLoadFromStackSlot(*Reload, Dummy) == Reg);
447  assert(Dummy == FI);
448  MBB->remove(Reload);
449  MBB->insertAfter(It, Reload);
450  }
451 
452  // Insert reloads of (relocated) registers spilled in statepoint.
453  void insertReloads(MachineInstr *NewStatepoint, RegReloadCache &RC) {
454  MachineBasicBlock *MBB = NewStatepoint->getParent();
455  auto InsertPoint = std::next(NewStatepoint->getIterator());
456 
457  for (auto Reg : RegsToReload) {
458  insertReloadBefore(Reg, InsertPoint, MBB);
459  LLVM_DEBUG(dbgs() << "Reloading " << printReg(Reg, &TRI) << " from FI "
460  << RegToSlotIdx[Reg] << " after statepoint\n");
461 
462  if (EHPad && !RC.hasReload(Reg, RegToSlotIdx[Reg], EHPad)) {
463  RC.recordReload(Reg, RegToSlotIdx[Reg], EHPad);
464  auto EHPadInsertPoint = EHPad->SkipPHIsLabelsAndDebug(EHPad->begin());
465  insertReloadBefore(Reg, EHPadInsertPoint, EHPad);
466  LLVM_DEBUG(dbgs() << "...also reload at EHPad "
467  << printMBBReference(*EHPad) << "\n");
468  }
469  }
470  }
471 
472  // Re-write statepoint machine instruction to replace caller saved operands
473  // with indirect memory location (frame index).
474  MachineInstr *rewriteStatepoint() {
475  MachineInstr *NewMI =
476  MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true);
477  MachineInstrBuilder MIB(MF, NewMI);
478 
479  unsigned NumOps = MI.getNumOperands();
480 
481  // New indices for the remaining defs.
482  SmallVector<unsigned, 8> NewIndices;
483  unsigned NumDefs = MI.getNumDefs();
484  for (unsigned I = 0; I < NumDefs; ++I) {
485  MachineOperand &DefMO = MI.getOperand(I);
486  assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand");
487  Register Reg = DefMO.getReg();
488  assert(DefMO.isTied() && "Def is expected to be tied");
489  // We skipped undef uses and did not spill them, so we should not
490  // proceed with defs here.
491  if (MI.getOperand(MI.findTiedOperandIdx(I)).isUndef()) {
492  if (AllowGCPtrInCSR) {
493  NewIndices.push_back(NewMI->getNumOperands());
494  MIB.addReg(Reg, RegState::Define);
495  }
496  continue;
497  }
498  if (!AllowGCPtrInCSR) {
499  assert(is_contained(RegsToSpill, Reg));
500  RegsToReload.push_back(Reg);
501  } else {
502  if (isCalleeSaved(Reg)) {
503  NewIndices.push_back(NewMI->getNumOperands());
504  MIB.addReg(Reg, RegState::Define);
505  } else {
506  NewIndices.push_back(NumOps);
507  RegsToReload.push_back(Reg);
508  }
509  }
510  }
511 
512  // Add End marker.
513  OpsToSpill.push_back(MI.getNumOperands());
514  unsigned CurOpIdx = 0;
515 
516  for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) {
517  MachineOperand &MO = MI.getOperand(I);
518  if (I == OpsToSpill[CurOpIdx]) {
519  int FI = RegToSlotIdx[MO.getReg()];
520  MIB.addImm(StackMaps::IndirectMemRefOp);
521  MIB.addImm(getRegisterSize(TRI, MO.getReg()));
522  assert(MO.isReg() && "Should be register");
523  assert(MO.getReg().isPhysical() && "Should be physical register");
524  MIB.addFrameIndex(FI);
525  MIB.addImm(0);
526  ++CurOpIdx;
527  } else {
528  MIB.add(MO);
529  unsigned OldDef;
530  if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(I, &OldDef)) {
531  assert(OldDef < NumDefs);
532  assert(NewIndices[OldDef] < NumOps);
533  MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1);
534  }
535  }
536  }
537  assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed");
538  // Add mem operands.
539  NewMI->setMemRefs(MF, MI.memoperands());
540  for (auto It : RegToSlotIdx) {
541  Register R = It.first;
542  int FrameIndex = It.second;
543  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
545  if (is_contained(RegsToReload, R))
547  auto *MMO =
548  MF.getMachineMemOperand(PtrInfo, Flags, getRegisterSize(TRI, R),
550  NewMI->addMemOperand(MF, MMO);
551  }
552 
553  // Insert new statepoint and erase old one.
554  MI.getParent()->insert(MI, NewMI);
555 
556  LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n");
557  MI.eraseFromParent();
558  return NewMI;
559  }
560 };
561 
562 class StatepointProcessor {
563 private:
564  MachineFunction &MF;
565  const TargetRegisterInfo &TRI;
566  FrameIndexesCache CacheFI;
567  RegReloadCache ReloadCache;
568 
569 public:
570  StatepointProcessor(MachineFunction &MF)
571  : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()),
572  CacheFI(MF.getFrameInfo(), TRI) {}
573 
574  bool process(MachineInstr &MI, bool AllowGCPtrInCSR) {
575  StatepointOpers SO(&MI);
576  uint64_t Flags = SO.getFlags();
577  // Do nothing for LiveIn, it supports all registers.
579  return false;
580  LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " "
581  << MI.getParent()->getName() << " : process statepoint "
582  << MI);
583  CallingConv::ID CC = SO.getCallingConv();
584  const uint32_t *Mask = TRI.getCallPreservedMask(MF, CC);
585  StatepointState SS(MI, Mask, CacheFI, AllowGCPtrInCSR);
586  CacheFI.reset(SS.getEHPad());
587 
588  if (!SS.findRegistersToSpill())
589  return false;
590 
591  SS.spillRegisters();
592  auto *NewStatepoint = SS.rewriteStatepoint();
593  SS.insertReloads(NewStatepoint, ReloadCache);
594  return true;
595  }
596 };
597 } // namespace
598 
599 bool FixupStatepointCallerSaved::runOnMachineFunction(MachineFunction &MF) {
600  if (skipFunction(MF.getFunction()))
601  return false;
602 
603  const Function &F = MF.getFunction();
604  if (!F.hasGC())
605  return false;
606 
608  for (MachineBasicBlock &BB : MF)
609  for (MachineInstr &I : BB)
610  if (I.getOpcode() == TargetOpcode::STATEPOINT)
611  Statepoints.push_back(&I);
612 
613  if (Statepoints.empty())
614  return false;
615 
616  bool Changed = false;
617  StatepointProcessor SPP(MF);
618  unsigned NumStatepoints = 0;
619  bool AllowGCPtrInCSR = PassGCPtrInCSR;
620  for (MachineInstr *I : Statepoints) {
621  ++NumStatepoints;
622  if (MaxStatepointsWithRegs.getNumOccurrences() &&
623  NumStatepoints >= MaxStatepointsWithRegs)
624  AllowGCPtrInCSR = false;
625  Changed |= SPP.process(*I, AllowGCPtrInCSR);
626  }
627  return Changed;
628 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:77
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:108
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1723
llvm::AArch64PACKey::ID
ID
Definition: AArch64BaseInfo.h:818
llvm::Function
Definition: Function.h:60
llvm::PseudoProbeReservedId::Last
@ Last
llvm::SmallVector< int, 8 >
Statistic.h
PassGCPtrInCSR
static cl::opt< bool > PassGCPtrInCSR("fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false), cl::desc("Allow passing GC Pointer arguments in callee saved registers"))
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:454
llvm::printMBBReference
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Definition: MachineBasicBlock.cpp:117
llvm::MachineOperand::isTied
bool isTied() const
Definition: MachineOperand.h:440
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
Saved
Fixup Statepoint Caller Saved
Definition: FixupStatepointCallerSaved.cpp:89
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:237
llvm::MachineFunction::CreateMachineInstr
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)
CreateMachineInstr - Allocate a new MachineInstr.
Definition: MachineFunction.cpp:373
TargetInstrInfo.h
llvm::SmallSet
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:136
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::count
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:145
llvm::MachineFrameInfo::setObjectSize
void setObjectSize(int ObjectIdx, int64_t Size)
Change the size of the specified stack object.
Definition: MachineFrameInfo.h:477
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
llvm::count_if
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:1877
llvm::MachineFrameInfo::setObjectAlignment
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
Definition: MachineFrameInfo.h:497
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:167
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::MachineInstr::addMemOperand
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
Definition: MachineInstr.cpp:368
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:265
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(FixupStatepointCallerSaved, DEBUG_TYPE, "Fixup Statepoint Caller Saved", false, false) INITIALIZE_PASS_END(FixupStatepointCallerSaved
llvm::X86AS::SS
@ SS
Definition: X86.h:201
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::Register::isPhysical
bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:97
llvm::MachineOperand::isImplicit
bool isImplicit() const
Definition: MachineOperand.h:379
llvm::MachineBasicBlock::insertAfter
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
Definition: MachineBasicBlock.h:930
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:98
llvm::MachineBasicBlock::remove
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
Definition: MachineBasicBlock.h:989
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:24
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineInstr::setMemRefs
void setMemRefs(MachineFunction &MF, ArrayRef< MachineMemOperand * > MemRefs)
Assign this MachineInstr's memory reference descriptor list.
Definition: MachineInstr.cpp:357
llvm::MachineBasicBlock::succ_end
succ_iterator succ_end()
Definition: MachineBasicBlock.h:371
performCopyPropagation
static Register performCopyPropagation(Register Reg, MachineBasicBlock::iterator &RI, bool &IsKill, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI)
Definition: FixupStatepointCallerSaved.cpp:110
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
false
Definition: StackSlotColoring.cpp:141
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
llvm::logicalview::LVSortMode::Line
@ Line
llvm::MachineBasicBlock::rend
reverse_iterator rend()
Definition: MachineBasicBlock.h:315
FixupSCSExtendSlotSize
static cl::opt< bool > FixupSCSExtendSlotSize("fixup-scs-extend-slot-size", cl::Hidden, cl::init(false), cl::desc("Allow spill in spill slot of greater size than register size"), cl::Hidden)
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
EnableCopyProp
static cl::opt< bool > EnableCopyProp("fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true), cl::desc("Enable simple copy propagation during register reloading"))
Align
uint64_t Align
Definition: ELFObjHandler.cpp:82
Statepoint.h
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::MachineFrameInfo::CreateSpillStackObject
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
Definition: MachineFrameInfo.cpp:66
llvm::sort
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1657
llvm::MachineInstrBundleIterator::getInstrIterator
instr_iterator getInstrIterator() const
Definition: MachineInstrBundleIterator.h:274
llvm::cl::opt< bool >
llvm::TargetRegisterInfo::getSpillSize
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
Definition: TargetRegisterInfo.h:286
llvm::MachineOperand::isUndef
bool isUndef() const
Definition: MachineOperand.h:394
llvm::SmallSet::count
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:165
llvm::MachineInstrBundleIterator::getReverse
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Definition: MachineInstrBundleIterator.h:283
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:320
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
uint64_t
llvm::MachineFrameInfo::getObjectSize
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Definition: MachineFrameInfo.h:470
getRegisterSize
Fixup Statepoint Caller static false unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg)
Definition: FixupStatepointCallerSaved.cpp:92
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::HexagonInstrInfo::storeRegToStackSlot
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const override
Store the specified register of the given register class to the specified stack frame index.
Definition: HexagonInstrInfo.cpp:955
llvm::DenseMap
Definition: DenseMap.h:714
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::MachineFrameInfo::getObjectAlign
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
Definition: MachineFrameInfo.h:484
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:447
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1843
MachineFunctionPass.h
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
llvm::MachineMemOperand::Flags
Flags
Flags values. These may be or'd together.
Definition: MachineMemOperand.h:130
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:359
MaxStatepointsWithRegs
static cl::opt< unsigned > MaxStatepointsWithRegs("fixup-max-csr-statepoints", cl::Hidden, cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"))
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::NVPTXISD::Dummy
@ Dummy
Definition: NVPTXISelLowering.h:60
llvm::MachineBasicBlock::iterator
MachineInstrBundleIterator< MachineInstr > iterator
Definition: MachineBasicBlock.h:269
llvm::MachineBasicBlock::successors
iterator_range< succ_iterator > successors()
Definition: MachineBasicBlock.h:392
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:265
llvm::HexagonInstrInfo::loadRegFromStackSlot
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const override
Load the specified register of the given register class from the specified stack frame index.
Definition: HexagonInstrInfo.cpp:1000
uint32_t
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
llvm::MachineOperand::isDef
bool isDef() const
Definition: MachineOperand.h:374
CC
auto CC
Definition: RISCVRedundantCopyElimination.cpp:79
llvm::HexagonInstrInfo::isLoadFromStackSlot
unsigned isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
TargetInstrInfo overrides.
Definition: HexagonInstrInfo.cpp:287
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:313
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:207
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:134
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1736
llvm::FixupStatepointCallerSavedID
char & FixupStatepointCallerSavedID
The pass fixups statepoint machine instruction to replace usage of caller saved registers with stack ...
Definition: FixupStatepointCallerSaved.cpp:84
llvm::ISD::FrameIndex
@ FrameIndex
Definition: ISDOpcodes.h:80
DEBUG_TYPE
#define DEBUG_TYPE
Definition: FixupStatepointCallerSaved.cpp:35
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::StatepointOpers
MI-level Statepoint operands.
Definition: StackMaps.h:158
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:623
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::end
iterator end()
Definition: DenseMap.h:84
MachineFrameInfo.h
StackMaps.h
llvm::SmallSet::insert
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:178
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:136
llvm::MachineFrameInfo
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Definition: MachineFrameInfo.h:106
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1019
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:305
llvm::MachineInstr::getNumOperands
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:519
llvm::MachineBasicBlock::empty
bool empty() const
Definition: MachineBasicBlock.h:277
llvm::SmallSet::clear
void clear()
Definition: SmallSet.h:217
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::StatepointFlags::DeoptLiveIn
@ DeoptLiveIn
Mark the deopt arguments associated with the statepoint as only being "live-in".
llvm::omp::RTLDependInfoFields::Flags
@ Flags
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::MachineBasicBlock::SkipPHIsLabelsAndDebug
iterator SkipPHIsLabelsAndDebug(iterator I, bool SkipPseudoOp=true)
Return the first instruction in MBB after I that is not a PHI, label or debug.
Definition: MachineBasicBlock.cpp:222
llvm::cl::desc
Definition: CommandLine.h:413
llvm::TargetRegisterInfo::getMinimalPhysRegClass
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
Definition: TargetRegisterInfo.cpp:212
llvm::printReg
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Definition: TargetRegisterInfo.cpp:111
llvm::MachineInstrBundleIterator< MachineInstr >
InitializePasses.h
llvm::initializeFixupStatepointCallerSavedPass
void initializeFixupStatepointCallerSavedPass(PassRegistry &)
Debug.h
llvm::TargetRegisterInfo::getCallPreservedMask
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function.
Definition: TargetRegisterInfo.h:483
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:307
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
SmallSet.h