LLVM 24.0.0git
RISCVFrameLowering.cpp
Go to the documentation of this file.
1//===-- RISCVFrameLowering.cpp - RISC-V Frame Information -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVFrameLowering.h"
17#include "RISCVSubtarget.h"
28#include "llvm/MC/MCDwarf.h"
29#include "llvm/Support/LEB128.h"
30
31#include <algorithm>
32#include <cstdint>
33
34#define DEBUG_TYPE "riscv-frame"
35
36using namespace llvm;
37
40 RISCVABI::getStackAlignment(STI.getTargetABI()),
41 /*LocalAreaOffset=*/0,
42 /*TransientStackAlignment=*/
43 RISCVABI::getStackAlignment(STI.getTargetABI())),
44 STI(STI) {}
45
46// The register used to hold the frame pointer.
47static constexpr MCPhysReg FPReg = RISCV::X8;
48
49// The register used to hold the stack pointer.
50static constexpr MCPhysReg SPReg = RISCV::X2;
51
52// The register used to hold the return address.
53static constexpr MCPhysReg RAReg = RISCV::X1;
54
55// LIst of CSRs that are given a fixed location by save/restore libcalls or
56// Zcmp/Xqccmp Push/Pop. The order in this table indicates the order the
57// registers are saved on the stack. Zcmp uses the reverse order of save/restore
58// and Xqccmp on the stack, but this is handled when offsets are calculated.
59static const MCPhysReg FixedCSRFIMap[] = {
60 /*ra*/ RAReg, /*s0*/ FPReg, /*s1*/ RISCV::X9,
61 /*s2*/ RISCV::X18, /*s3*/ RISCV::X19, /*s4*/ RISCV::X20,
62 /*s5*/ RISCV::X21, /*s6*/ RISCV::X22, /*s7*/ RISCV::X23,
63 /*s8*/ RISCV::X24, /*s9*/ RISCV::X25, /*s10*/ RISCV::X26,
64 /*s11*/ RISCV::X27};
65
66// The number of stack bytes allocated by `QC.C.MIENTER(.NEST)` and popped by
67// `QC.C.MILEAVERET`.
68static constexpr uint64_t QCIInterruptPushAmount = 96;
69
70static const std::pair<MCPhysReg, int8_t> FixedCSRFIQCIInterruptMap[] = {
71 /* -1 is a gap for mepc/mnepc */
72 {/*fp*/ FPReg, -2},
73 /* -3 is a gap for qc.mcause */
74 {/*ra*/ RAReg, -4},
75 /* -5 is reserved */
76 {/*t0*/ RISCV::X5, -6},
77 {/*t1*/ RISCV::X6, -7},
78 {/*t2*/ RISCV::X7, -8},
79 {/*a0*/ RISCV::X10, -9},
80 {/*a1*/ RISCV::X11, -10},
81 {/*a2*/ RISCV::X12, -11},
82 {/*a3*/ RISCV::X13, -12},
83 {/*a4*/ RISCV::X14, -13},
84 {/*a5*/ RISCV::X15, -14},
85 {/*a6*/ RISCV::X16, -15},
86 {/*a7*/ RISCV::X17, -16},
87 {/*t3*/ RISCV::X28, -17},
88 {/*t4*/ RISCV::X29, -18},
89 {/*t5*/ RISCV::X30, -19},
90 {/*t6*/ RISCV::X31, -20},
91 /* -21, -22, -23, -24 are reserved */
92};
93
94/// Returns true if DWARF CFI instructions ("frame moves") should be emitted.
95static bool needsDwarfCFI(const MachineFunction &MF) {
96 return MF.needsFrameMoves();
97}
98
99// For now we use x3, a.k.a gp, as pointer to shadow call stack.
100// User should not use x3 in their asm.
103 const DebugLoc &DL) {
104 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
105 RISCVMachineFunctionInfo::ShadowStackKind SSK = RVFI->getShadowStackKind(MF);
107 return;
108
109 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
110 const llvm::RISCVRegisterInfo *TRI = STI.getRegisterInfo();
111
112 // Do not save RA to the SCS if it's not saved to the regular stack,
113 // i.e. RA is not at risk of being overwritten.
114 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
115 if (llvm::none_of(
116 CSI, [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
117 return;
118
119 const RISCVInstrInfo *TII = STI.getInstrInfo();
121 BuildMI(MBB, MI, DL, TII->get(RISCV::SSPUSH))
122 .addReg(RAReg)
124 return;
125 }
126
128 "Unexpected Shadow Stack Kind");
129
130 Register SCSPReg = RISCVABI::getSCSPReg();
131
132 bool IsRV64 = STI.is64Bit();
133 int64_t SlotSize = STI.getXLen() / 8;
134 // Store return address to shadow call stack
135 // addi gp, gp, [4|8]
136 // s[w|d] ra, -[4|8](gp)
137 BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI), SCSPReg)
138 .addReg(SCSPReg)
139 .addImm(SlotSize)
141 BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
142 .addReg(RAReg)
143 .addReg(SCSPReg)
144 .addImm(-SlotSize)
146
147 if (!needsDwarfCFI(MF))
148 return;
149
150 // Emit a CFI instruction that causes SlotSize to be subtracted from the value
151 // of the shadow stack pointer when unwinding past this frame.
152 char DwarfSCSReg = TRI->getDwarfRegNum(SCSPReg, /*IsEH*/ true);
153 assert(DwarfSCSReg < 32 && "SCS Register should be < 32 (X3).");
154
155 char Offset = static_cast<char>(-SlotSize) & 0x7f;
156 const char CFIInst[] = {
157 dwarf::DW_CFA_val_expression,
158 DwarfSCSReg, // register
159 2, // length
160 static_cast<char>(unsigned(dwarf::DW_OP_breg0 + DwarfSCSReg)),
161 Offset, // addend (sleb128)
162 };
163
165 .buildEscape(StringRef(CFIInst, sizeof(CFIInst)));
166}
167
170 const DebugLoc &DL) {
171 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
172 RISCVMachineFunctionInfo::ShadowStackKind SSK = RVFI->getShadowStackKind(MF);
174 return;
175
176 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
177
178 // See emitSCSPrologue() above.
179 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
180 if (llvm::none_of(
181 CSI, [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
182 return;
183
184 // The shadow call stack popchk needs to happen after cm.pop that loads ra.
185 if (MI != MBB.end() &&
186 (MI->getOpcode() == RISCV::CM_POP || MI->getOpcode() == RISCV::QC_CM_POP))
187 ++MI;
188 const RISCVInstrInfo *TII = STI.getInstrInfo();
190 // `sspopchk x5` is the only compressible form, but this would require using
191 // `x5` as the return address everywhere, which would also mean reserving
192 // it. We prefer to just use `x1` to avoid that complexity.
193 BuildMI(MBB, MI, DL, TII->get(RISCV::SSPOPCHK))
194 .addReg(RAReg)
196 return;
197 }
198
200 "Unexpected Shadow Stack Kind");
201
202 Register SCSPReg = RISCVABI::getSCSPReg();
203
204 bool IsRV64 = STI.is64Bit();
205 int64_t SlotSize = STI.getXLen() / 8;
206 // Load return address from shadow call stack
207 // l[w|d] ra, -[4|8](gp)
208 // addi gp, gp, -[4|8]
209 BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::LD : RISCV::LW), RAReg)
210 .addReg(SCSPReg)
211 .addImm(-SlotSize)
213 BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI), SCSPReg)
214 .addReg(SCSPReg)
215 .addImm(-SlotSize)
217 if (needsDwarfCFI(MF)) {
218 // Restore the SCS pointer
220 }
221}
222
223// Insert instruction to swap mscratchsw with sp
226 const DebugLoc &DL,
227 MachineInstr::MIFlag FrameFlag) {
228 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
229
230 if (!RVFI->isSiFiveStackSwapInterrupt(MF))
231 return;
232
233 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
234 const RISCVInstrInfo *TII = STI.getInstrInfo();
235
236 assert(STI.hasVendorXSfmclic() && "Stack Swapping Requires XSfmclic");
237
238 BuildMI(MBB, MBBI, DL, TII->get(RISCV::CSRRW), SPReg)
239 .addImm(RISCVSysReg::sf_mscratchcsw)
241 .setMIFlag(FrameFlag);
242
243 // FIXME: CFI Information for this swap.
244}
245
246static void
249 if (!RVFI.isSiFivePreemptibleInterrupt(MF))
250 return;
251
252 const TargetRegisterClass &RC = RISCV::GPRRegClass;
253 const TargetRegisterInfo &TRI =
254 *MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
255 MachineFrameInfo &MFI = MF.getFrameInfo();
256
257 // Create two frame objects for saving `mcause` and `mepc`.
258 for (int I = 0; I < 2; ++I) {
259 int FI = MFI.CreateStackObject(TRI.getSpillSize(RC), TRI.getSpillAlign(RC),
260 true);
262 }
263}
264
265// The scratch register retains an ordinary CSI slot, but its save and restore
266// are emitted explicitly as part of the SiFive CLIC interrupt sequence.
268 const auto &CSI = MF.getFrameInfo().getCalleeSavedInfo();
269 auto ScratchCS = llvm::find_if(
270 CSI, [](const CalleeSavedInfo &CS) { return CS.getReg() == RISCV::X5; });
271 assert(ScratchCS != CSI.end() && "Missing SiFive CLIC scratch spill slot");
272 return ScratchCS->getFrameIdx();
273}
274
278 const DebugLoc &DL) {
279 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
280
281 if (!RVFI->isSiFivePreemptibleInterrupt(MF))
282 return;
283
284 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
285 const RISCVInstrInfo *TII = STI.getInstrInfo();
286
287 // FIXME: CFI information for `mcause` and `mepc` is missing.
288
289 // Preserve X5 before using it to save the interrupt CSRs. Other GPRs
290 // are saved by the ordinary spill sequence after preemption is enabled.
291 int ScratchFI = getSiFiveCLICScratchFrameIndex(MF);
292 TII->storeRegToStackSlot(MBB, MBBI, RISCV::X5, /*IsKill=*/true, ScratchFI,
293 &RISCV::GPRRegClass, Register(),
295 if (needsDwarfCFI(MF))
297 .buildOffset(RISCV::X5, MF.getFrameInfo().getObjectOffset(ScratchFI));
298
299 BuildMI(MBB, MBBI, DL, TII->get(RISCV::CSRRS), RISCV::X5)
300 .addImm(RISCVSysReg::mcause)
301 .addReg(RISCV::X0)
303 TII->storeRegToStackSlot(MBB, MBBI, RISCV::X5, /* IsKill=*/true,
304 RVFI->getInterruptCSRFrameIndex(0),
305 &RISCV::GPRRegClass, Register(),
307
308 BuildMI(MBB, MBBI, DL, TII->get(RISCV::CSRRS), RISCV::X5)
309 .addImm(RISCVSysReg::mepc)
310 .addReg(RISCV::X0)
312
313 // Enable interrupts.
314 BuildMI(MBB, MBBI, DL, TII->get(RISCV::CSRRSI), RISCV::X0)
315 .addImm(RISCVSysReg::mstatus)
316 .addImm(8)
318 TII->storeRegToStackSlot(MBB, MBBI, RISCV::X5, /* IsKill=*/true,
319 RVFI->getInterruptCSRFrameIndex(1),
320 &RISCV::GPRRegClass, Register(),
322}
323
327 CFIInstBuilder &CFIBuilder,
328 const DebugLoc &DL) {
329 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
330
331 if (!RVFI->isSiFivePreemptibleInterrupt(MF))
332 return;
333
334 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
335 const RISCVInstrInfo *TII = STI.getInstrInfo();
336
337 // FIXME: CFI information for `mcause` and `mepc` is missing.
338
339 // Load mepc while preemption is still enabled. A nested handler preserves
340 // X5. Interrupts only need to be disabled before writing the CSRs back.
341 TII->loadRegFromStackSlot(MBB, MBBI, RISCV::X5,
342 RVFI->getInterruptCSRFrameIndex(1),
343 &RISCV::GPRRegClass, Register(),
344 RISCV::NoSubRegister, MachineInstr::FrameDestroy);
345
346 // Disable interrupts.
347 BuildMI(MBB, MBBI, DL, TII->get(RISCV::CSRRCI), RISCV::X0)
348 .addImm(RISCVSysReg::mstatus)
349 .addImm(8)
351
352 // Restore `mepc` and `mcause` through X5, then restore the value X5 held
353 // on entry to the handler.
354 BuildMI(MBB, MBBI, DL, TII->get(RISCV::CSRRW), RISCV::X0)
355 .addImm(RISCVSysReg::mepc)
356 .addReg(RISCV::X5, RegState::Kill)
358
359 TII->loadRegFromStackSlot(MBB, MBBI, RISCV::X5,
360 RVFI->getInterruptCSRFrameIndex(0),
361 &RISCV::GPRRegClass, Register(),
362 RISCV::NoSubRegister, MachineInstr::FrameDestroy);
363 BuildMI(MBB, MBBI, DL, TII->get(RISCV::CSRRW), RISCV::X0)
364 .addImm(RISCVSysReg::mcause)
365 .addReg(RISCV::X5, RegState::Kill)
367
368 // The ordinary reloads have finished. Recover the interrupted value of X5
369 // only after it has restored both CSRs.
370 TII->loadRegFromStackSlot(MBB, MBBI, RISCV::X5,
372 &RISCV::GPRRegClass, Register(),
373 RISCV::NoSubRegister, MachineInstr::FrameDestroy);
374 if (needsDwarfCFI(MF))
375 CFIBuilder.buildRestore(RISCV::X5);
376}
377
378// Get the ID of the libcall used for spilling and restoring callee saved
379// registers. The ID is representative of the number of registers saved or
380// restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
381// single register.
382static int getLibCallID(const MachineFunction &MF,
383 const std::vector<CalleeSavedInfo> &CSI) {
384 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
385
386 if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
387 return -1;
388
389 MCRegister MaxReg;
390 for (auto &CS : CSI)
391 // assignCalleeSavedSpillSlots assigns negative frame indexes to
392 // registers which can be saved by libcall.
393 if (CS.getFrameIdx() < 0)
394 MaxReg = std::max(MaxReg.id(), CS.getReg().id());
395
396 if (!MaxReg)
397 return -1;
398
399 switch (MaxReg.id()) {
400 default:
401 llvm_unreachable("Something has gone wrong!");
402 // clang-format off
403 case /*s11*/ RISCV::X27: return 12;
404 case /*s10*/ RISCV::X26: return 11;
405 case /*s9*/ RISCV::X25: return 10;
406 case /*s8*/ RISCV::X24: return 9;
407 case /*s7*/ RISCV::X23: return 8;
408 case /*s6*/ RISCV::X22: return 7;
409 case /*s5*/ RISCV::X21: return 6;
410 case /*s4*/ RISCV::X20: return 5;
411 case /*s3*/ RISCV::X19: return 4;
412 case /*s2*/ RISCV::X18: return 3;
413 case /*s1*/ RISCV::X9: return 2;
414 case /*s0*/ FPReg: return 1;
415 case /*ra*/ RAReg: return 0;
416 // clang-format on
417 }
418}
419
420// Get the name of the libcall used for spilling callee saved registers.
421// If this function will not use save/restore libcalls, then return a nullptr.
422static const char *
424 const std::vector<CalleeSavedInfo> &CSI) {
425 static const char *const SpillLibCalls[] = {
426 "__riscv_save_0",
427 "__riscv_save_1",
428 "__riscv_save_2",
429 "__riscv_save_3",
430 "__riscv_save_4",
431 "__riscv_save_5",
432 "__riscv_save_6",
433 "__riscv_save_7",
434 "__riscv_save_8",
435 "__riscv_save_9",
436 "__riscv_save_10",
437 "__riscv_save_11",
438 "__riscv_save_12"
439 };
440
441 int LibCallID = getLibCallID(MF, CSI);
442 if (LibCallID == -1)
443 return nullptr;
444 return SpillLibCalls[LibCallID];
445}
446
447// Get the name of the libcall used for restoring callee saved registers.
448// If this function will not use save/restore libcalls, then return a nullptr.
449static const char *
451 const std::vector<CalleeSavedInfo> &CSI) {
452 static const char *const RestoreLibCalls[] = {
453 "__riscv_restore_0",
454 "__riscv_restore_1",
455 "__riscv_restore_2",
456 "__riscv_restore_3",
457 "__riscv_restore_4",
458 "__riscv_restore_5",
459 "__riscv_restore_6",
460 "__riscv_restore_7",
461 "__riscv_restore_8",
462 "__riscv_restore_9",
463 "__riscv_restore_10",
464 "__riscv_restore_11",
465 "__riscv_restore_12"
466 };
467
468 int LibCallID = getLibCallID(MF, CSI);
469 if (LibCallID == -1)
470 return nullptr;
471 return RestoreLibCalls[LibCallID];
472}
473
474// Get the max reg of Push/Pop for restoring callee saved registers.
475static unsigned getNumPushPopRegs(const std::vector<CalleeSavedInfo> &CSI) {
476 unsigned NumPushPopRegs = 0;
477 for (auto &CS : CSI) {
478 auto *FII = llvm::find_if(FixedCSRFIMap,
479 [&](MCPhysReg P) { return P == CS.getReg(); });
480 if (FII != std::end(FixedCSRFIMap)) {
481 unsigned RegNum = std::distance(std::begin(FixedCSRFIMap), FII);
482 NumPushPopRegs = std::max(NumPushPopRegs, RegNum + 1);
483 }
484 }
485 assert(NumPushPopRegs != 12 && "x26 requires x27 to also be pushed");
486 return NumPushPopRegs;
487}
488
489// Return true if the specified function should have a dedicated frame
490// pointer register. This is true if frame pointer elimination is
491// disabled, if it needs dynamic stack realignment, if the function has
492// variable sized allocas, or if the frame address is taken.
494 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
495
496 const MachineFrameInfo &MFI = MF.getFrameInfo();
497 if (MF.disableFramePointerElim() || RegInfo->hasStackRealignment(MF) ||
499 return true;
500
501 // With large callframes around we may need to use FP to access the scavenging
502 // emergency spillslot.
503 //
504 // We calculate the MaxCallFrameSize at the end of isel so this value should
505 // be stable for the whole post-isel MIR pipeline.
506 //
507 // NOTE: The idea of forcing a frame pointer is copied from AArch64, but they
508 // conservatively return true when the call frame size hasd not been
509 // computed yet. On RISC-V that caused MachineOutliner tests to fail the
510 // MachineVerifier due to outlined functions not computing max call frame
511 // size thus the frame pointer would always be reserved.
512 if (MFI.isMaxCallFrameSizeComputed() && MFI.getMaxCallFrameSize() > 2047)
513 return true;
514
515 return false;
516}
517
519 const MachineFrameInfo &MFI = MF.getFrameInfo();
520 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
521
522 // If we do not reserve stack space for outgoing arguments in prologue,
523 // we will adjust the stack pointer before call instruction. After the
524 // adjustment, we can not use SP to access the stack objects for the
525 // arguments. Instead, use BP to access these stack objects.
526 return (MFI.hasVarSizedObjects() ||
528 MFI.getMaxCallFrameSize() != 0))) &&
529 TRI->hasStackRealignment(MF);
530}
531
532// Determines the size of the frame and maximum call frame size.
533void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
534 MachineFrameInfo &MFI = MF.getFrameInfo();
535 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
536
537 // Get the number of bytes to allocate from the FrameInfo.
538 uint64_t FrameSize = MFI.getStackSize();
539
540 // QCI Interrupts use at least 96 bytes of stack space
541 if (RVFI->useQCIInterrupt(MF))
542 FrameSize = std::max(FrameSize, QCIInterruptPushAmount);
543
544 // Get the alignment.
545 Align StackAlign = getStackAlign();
546
547 // Make sure the frame is aligned.
548 FrameSize = alignTo(FrameSize, StackAlign);
549
550 // Update frame info.
551 MFI.setStackSize(FrameSize);
552
553 // When using SP or BP to access stack objects, we may require extra padding
554 // to ensure the bottom of the RVV stack is correctly aligned within the main
555 // stack. We calculate this as the amount required to align the scalar local
556 // variable section up to the RVV alignment.
558 if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) {
559 int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() -
560 RVFI->getVarArgsSaveSize();
561 if (auto RVVPadding =
562 offsetToAlignment(ScalarLocalVarSize, RVFI->getRVVStackAlign()))
563 RVFI->setRVVPadding(RVVPadding);
564 }
565}
566
567// Returns the stack size including RVV padding (when required), rounded back
568// up to the required stack alignment.
570 const MachineFunction &MF) const {
571 const MachineFrameInfo &MFI = MF.getFrameInfo();
572 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
573 return alignTo(MFI.getStackSize() + RVFI->getRVVPadding(), getStackAlign());
574}
575
578 const std::vector<CalleeSavedInfo> &CSI,
579 bool ReverseOrder = false) {
580 const MachineFrameInfo &MFI = MF.getFrameInfo();
582
583 for (auto &CS : CSI) {
584 int FI = CS.getFrameIdx();
585 // Callee saves with negative frame indices have fixed positions on the
586 // stack, usually relating to libcalls or push/pop.
587 bool IsManagedFI = (FI < 0);
588 if (!IsManagedFI && MFI.getStackID(FI) == TargetStackID::Default)
589 NonLibcallCSI.push_back(CS);
590 }
591
592 // Reverse the order so that load/store operations use ascending addresses,
593 // enabling better load/store clustering and fusion.
594 if (ReverseOrder)
595 std::reverse(NonLibcallCSI.begin(), NonLibcallCSI.end());
596
597 return NonLibcallCSI;
598}
599
600// Exclude X5 from ordinary spills and restores for SiFive CLIC preemptible
601// handlers, which save and restore it explicitly.
604 const std::vector<CalleeSavedInfo> &CSI,
605 bool ReverseOrder = false) {
606 auto InterruptCSI = getUnmanagedCSI(MF, CSI, ReverseOrder);
608 llvm::erase_if(InterruptCSI, [](const CalleeSavedInfo &CS) {
609 return CS.getReg() == RISCV::X5;
610 });
611 return InterruptCSI;
612}
613
616 const std::vector<CalleeSavedInfo> &CSI) {
617 const MachineFrameInfo &MFI = MF.getFrameInfo();
619
620 for (auto &CS : CSI) {
621 int FI = CS.getFrameIdx();
622 if (FI >= 0 && MFI.getStackID(FI) == TargetStackID::ScalableVector)
623 RVVCSI.push_back(CS);
624 }
625
626 return RVVCSI;
627}
628
631 const std::vector<CalleeSavedInfo> &CSI) {
632 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
633
634 SmallVector<CalleeSavedInfo, 8> PushOrLibCallsCSI;
635 if (!RVFI->useSaveRestoreLibCalls(MF) && !RVFI->isPushable(MF))
636 return PushOrLibCallsCSI;
637
638 for (const auto &CS : CSI) {
639 if (RVFI->useQCIInterrupt(MF)) {
640 // Some registers are saved by both `QC.C.MIENTER(.NEST)` and
641 // `QC.CM.PUSH(FP)`. In these cases, prioritise the CFI info that points
642 // to the versions saved by `QC.C.MIENTER(.NEST)` which is what FP
643 // unwinding would use.
645 CS.getReg()))
646 continue;
647 }
648
649 if (llvm::is_contained(FixedCSRFIMap, CS.getReg()))
650 PushOrLibCallsCSI.push_back(CS);
651 }
652
653 return PushOrLibCallsCSI;
654}
655
658 const std::vector<CalleeSavedInfo> &CSI) {
659 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
660
661 SmallVector<CalleeSavedInfo, 8> QCIInterruptCSI;
662 if (!RVFI->useQCIInterrupt(MF))
663 return QCIInterruptCSI;
664
665 for (const auto &CS : CSI) {
667 CS.getReg()))
668 QCIInterruptCSI.push_back(CS);
669 }
670
671 return QCIInterruptCSI;
672}
673
675 const MachineBasicBlock &MBB) {
676 const MachineFunction *MF = MBB.getParent();
677 LiveRegs.addLiveIns(MBB);
678 const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
679 for (unsigned i = 0; CSRegs[i]; ++i)
680 LiveRegs.addReg(CSRegs[i]);
681}
682
684 MachineBasicBlock *MBB, Register PreferredReg, Register DontUseReg) const {
685 MachineFunction *MF = MBB->getParent();
686
687 // Stack protection code is being inserted at beginning of function, use
688 // register which has been historically used
689 if (&MF->front() == MBB)
690 return PreferredReg;
691
693 const TargetRegisterInfo &TRI = *Subtarget.getRegisterInfo();
696
697 const MachineRegisterInfo &MRI = MF->getRegInfo();
698 // Prefer the register which has been historically used for stack protector
699 if (LiveRegs.available(MRI, PreferredReg))
700 return PreferredReg;
701
702 static const MCPhysReg CandidateRegs[] = {
703 RISCV::X5, RISCV::X6, RISCV::X7, RISCV::X28,
704 RISCV::X29, RISCV::X30, RISCV::X31,
705 };
706
707 for (unsigned Reg : CandidateRegs) {
708 if (Reg != DontUseReg && LiveRegs.available(MRI, Reg))
709 return Reg;
710 }
711
712 return Register();
713}
714
715void RISCVFrameLowering::allocateAndProbeStackForRVV(
717 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int64_t Amount,
718 MachineInstr::MIFlag Flag, bool EmitCFI, bool DynAllocation) const {
719 assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
720
721 // Emit a variable-length allocation probing loop.
722
723 // Get VLEN in TargetReg
724 Register TargetReg = findScratchNonCalleeSaveRegister(&MBB, RISCV::X6);
725 assert(TargetReg.isValid() &&
726 "No available scratch register for stack probing");
728 uint32_t NumOfVReg = Amount / RISCV::RVVBytesPerBlock;
729 BuildMI(MBB, MBBI, DL, TII->get(RISCV::PseudoReadVLENB), TargetReg)
730 .setMIFlag(Flag);
731 TII->mulImm(MF, MBB, MBBI, DL, TargetReg, NumOfVReg, Flag);
732
734 if (EmitCFI) {
735 // Set the CFA register to TargetReg.
736 CFIBuilder.buildDefCFA(TargetReg, -Amount);
737 }
738
739 // It will be expanded to a probe loop in `inlineStackProbe`.
740 BuildMI(MBB, MBBI, DL, TII->get(RISCV::PROBED_STACKALLOC_RVV))
741 .addReg(TargetReg);
742
743 if (EmitCFI) {
744 // Set the CFA register back to SP.
745 CFIBuilder.buildDefCFARegister(SPReg);
746 }
747
748 // SUB SP, SP, T1
749 BuildMI(MBB, MBBI, DL, TII->get(RISCV::SUB), SPReg)
750 .addReg(SPReg)
751 .addReg(TargetReg)
752 .setMIFlag(Flag);
753
754 // If we have a dynamic allocation later we need to probe any residuals.
755 if (DynAllocation) {
756 BuildMI(MBB, MBBI, DL, TII->get(STI.is64Bit() ? RISCV::SD : RISCV::SW))
757 .addReg(RISCV::X0)
758 .addReg(SPReg)
759 .addImm(0)
761 }
762}
763
767 llvm::raw_string_ostream &Comment) {
768 int64_t FixedOffset = Offset.getFixed();
769 int64_t ScalableOffset = Offset.getScalable();
770 unsigned DwarfVLenB = TRI.getDwarfRegNum(RISCV::VLENB, true);
771 if (FixedOffset) {
772 Expr.push_back(dwarf::DW_OP_consts);
773 appendLEB128<LEB128Sign::Signed>(Expr, FixedOffset);
774 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
775 Comment << (FixedOffset < 0 ? " - " : " + ") << std::abs(FixedOffset);
776 }
777
778 Expr.push_back((uint8_t)dwarf::DW_OP_consts);
779 appendLEB128<LEB128Sign::Signed>(Expr, ScalableOffset);
780
781 Expr.push_back((uint8_t)dwarf::DW_OP_bregx);
782 appendLEB128<LEB128Sign::Unsigned>(Expr, DwarfVLenB);
783 Expr.push_back(0);
784
785 Expr.push_back((uint8_t)dwarf::DW_OP_mul);
786 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
787
788 Comment << (ScalableOffset < 0 ? " - " : " + ") << std::abs(ScalableOffset)
789 << " * vlenb";
790}
791
795 assert(Offset.getScalable() != 0 && "Did not need to adjust CFA for RVV");
796 SmallString<64> Expr;
797 std::string CommentBuffer;
798 llvm::raw_string_ostream Comment(CommentBuffer);
799 // Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB).
800 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
801 Expr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
802 Expr.push_back(0);
803 if (Reg == SPReg)
804 Comment << "sp";
805 else
806 Comment << printReg(Reg, &TRI);
807
809
810 SmallString<64> DefCfaExpr;
811 DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
812 appendLEB128<LEB128Sign::Unsigned>(DefCfaExpr, Expr.size());
813 DefCfaExpr.append(Expr.str());
814
815 return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(), SMLoc(),
816 Comment.str());
817}
818
821 assert(Offset.getScalable() != 0 && "Did not need to adjust CFA for RVV");
822 SmallString<64> Expr;
823 std::string CommentBuffer;
824 llvm::raw_string_ostream Comment(CommentBuffer);
825 Comment << printReg(Reg, &TRI) << " @ cfa";
826
827 // Build up the expression (FixedOffset + ScalableOffset * VLENB).
829
830 SmallString<64> DefCfaExpr;
831 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
832 DefCfaExpr.push_back(dwarf::DW_CFA_expression);
833 appendLEB128<LEB128Sign::Unsigned>(DefCfaExpr, DwarfReg);
834 appendLEB128<LEB128Sign::Unsigned>(DefCfaExpr, Expr.size());
835 DefCfaExpr.append(Expr.str());
836
837 return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(), SMLoc(),
838 Comment.str());
839}
840
841// Allocate stack space and probe it if necessary.
844 MachineFunction &MF, uint64_t Offset,
845 uint64_t RealStackSize, bool EmitCFI,
846 bool NeedProbe, uint64_t ProbeSize,
847 bool DynAllocation,
848 MachineInstr::MIFlag Flag) const {
849 DebugLoc DL;
850 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
851 const RISCVInstrInfo *TII = STI.getInstrInfo();
852 bool IsRV64 = STI.is64Bit();
854
855 // Simply allocate the stack if it's not big enough to require a probe.
856 if (!NeedProbe || Offset <= ProbeSize) {
858 Flag, getStackAlign());
859
860 if (EmitCFI)
861 CFIBuilder.buildDefCFAOffset(RealStackSize);
862
863 if (NeedProbe && DynAllocation) {
864 // s[d|w] zero, 0(sp)
865 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
866 .addReg(RISCV::X0)
867 .addReg(SPReg)
868 .addImm(0)
869 .setMIFlags(Flag);
870 }
871
872 return;
873 }
874
875 // The amount of stack that was already allocated before this allocation.
876 uint64_t CFAAdjust = RealStackSize - Offset;
877
878 // Unroll the probe loop depending on the number of iterations.
879 if (Offset < ProbeSize * 5) {
880 uint64_t CurrentOffset = 0;
881 while (CurrentOffset + ProbeSize <= Offset) {
882 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
883 StackOffset::getFixed(-ProbeSize), Flag, getStackAlign());
884 // s[d|w] zero, 0(sp)
885 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
886 .addReg(RISCV::X0)
887 .addReg(SPReg)
888 .addImm(0)
889 .setMIFlags(Flag);
890
891 CurrentOffset += ProbeSize;
892 if (EmitCFI)
893 CFIBuilder.buildDefCFAOffset(CurrentOffset + CFAAdjust);
894 }
895
896 uint64_t Residual = Offset - CurrentOffset;
897 if (Residual) {
898 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
899 StackOffset::getFixed(-Residual), Flag, getStackAlign());
900 if (EmitCFI)
901 CFIBuilder.buildDefCFAOffset(RealStackSize);
902
903 if (DynAllocation) {
904 // s[d|w] zero, 0(sp)
905 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
906 .addReg(RISCV::X0)
907 .addReg(SPReg)
908 .addImm(0)
909 .setMIFlags(Flag);
910 }
911 }
912
913 return;
914 }
915
916 // Emit a variable-length allocation probing loop.
917 uint64_t RoundedSize = alignDown(Offset, ProbeSize);
918 uint64_t Residual = Offset - RoundedSize;
919
920 Register TargetReg = findScratchNonCalleeSaveRegister(&MBB, RISCV::X6);
921 assert(TargetReg.isValid() &&
922 "No available scratch register for stack probing");
923 // SUB TargetReg, SP, RoundedSize
924 RI->adjustReg(MBB, MBBI, DL, TargetReg, SPReg,
925 StackOffset::getFixed(-RoundedSize), Flag, getStackAlign());
926
927 if (EmitCFI) {
928 // Set the CFA register to TargetReg.
929 CFIBuilder.buildDefCFA(TargetReg, RoundedSize + CFAAdjust);
930 }
931
932 // It will be expanded to a probe loop in `inlineStackProbe`.
933 BuildMI(MBB, MBBI, DL, TII->get(RISCV::PROBED_STACKALLOC)).addReg(TargetReg);
934
935 if (EmitCFI) {
936 // Set the CFA register back to SP.
937 CFIBuilder.buildDefCFARegister(SPReg);
938 }
939
940 if (Residual) {
942 Flag, getStackAlign());
943 if (DynAllocation) {
944 // s[d|w] zero, 0(sp)
945 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
946 .addReg(RISCV::X0)
947 .addReg(SPReg)
948 .addImm(0)
949 .setMIFlags(Flag);
950 }
951 }
952
953 if (EmitCFI)
954 CFIBuilder.buildDefCFAOffset(RealStackSize);
955}
956
957static bool isPush(unsigned Opcode) {
958 switch (Opcode) {
959 case RISCV::CM_PUSH:
960 case RISCV::QC_CM_PUSH:
961 case RISCV::QC_CM_PUSHFP:
962 return true;
963 default:
964 return false;
965 }
966}
967
968static bool isPop(unsigned Opcode) {
969 // There are other pops but these are the only ones introduced during this
970 // pass.
971 switch (Opcode) {
972 case RISCV::CM_POP:
973 case RISCV::QC_CM_POP:
974 return true;
975 default:
976 return false;
977 }
978}
979
981 bool UpdateFP) {
982 switch (Kind) {
984 return RISCV::CM_PUSH;
986 return UpdateFP ? RISCV::QC_CM_PUSHFP : RISCV::QC_CM_PUSH;
987 default:
988 llvm_unreachable("Unhandled PushPopKind");
989 }
990}
991
993 // There are other pops but they are introduced later by the Push/Pop
994 // Optimizer.
995 switch (Kind) {
997 return RISCV::CM_POP;
999 return RISCV::QC_CM_POP;
1000 default:
1001 llvm_unreachable("Unhandled PushPopKind");
1002 }
1003}
1004
1006 MachineBasicBlock &MBB) const {
1007 MachineFrameInfo &MFI = MF.getFrameInfo();
1008 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1009 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1011 bool PreferAscendingLS = STI.preferAscendingLoadStore();
1012
1013 Register BPReg = RISCVABI::getBPReg();
1014
1015 // Debug location must be unknown since the first debug location is used
1016 // to determine the end of the prologue.
1017 DebugLoc DL;
1018
1019 // All calls are tail calls in GHC calling conv, and functions have no
1020 // prologue/epilogue.
1022 return;
1023
1024 // SiFive CLIC needs to swap `sp` into `sf.mscratchcsw`
1026
1027 // Emit prologue for shadow call stack.
1028 emitSCSPrologue(MF, MBB, MBBI, DL);
1029
1030 // We keep track of the first instruction because it might be a
1031 // `(QC.)CM.PUSH(FP)`, and we may need to adjust the immediate rather than
1032 // inserting an `addi sp, sp, -N*16`
1033 auto PossiblePush = MBBI;
1034
1035 // Skip past all callee-saved register spill instructions.
1036 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
1037 ++MBBI;
1038
1039 // Determine the correct frame layout
1040 determineFrameLayout(MF);
1041
1042 const auto &CSI = MFI.getCalleeSavedInfo();
1043
1044 // Skip to before the spills of scalar callee-saved registers
1045 // FIXME: assumes exactly one instruction is used to restore each
1046 // callee-saved register.
1047 MBBI = std::prev(
1048 MBBI, getRVVCalleeSavedInfo(MF, CSI).size() +
1049 getUnmanagedInterruptCSI(MF, CSI, PreferAscendingLS).size());
1051 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1052
1053 if (unsigned LibCallFrameSize = RVFI->getLibCallStackSize()) {
1054 if (NeedsDwarfCFI) {
1055 CFIBuilder.buildDefCFAOffset(LibCallFrameSize);
1056 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1057 CFIBuilder.buildOffset(CS.getReg(),
1058 MFI.getObjectOffset(CS.getFrameIdx()));
1059 }
1060 }
1061
1062 // FIXME (note copied from Lanai): This appears to be overallocating. Needs
1063 // investigation. Get the number of bytes to allocate from the FrameInfo.
1064 uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
1065 uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
1066 uint64_t RVVStackSize = RVFI->getRVVStackSize();
1067
1068 // Early exit if there is no need to allocate on the stack
1069 if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0)
1070 return;
1071
1072 // If the stack pointer has been marked as reserved, then produce an error if
1073 // the frame requires stack allocation
1074 if (STI.isRegisterReservedByUser(SPReg))
1076 MF.getFunction(), "Stack pointer required, but has been reserved."});
1077
1078 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1079 // Split the SP adjustment to reduce the offsets of callee saved spill.
1080 if (FirstSPAdjustAmount) {
1081 StackSize = FirstSPAdjustAmount;
1082 RealStackSize = FirstSPAdjustAmount;
1083 }
1084
1085 if (RVFI->useQCIInterrupt(MF)) {
1086 // The function starts with `QC.C.MIENTER(.NEST)`, so the `(QC.)CM.PUSH(FP)`
1087 // could only be the next instruction.
1088 ++PossiblePush;
1089
1090 if (NeedsDwarfCFI) {
1091 // Insert the CFI metadata before where we think the `(QC.)CM.PUSH(FP)`
1092 // could be. The PUSH will also get its own CFI metadata for its own
1093 // modifications, which should come after the PUSH.
1094 CFIInstBuilder PushCFIBuilder(MBB, PossiblePush,
1097 for (const CalleeSavedInfo &CS : getQCISavedInfo(MF, CSI))
1098 PushCFIBuilder.buildOffset(CS.getReg(),
1099 MFI.getObjectOffset(CS.getFrameIdx()));
1100 }
1101 }
1102
1103 if (RVFI->isPushable(MF) && PossiblePush != MBB.end() &&
1104 isPush(PossiblePush->getOpcode())) {
1105 // Use available stack adjustment in push instruction to allocate additional
1106 // stack space. Align the stack size down to a multiple of 16. This is
1107 // needed for RVE.
1108 // FIXME: Can we increase the stack size to a multiple of 16 instead?
1109 uint64_t StackAdj =
1110 std::min(alignDown(StackSize, 16), static_cast<uint64_t>(48));
1111 PossiblePush->getOperand(1).setImm(StackAdj);
1112 StackSize -= StackAdj;
1113
1114 if (NeedsDwarfCFI) {
1115 CFIBuilder.buildDefCFAOffset(RealStackSize - StackSize);
1116 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1117 CFIBuilder.buildOffset(CS.getReg(),
1118 MFI.getObjectOffset(CS.getFrameIdx()));
1119 }
1120 }
1121
1122 // Allocate space on the stack if necessary.
1124 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
1125 bool NeedProbe = TLI->hasInlineStackProbe(MF);
1126 uint64_t ProbeSize = TLI->getStackProbeSize(MF, getStackAlign());
1127 bool DynAllocation =
1128 MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
1129 if (StackSize != 0)
1130 allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, NeedsDwarfCFI,
1131 NeedProbe, ProbeSize, DynAllocation,
1133
1134 // Save SiFive CLIC CSRs into Stack
1136
1137 // The frame pointer is callee-saved, and code has been generated for us to
1138 // save it to the stack. We need to skip over the storing of callee-saved
1139 // registers as the frame pointer must be modified after it has been saved
1140 // to the stack, not before.
1141 // FIXME: assumes exactly one instruction is used to save each callee-saved
1142 // register.
1143 std::advance(MBBI,
1144 getUnmanagedInterruptCSI(MF, CSI, PreferAscendingLS).size());
1145 CFIBuilder.setInsertPoint(MBBI);
1146
1147 // Iterate over list of callee-saved registers and emit .cfi_offset
1148 // directives.
1149 if (NeedsDwarfCFI) {
1150 for (const CalleeSavedInfo &CS :
1151 getUnmanagedInterruptCSI(MF, CSI, PreferAscendingLS)) {
1152 MCRegister Reg = CS.getReg();
1153 int64_t Offset = MFI.getObjectOffset(CS.getFrameIdx());
1154 // Emit CFI for both sub-registers. The even register is at the base
1155 // offset and odd at base+4.
1156 if (RISCV::GPRPairRegClass.contains(Reg)) {
1157 MCRegister EvenReg = RI->getSubReg(Reg, RISCV::sub_gpr_even);
1158 MCRegister OddReg = RI->getSubReg(Reg, RISCV::sub_gpr_odd);
1159 CFIBuilder.buildOffset(EvenReg, Offset);
1160 CFIBuilder.buildOffset(OddReg, Offset + 4);
1161 } else {
1162 CFIBuilder.buildOffset(Reg, Offset);
1163 }
1164 }
1165 }
1166
1167 // Generate new FP.
1168 if (hasFP(MF)) {
1169 if (STI.isRegisterReservedByUser(FPReg))
1171 MF.getFunction(), "Frame pointer required, but has been reserved."});
1172 // The frame pointer does need to be reserved from register allocation.
1173 assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
1174
1175 // Some stack management variants automatically keep FP updated, so we don't
1176 // need an instruction to do so.
1177 if (!RVFI->hasImplicitFPUpdates(MF)) {
1178 RI->adjustReg(
1179 MBB, MBBI, DL, FPReg, SPReg,
1180 StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()),
1182 }
1183
1184 if (NeedsDwarfCFI)
1185 CFIBuilder.buildDefCFA(FPReg, RVFI->getVarArgsSaveSize());
1186 }
1187
1188 uint64_t SecondSPAdjustAmount = 0;
1189 // Emit the second SP adjustment after saving callee saved registers.
1190 if (FirstSPAdjustAmount) {
1191 SecondSPAdjustAmount = getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
1192 assert(SecondSPAdjustAmount > 0 &&
1193 "SecondSPAdjustAmount should be greater than zero");
1194
1195 allocateStack(MBB, MBBI, MF, SecondSPAdjustAmount,
1196 getStackSizeWithRVVPadding(MF), NeedsDwarfCFI && !hasFP(MF),
1197 NeedProbe, ProbeSize, DynAllocation,
1199 }
1200
1201 if (RVVStackSize) {
1202 if (NeedProbe) {
1203 allocateAndProbeStackForRVV(MF, MBB, MBBI, DL, RVVStackSize,
1205 NeedsDwarfCFI && !hasFP(MF), DynAllocation);
1206 } else {
1207 // We must keep the stack pointer aligned through any intermediate
1208 // updates.
1209 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
1210 StackOffset::getScalable(-RVVStackSize),
1212 }
1213
1214 if (NeedsDwarfCFI && !hasFP(MF)) {
1215 // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
1217 *RI, SPReg,
1218 StackOffset::get(getStackSizeWithRVVPadding(MF), RVVStackSize / 8)));
1219 }
1220
1221 std::advance(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
1222 if (NeedsDwarfCFI)
1223 emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
1224 }
1225
1226 if (hasFP(MF)) {
1227 // Realign Stack
1228 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1229 if (RI->hasStackRealignment(MF)) {
1230 Align MaxAlignment = MFI.getMaxAlign();
1231
1232 const RISCVInstrInfo *TII = STI.getInstrInfo();
1233 if (isInt<12>(-(int64_t)MaxAlignment.value())) {
1234 BuildMI(MBB, MBBI, DL, TII->get(RISCV::ANDI), SPReg)
1235 .addReg(SPReg)
1236 .addImm(-(int64_t)MaxAlignment.value())
1238 } else {
1239 unsigned ShiftAmount = Log2(MaxAlignment);
1240 Register VR =
1241 MF.getRegInfo().createVirtualRegister(&RISCV::GPRRegClass);
1242 BuildMI(MBB, MBBI, DL, TII->get(RISCV::SRLI), VR)
1243 .addReg(SPReg)
1244 .addImm(ShiftAmount)
1246 BuildMI(MBB, MBBI, DL, TII->get(RISCV::SLLI), SPReg)
1247 .addReg(VR)
1248 .addImm(ShiftAmount)
1250 }
1251 if (NeedProbe && RVVStackSize == 0) {
1252 // Do a probe if the align + size allocated just passed the probe size
1253 // and was not yet probed.
1254 if (SecondSPAdjustAmount < ProbeSize &&
1255 SecondSPAdjustAmount + MaxAlignment.value() >= ProbeSize) {
1256 bool IsRV64 = STI.is64Bit();
1257 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
1258 .addReg(RISCV::X0)
1259 .addReg(SPReg)
1260 .addImm(0)
1262 }
1263 }
1264 // FP will be used to restore the frame in the epilogue, so we need
1265 // another base register BP to record SP after re-alignment. SP will
1266 // track the current stack after allocating variable sized objects.
1267 if (hasBP(MF)) {
1268 // move BP, SP
1269 BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), BPReg)
1270 .addReg(SPReg)
1271 .addImm(0)
1273 }
1274 }
1275 }
1276}
1277
1278void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
1281 const DebugLoc &DL,
1282 uint64_t &StackSize,
1283 int64_t CFAOffset) const {
1285
1286 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(StackSize),
1288 StackSize = 0;
1289
1290 if (needsDwarfCFI(MF))
1292 .buildDefCFAOffset(CFAOffset);
1293}
1294
1296 MachineBasicBlock &MBB) const {
1297 const RISCVRegisterInfo *RI = STI.getRegisterInfo();
1298 MachineFrameInfo &MFI = MF.getFrameInfo();
1299 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1300 bool PreferAscendingLS = STI.preferAscendingLoadStore();
1301
1302 // All calls are tail calls in GHC calling conv, and functions have no
1303 // prologue/epilogue.
1305 return;
1306
1307 // Get the insert location for the epilogue. If there were no terminators in
1308 // the block, get the last instruction.
1310 DebugLoc DL;
1311 if (!MBB.empty()) {
1312 MBBI = MBB.getLastNonDebugInstr();
1313 if (MBBI != MBB.end())
1314 DL = MBBI->getDebugLoc();
1315
1316 MBBI = MBB.getFirstTerminator();
1317
1318 // Skip to before the restores of all callee-saved registers.
1319 while (MBBI != MBB.begin() &&
1320 std::prev(MBBI)->getFlag(MachineInstr::FrameDestroy))
1321 --MBBI;
1322 }
1323
1324 const auto &CSI = MFI.getCalleeSavedInfo();
1325
1326 // Skip to before the restores of scalar callee-saved registers
1327 // FIXME: assumes exactly one instruction is used to restore each
1328 // callee-saved register.
1329 auto FirstScalarCSRRestoreInsn =
1330 std::next(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
1331 CFIInstBuilder CFIBuilder(MBB, FirstScalarCSRRestoreInsn,
1333 bool NeedsDwarfCFI = needsDwarfCFI(MF);
1334
1335 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1336 uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1338 uint64_t StackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1340 RVFI->getReservedSpillsSize();
1341 uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
1342 uint64_t RVVStackSize = RVFI->getRVVStackSize();
1343
1344 // An unreserved call frame moves SP only around calls that pass arguments
1345 // on the stack. Otherwise SP is where the prologue left it, and the FP
1346 // restore is worth keeping only where it replaces the multi-instruction
1347 // undo of an RVV area or a split SP adjustment.
1348 bool CallFrameMovesSP =
1349 !hasReservedCallFrame(MF) &&
1350 (!MFI.isMaxCallFrameSizeComputed() || MFI.getMaxCallFrameSize() != 0);
1351 bool RestoreSPFromFP =
1352 RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
1353 CallFrameMovesSP ||
1354 (!hasReservedCallFrame(MF) && (RVVStackSize || FirstSPAdjustAmount));
1355 if (RVVStackSize) {
1356 // If RestoreSPFromFP the stack pointer will be restored using the frame
1357 // pointer value.
1358 if (!RestoreSPFromFP)
1359 RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, SPReg,
1360 StackOffset::getScalable(RVVStackSize),
1362
1363 if (NeedsDwarfCFI) {
1364 if (!hasFP(MF))
1365 CFIBuilder.buildDefCFA(SPReg, RealStackSize);
1366 emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
1367 }
1368 }
1369
1370 if (FirstSPAdjustAmount) {
1371 uint64_t SecondSPAdjustAmount =
1372 getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
1373 assert(SecondSPAdjustAmount > 0 &&
1374 "SecondSPAdjustAmount should be greater than zero");
1375
1376 // If RestoreSPFromFP the stack pointer will be restored using the frame
1377 // pointer value.
1378 if (!RestoreSPFromFP)
1379 RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, SPReg,
1380 StackOffset::getFixed(SecondSPAdjustAmount),
1382
1383 if (NeedsDwarfCFI && !hasFP(MF))
1384 CFIBuilder.buildDefCFAOffset(FirstSPAdjustAmount);
1385 }
1386
1387 // Restore the stack pointer using the value of the frame pointer. Only
1388 // necessary if the stack pointer was modified, meaning the stack size is
1389 // unknown.
1390 //
1391 // In order to make sure the stack point is right through the EH region,
1392 // we also need to restore stack pointer from the frame pointer if we
1393 // don't preserve stack space within prologue/epilogue for outgoing variables,
1394 // normally it's just checking the variable sized object is present or not
1395 // is enough, but we also don't preserve that at prologue/epilogue when
1396 // have vector objects in stack.
1397 if (RestoreSPFromFP) {
1398 assert(hasFP(MF) && "frame pointer should not have been eliminated");
1399 RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, FPReg,
1401 getStackAlign());
1402 }
1403
1404 if (NeedsDwarfCFI && hasFP(MF))
1405 CFIBuilder.buildDefCFA(SPReg, RealStackSize);
1406
1407 // Skip to after the restores of scalar callee-saved registers
1408 // FIXME: assumes exactly one instruction is used to restore each
1409 // callee-saved register.
1410 MBBI = std::next(FirstScalarCSRRestoreInsn,
1411 getUnmanagedInterruptCSI(MF, CSI, PreferAscendingLS).size());
1412 CFIBuilder.setInsertPoint(MBBI);
1413 emitSiFiveCLICPreemptibleRestores(MF, MBB, MBBI, CFIBuilder, DL);
1414
1415 auto emitRestoreCFI = [&](auto CSInfo) {
1416 for (auto &CS : CSInfo) {
1417 MCRegister Reg = CS.getReg();
1418 // Emit CFI for both sub-registers.
1419 if (RISCV::GPRPairRegClass.contains(Reg)) {
1420 MCRegister EvenReg = RI->getSubReg(Reg, RISCV::sub_gpr_even);
1421 MCRegister OddReg = RI->getSubReg(Reg, RISCV::sub_gpr_odd);
1422 CFIBuilder.buildRestore(EvenReg);
1423 CFIBuilder.buildRestore(OddReg);
1424 } else {
1425 CFIBuilder.buildRestore(Reg);
1426 }
1427 }
1428 };
1429
1430 if (RVFI->useSaveRestoreLibCalls(MF)) {
1431 if (RVFI->hasShadowStack(MF)) {
1432 // We don't use `__riscv_restore_<N>` to restore in this case, so we need
1433 // to make sure we're correctly accounting for the stack deallocation we
1434 // need to do, which now must include the allocation made by
1435 // `__riscv_save_<N>`.
1436 StackSize += RVFI->getLibCallStackSize();
1437
1440 MBBI = std::next(MBBI, LibcallCSI.size());
1441 CFIBuilder.setInsertPoint(MBBI);
1442
1443 if (NeedsDwarfCFI)
1444 emitRestoreCFI(LibcallCSI);
1445 } else {
1446 // tail __riscv_restore_[0-12] instruction is considered as a terminator,
1447 // therefore it is unnecessary to place any CFI instructions after it.
1448 // Just deallocate stack if needed and return.
1449 if (StackSize != 0)
1450 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1451 RVFI->getLibCallStackSize());
1452
1453 // Emit epilogue for shadow call stack.
1454 emitSCSEpilogue(MF, MBB, MBBI, DL);
1455 return;
1456 }
1457 }
1458
1459 // Recover callee-saved registers.
1460 if (NeedsDwarfCFI)
1461 emitRestoreCFI(getUnmanagedInterruptCSI(MF, CSI, PreferAscendingLS));
1462
1463 if (RVFI->isPushable(MF) && MBBI != MBB.end() && isPop(MBBI->getOpcode())) {
1464 // Use available stack adjustment in pop instruction to deallocate stack
1465 // space. Align the stack size down to a multiple of 16. This is needed for
1466 // RVE.
1467 // FIXME: Can we increase the stack size to a multiple of 16 instead?
1468 uint64_t StackAdj =
1469 std::min(alignDown(StackSize, 16), static_cast<uint64_t>(48));
1470 MBBI->getOperand(1).setImm(StackAdj);
1471 StackSize -= StackAdj;
1472
1473 if (StackSize != 0)
1474 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1475 /*stack_adj of cm.pop instr*/ RealStackSize - StackSize);
1476
1477 auto NextI = next_nodbg(MBBI, MBB.end());
1478 if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
1479 ++MBBI;
1480 if (NeedsDwarfCFI) {
1481 CFIBuilder.setInsertPoint(MBBI);
1482
1483 for (const CalleeSavedInfo &CS : getPushOrLibCallsSavedInfo(MF, CSI))
1484 CFIBuilder.buildRestore(CS.getReg());
1485
1486 // Update CFA Offset. If this is a QCI interrupt function, there will
1487 // be a leftover offset which is deallocated by `QC.C.MILEAVERET`,
1488 // otherwise getQCIInterruptStackSize() will be 0.
1489 CFIBuilder.buildDefCFAOffset(RVFI->getQCIInterruptStackSize());
1490 }
1491 }
1492 }
1493
1494 // Deallocate stack if StackSize isn't a zero yet. If this is a QCI interrupt
1495 // function, there will be a leftover offset which is deallocated by
1496 // `QC.C.MILEAVERET`, otherwise getQCIInterruptStackSize() will be 0.
1497 if (StackSize != 0)
1498 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1499 RVFI->getQCIInterruptStackSize());
1500
1501 // Emit epilogue for shadow call stack.
1502 emitSCSEpilogue(MF, MBB, MBBI, DL);
1503
1504 // SiFive CLIC needs to swap `sf.mscratchcsw` into `sp`
1506}
1507
1509 MCRegister Reg) {
1510 if (RISCV::GPRRegClass.contains(Reg))
1511 return Reg;
1512
1513 std::array<TargetRegisterClass const *, 2> RegisterClasses = {
1514 &RISCV::GPRF16RegClass, &RISCV::GPRF32RegClass};
1515 std::array<unsigned, 2> SubIdx = {RISCV::sub_16, RISCV::sub_32};
1516
1517 for (auto [RegClass, SubReg] : zip(RegisterClasses, SubIdx)) {
1518 if (RegClass->contains(Reg)) {
1519 if (MCRegister Super =
1520 TRI.getMatchingSuperReg(Reg, SubReg, &RISCV::GPRRegClass))
1521 return Super;
1522 }
1523 }
1524
1526 "getPhysicalGPR called with unsupported register");
1527}
1528
1530 const TargetRegisterInfo &TRI,
1531 MCRegister Reg) {
1532 if (!STI.hasStdExtF())
1533 return MCRegister();
1534
1535 TargetRegisterClass const *LargestFPRegClass = STI.getLargestFPRegClass();
1536 assert(LargestFPRegClass);
1537
1538 if (LargestFPRegClass->contains(Reg))
1539 return Reg;
1540
1541 std::array<TargetRegisterClass const *, 3> RegisterClasses = {
1542 &RISCV::FPR16RegClass, &RISCV::FPR32RegClass, &RISCV::FPR64RegClass};
1543 std::array<unsigned, 3> SubIdx = {RISCV::sub_16, RISCV::sub_32,
1544 RISCV::sub_64};
1545
1546 for (auto [RegClass, SubReg] : zip(RegisterClasses, SubIdx)) {
1547 if (RegClass->contains(Reg)) {
1548 if (MCRegister Super =
1549 TRI.getMatchingSuperReg(Reg, SubReg, LargestFPRegClass))
1550 return Super;
1551 }
1552 }
1553
1554 // Reg is bigger than what's currently available for the target, we can ignore
1555 // it.
1556 return MCRegister();
1557}
1558
1559void RISCVFrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
1561 RegScavenger *RS) const {
1562 // Insertion point.
1564
1565 // Fake a debug loc.
1566 DebugLoc DL;
1567 if (MBBI != MBB.end())
1568 DL = MBBI->getDebugLoc();
1569
1570 const MachineFunction &MF = *MBB.getParent();
1571 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
1572 const RISCVInstrInfo &TII = *STI.getInstrInfo();
1573
1574 BitVector FinalRegsToZero(TRI.getNumRegs());
1575
1576 bool HasVRegister = false;
1577
1578 for (MCRegister Reg : RegsToZero.set_bits()) {
1579 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
1580 FinalRegsToZero.set(getPhysicalGPR(TRI, Reg).id());
1581 } else if (RISCV::GPRPairRegClass.contains(Reg)) {
1582 FinalRegsToZero.set(
1583 getPhysicalGPR(TRI, TRI.getSubReg(Reg, RISCV::sub_gpr_even)).id());
1584 FinalRegsToZero.set(
1585 getPhysicalGPR(TRI, TRI.getSubReg(Reg, RISCV::sub_gpr_odd)).id());
1586 } else if (TRI.isFPRegister(Reg)) {
1587 if (MCRegister MaybeReg = getLargestFPRegisterOrZero(STI, TRI, Reg))
1588 FinalRegsToZero.set(MaybeReg.id());
1590 TRI.getMinimalPhysRegClass(Reg))) {
1591 if (!STI.hasVInstructions())
1592 continue;
1593 HasVRegister = true;
1594
1595 for (MCRegister SubReg : TRI.subregs_inclusive(Reg)) {
1596 if (TRI.subregs(SubReg).empty())
1597 FinalRegsToZero.set(SubReg.id());
1598 }
1599 }
1600 }
1601
1602 if (HasVRegister) {
1603 RISCVVType::VLMUL VLMUL = RISCVVType::encodeLMUL(1, /*Fractional=*/false);
1604 unsigned VTypeImm = RISCVVType::encodeVTYPE(
1605 VLMUL, /*SEW=*/32, /*TailAgnostic=*/true, /*MaskAgnostic=*/true);
1606
1607 MCRegister TemporaryReg = RISCV::NoRegister;
1608 for (MCRegister Reg : FinalRegsToZero.set_bits()) {
1609 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
1610 TemporaryReg = Reg;
1611 break;
1612 }
1613 }
1614
1615 if (TemporaryReg == RISCV::NoRegister) {
1616 RS->enterBasicBlockEnd(MBB);
1617 TemporaryReg = RS->scavengeRegisterBackwards(RISCV::GPRRegClass, MBBI,
1618 /*RestoreAfter=*/false,
1619 /*SPAdj=*/0);
1620 }
1621
1622 if (MBB.getParent()
1623 ->getFunction()
1624 .getFnAttribute("zero-call-used-regs")
1625 .getValueAsString() == "used")
1626 FinalRegsToZero.set(TemporaryReg.id());
1627
1628 BuildMI(MBB, MBBI, DL, TII.get(RISCV::VSETVLI), TemporaryReg)
1629 .addReg(RISCV::X0)
1630 .addImm(VTypeImm)
1631 .addReg(RISCV::VL, RegState::ImplicitDefine)
1632 .addReg(RISCV::VTYPE, RegState::ImplicitDefine);
1633 }
1634
1635 for (MCRegister Reg : FinalRegsToZero.set_bits())
1636 TII.buildClearRegister(Reg, MBB, MBBI, DL);
1637}
1638
1641 Register &FrameReg) const {
1642 const MachineFrameInfo &MFI = MF.getFrameInfo();
1644 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1645
1646 // Callee-saved registers should be referenced relative to the stack
1647 // pointer (positive offset), otherwise use the frame pointer (negative
1648 // offset).
1649 const auto &CSI = getUnmanagedCSI(MF, MFI.getCalleeSavedInfo(),
1650 STI.preferAscendingLoadStore());
1651 int MinCSFI = 0;
1652 int MaxCSFI = -1;
1654 auto StackID = MFI.getStackID(FI);
1655
1656 assert((StackID == TargetStackID::Default ||
1657 StackID == TargetStackID::ScalableVector) &&
1658 "Unexpected stack ID for the frame object.");
1659 if (StackID == TargetStackID::Default) {
1660 assert(getOffsetOfLocalArea() == 0 && "LocalAreaOffset is not 0!");
1662 MFI.getOffsetAdjustment());
1663 } else if (StackID == TargetStackID::ScalableVector) {
1665 }
1666
1667 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1668
1669 if (CSI.size()) {
1670 MinCSFI = std::min(CSI.front().getFrameIdx(), CSI.back().getFrameIdx());
1671 MaxCSFI = std::max(CSI.front().getFrameIdx(), CSI.back().getFrameIdx());
1672 }
1673
1674 auto FIIsCalleeSave = [&](const CalleeSavedInfo &CS) -> bool {
1675 return FI == CS.getFrameIdx();
1676 };
1677
1678 bool IsInterruptCSR = RVFI->isSiFivePreemptibleInterrupt(MF) &&
1679 (FI == RVFI->getInterruptCSRFrameIndex(0) ||
1680 FI == RVFI->getInterruptCSRFrameIndex(1));
1681 bool IsInlineRestoredLibcallCSR = llvm::any_of(
1682 getPushOrLibCallsSavedInfo(MF, MFI.getCalleeSavedInfo()), FIIsCalleeSave);
1683 if ((FI >= MinCSFI && FI <= MaxCSFI) || IsInterruptCSR ||
1684 IsInlineRestoredLibcallCSR) {
1685 FrameReg = SPReg;
1686
1687 if (FirstSPAdjustAmount)
1688 Offset += StackOffset::getFixed(FirstSPAdjustAmount);
1689 else
1691 return Offset;
1692 }
1693
1694 if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(FI)) {
1695 // If the stack was realigned, the frame pointer is set in order to allow
1696 // SP to be restored, so we need another base register to record the stack
1697 // after realignment.
1698 // |--------------------------| --
1699 // | callee-allocated save | | <----|
1700 // | area for register varargs| | |
1701 // |--------------------------| <-- FP |
1702 // | callee-saved registers | | |
1703 // |--------------------------| -- |
1704 // | realignment (the size of | | |
1705 // | this area is not counted | | |
1706 // | in MFI.getStackSize()) | | |
1707 // |--------------------------| -- |-- MFI.getStackSize()
1708 // | RVV alignment padding | | |
1709 // | (not counted in | | |
1710 // | MFI.getStackSize() but | | |
1711 // | counted in | | |
1712 // | RVFI.getRVVStackSize()) | | |
1713 // |--------------------------| -- |
1714 // | RVV objects | | |
1715 // | (not counted in | | |
1716 // | MFI.getStackSize()) | | |
1717 // |--------------------------| -- |
1718 // | padding before RVV | | |
1719 // | (not counted in | | |
1720 // | MFI.getStackSize() or in | | |
1721 // | RVFI.getRVVStackSize()) | | |
1722 // |--------------------------| -- |
1723 // | scalar local variables | | <----'
1724 // |--------------------------| -- <-- BP (if var sized objects present)
1725 // | VarSize objects | |
1726 // |--------------------------| -- <-- SP
1727 if (hasBP(MF)) {
1728 FrameReg = RISCVABI::getBPReg();
1729 } else {
1730 // VarSize objects must be empty in this case!
1731 assert(!MFI.hasVarSizedObjects());
1732 FrameReg = SPReg;
1733 }
1734 } else if (!RI->hasStackRealignment(MF)) {
1735 // Note: Keeping the following as multiple 'if' statements rather than
1736 // merging to a single expression for readability.
1737 if (!hasFP(MF)) {
1738 // No FP available, must use SP.
1739 FrameReg = SPReg;
1740 } else {
1741 FrameReg = FPReg;
1742 // SP-relative addressing is only valid when SP is stable throughout
1743 // the function body: no dynamic SP adjustments for outgoing call args,
1744 // no variable-sized objects, and no RVV scalable stack regions.
1745 // hasReservedCallFrame() conservatively encompasses all these checks.
1746 if (hasReservedCallFrame(MF)) {
1747 // Both FP and SP are candidates.
1748 // Prefer SP when the SP-relative offset fits in the compressed
1749 // instruction immediate range.
1750 int64_t SPOff = Offset.getFixed() + MFI.getStackSize();
1751 int64_t CLWSPMaxOffset = 252;
1752 int64_t CLDSPMaxOffset = 504;
1753 int64_t SPThreshold = STI.is64Bit() ? CLDSPMaxOffset : CLWSPMaxOffset;
1754 if (SPOff >= 0 && SPOff <= SPThreshold)
1755 FrameReg = SPReg;
1756 }
1757 }
1758 } else {
1759 assert(RI->hasStackRealignment(MF) && MFI.isFixedObjectIndex(FI) &&
1760 "Expected fixed object with stack realignment");
1761 assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
1762 FrameReg = FPReg;
1763 }
1764
1765 if (FrameReg == FPReg) {
1766 Offset += StackOffset::getFixed(RVFI->getVarArgsSaveSize());
1767 // When using FP to access scalable vector objects, we need to minus
1768 // the frame size.
1769 //
1770 // |--------------------------| --
1771 // | callee-allocated save | |
1772 // | area for register varargs| |
1773 // |--------------------------| | -- <-- FP
1774 // | callee-saved registers | |
1775 // |--------------------------| | MFI.getStackSize()
1776 // | scalar local variables | |
1777 // |--------------------------| -- (Offset of RVV objects is from here.)
1778 // | RVV objects |
1779 // |--------------------------|
1780 // | VarSize objects |
1781 // |--------------------------| <-- SP
1782 if (StackID == TargetStackID::ScalableVector) {
1783 assert(!RI->hasStackRealignment(MF) &&
1784 "Can't index across variable sized realign");
1785 // We don't expect any extra RVV alignment padding, as the stack size
1786 // and RVV object sections should be correct aligned in their own
1787 // right.
1789 "Inconsistent stack layout");
1791 }
1792 return Offset;
1793 }
1794
1795 // This case handles indexing off both SP and BP.
1796 // If indexing off SP, there must not be any var sized objects
1797 assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects());
1798
1799 // When using SP to access frame objects, we need to add RVV stack size.
1800 //
1801 // |--------------------------| --
1802 // | callee-allocated save | | <----|
1803 // | area for register varargs| | |
1804 // |--------------------------| | | <-- FP
1805 // | callee-saved registers | | |
1806 // |--------------------------| -- |
1807 // | RVV alignment padding | | |
1808 // | (not counted in | | |
1809 // | MFI.getStackSize() but | | |
1810 // | counted in | | |
1811 // | RVFI.getRVVStackSize()) | | |
1812 // |--------------------------| -- |
1813 // | RVV objects | | |-- MFI.getStackSize()
1814 // | (not counted in | | |
1815 // | MFI.getStackSize()) | | |
1816 // |--------------------------| -- |
1817 // | padding before RVV | | |
1818 // | (not counted in | | |
1819 // | MFI.getStackSize()) | | |
1820 // |--------------------------| -- |
1821 // | scalar local variables | | <----'
1822 // |--------------------------| -- <-- BP (if var sized objects present)
1823 // | VarSize objects | |
1824 // |--------------------------| -- <-- SP
1825 //
1826 // The total amount of padding surrounding RVV objects is described by
1827 // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV
1828 // objects to the required alignment.
1829 if (MFI.getStackID(FI) == TargetStackID::Default) {
1830 if (MFI.isFixedObjectIndex(FI)) {
1831 assert(!RI->hasStackRealignment(MF) &&
1832 "Can't index across variable sized realign");
1834 RVFI->getRVVStackSize());
1835 } else {
1837 }
1838 } else if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
1839 // Ensure the base of the RVV stack is correctly aligned: add on the
1840 // alignment padding.
1841 int64_t ScalarLocalVarSize =
1842 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
1843 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
1844 Offset += StackOffset::get(ScalarLocalVarSize, RVFI->getRVVStackSize());
1845 }
1846 return Offset;
1847}
1848
1850 const Register &Reg) {
1851 MCRegister BaseReg = TRI.getSubReg(Reg, RISCV::sub_vrm1_0);
1852 // If it's not a grouped vector register, it doesn't have subregister, so
1853 // the base register is just itself.
1854 if (!BaseReg.isValid())
1855 BaseReg = Reg;
1856 return BaseReg;
1857}
1858
1860 BitVector &SavedRegs,
1861 RegScavenger *RS) const {
1863
1864 // In TargetFrameLowering::determineCalleeSaves, any vector register is marked
1865 // as saved if any of its subregister is clobbered, this is not correct in
1866 // vector registers. We only want the vector register to be marked as saved
1867 // if all of its subregisters are clobbered.
1868 // For example:
1869 // Original behavior: If v24 is marked, v24m2, v24m4, v24m8 are also marked.
1870 // Correct behavior: v24m2 is marked only if v24 and v25 are marked.
1872 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
1873 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
1874 for (unsigned i = 0; CSRegs[i]; ++i) {
1875 unsigned CSReg = CSRegs[i];
1876 // Only vector registers need special care.
1877 if (!RISCV::VRRegClass.contains(getRVVBaseRegister(TRI, CSReg)))
1878 continue;
1879
1880 SavedRegs.reset(CSReg);
1881
1882 auto SubRegs = TRI.subregs(CSReg);
1883 // Set the register and all its subregisters.
1884 if (!MRI.def_empty(CSReg) || MRI.getUsedPhysRegsMask().test(CSReg)) {
1885 SavedRegs.set(CSReg);
1886 for (unsigned Reg : SubRegs)
1887 SavedRegs.set(Reg);
1888 }
1889
1890 }
1891
1892 // Unconditionally spill RA and FP only if the function uses a frame
1893 // pointer.
1894 if (hasFP(MF)) {
1895 SavedRegs.set(RAReg);
1896 SavedRegs.set(FPReg);
1897 }
1898 // Mark BP as used if function has dedicated base pointer.
1899 if (hasBP(MF))
1900 SavedRegs.set(RISCVABI::getBPReg());
1901
1902 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1903 // X5 is used as a temporary for saving and restoring `mcause` and `mepc`.
1904 if (RVFI->isSiFivePreemptibleInterrupt(MF))
1905 SavedRegs.set(RISCV::X5);
1906
1907 // When using cm.push/pop we must save X27 if we save X26.
1908 if (RVFI->isPushable(MF) && SavedRegs.test(RISCV::X26))
1909 SavedRegs.set(RISCV::X27);
1910
1911 // For Zilsd on RV32, append GPRPair registers to the CSR list. This prevents
1912 // the need to create register sets for each abi which is a lot more complex.
1913 // Don't use Zilsd for callee-saved coalescing if the required alignment
1914 // exceeds the stack alignment or when Zcmp/Xqccmp or save/restore libcalls
1915 // are enabled.
1916 bool UseZilsd = !STI.is64Bit() && STI.hasStdExtZilsd() &&
1917 STI.getZilsdAlign() <= getStackAlign() &&
1918 !RVFI->isPushable(MF) && !RVFI->useSaveRestoreLibCalls(MF);
1919 if (UseZilsd) {
1922 for (unsigned i = 0; CSRegs[i]; ++i) {
1923 NewCSRs.push_back(CSRegs[i]);
1924 CSRSet.insert(CSRegs[i]);
1925 }
1926
1927 // Append GPRPair registers for pairs where both sub-registers are in CSR
1928 // list. Iterate through all GPRPairs and check if both sub-regs are CSRs.
1929 for (MCPhysReg Pair : RISCV::GPRPairRegClass) {
1930 // Do not append a pair that's already in the CSR list.
1931 if (CSRSet.contains(Pair))
1932 continue;
1933 MCRegister EvenReg = TRI.getSubReg(Pair, RISCV::sub_gpr_even);
1934 MCRegister OddReg = TRI.getSubReg(Pair, RISCV::sub_gpr_odd);
1935 if (CSRSet.contains(EvenReg.id()) && CSRSet.contains(OddReg.id())) {
1936 NewCSRs.push_back(Pair);
1937 CSRSet.insert(Pair);
1938 }
1939 }
1940
1941 MRI.setCalleeSavedRegs(NewCSRs);
1942 CSRegs = MRI.getCalleeSavedRegs();
1943 }
1944
1945 // Check if all subregisters are marked for saving. If so, set the super
1946 // register bit. For GPRPair, only check sub_gpr_even and sub_gpr_odd, not
1947 // aliases like X8_W or X8_H which are not set in SavedRegs.
1948 for (unsigned i = 0; CSRegs[i]; ++i) {
1949 MCRegister CSReg = CSRegs[i];
1950 bool CombineToSuperReg;
1951 if (RISCV::GPRPairRegClass.contains(CSReg)) {
1952 MCRegister EvenReg = TRI.getSubReg(CSReg, RISCV::sub_gpr_even);
1953 MCRegister OddReg = TRI.getSubReg(CSReg, RISCV::sub_gpr_odd);
1954 CombineToSuperReg =
1955 SavedRegs.test(EvenReg.id()) && SavedRegs.test(OddReg.id());
1956 // If s0(x8) is used as FP we can't generate load/store pair because it
1957 // breaks the frame chain.
1958 if (hasFP(MF) && CSReg == RISCV::X8_X9)
1959 CombineToSuperReg = false;
1960 } else {
1961 auto SubRegs = TRI.subregs(CSReg);
1962 CombineToSuperReg =
1963 !SubRegs.empty() && llvm::all_of(SubRegs, [&](unsigned Reg) {
1964 return SavedRegs.test(Reg);
1965 });
1966 }
1967
1968 if (CombineToSuperReg)
1969 SavedRegs.set(CSReg);
1970 }
1971
1972 // SiFive Preemptible Interrupt Handlers need additional frame entries
1974}
1975
1976std::pair<int64_t, Align>
1977RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
1978 MachineFrameInfo &MFI = MF.getFrameInfo();
1979 // Create a buffer of RVV objects to allocate.
1980 SmallVector<int, 8> ObjectsToAllocate;
1981 auto pushRVVObjects = [&](int FIBegin, int FIEnd) {
1982 for (int I = FIBegin, E = FIEnd; I != E; ++I) {
1983 unsigned StackID = MFI.getStackID(I);
1984 if (StackID != TargetStackID::ScalableVector)
1985 continue;
1986 if (MFI.isDeadObjectIndex(I))
1987 continue;
1988
1989 ObjectsToAllocate.push_back(I);
1990 }
1991 };
1992 // First push RVV Callee Saved object, then push RVV stack object
1993 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
1994 const auto &RVVCSI = getRVVCalleeSavedInfo(MF, CSI);
1995 if (!RVVCSI.empty())
1996 pushRVVObjects(RVVCSI[0].getFrameIdx(),
1997 RVVCSI[RVVCSI.size() - 1].getFrameIdx() + 1);
1998 pushRVVObjects(0, MFI.getObjectIndexEnd() - RVVCSI.size());
1999
2000 // The minimum alignment is 16 bytes.
2001 Align RVVStackAlign(16);
2002 const auto &ST = MF.getSubtarget<RISCVSubtarget>();
2003
2004 if (!ST.hasVInstructions()) {
2005 assert(ObjectsToAllocate.empty() &&
2006 "Can't allocate scalable-vector objects without V instructions");
2007 return std::make_pair(0, RVVStackAlign);
2008 }
2009
2010 // Allocate all RVV locals and spills
2011 int64_t Offset = 0;
2012 for (int FI : ObjectsToAllocate) {
2013 // ObjectSize in bytes.
2014 int64_t ObjectSize = MFI.getObjectSize(FI);
2015 auto ObjectAlign =
2016 std::max(Align(RISCV::RVVBytesPerBlock), MFI.getObjectAlign(FI));
2017 // If the data type is the fractional vector type, reserve one vector
2018 // register for it.
2019 if (ObjectSize < RISCV::RVVBytesPerBlock)
2020 ObjectSize = RISCV::RVVBytesPerBlock;
2021 Offset = alignTo(Offset + ObjectSize, ObjectAlign);
2022 MFI.setObjectOffset(FI, -Offset);
2023 // Update the maximum alignment of the RVV stack section
2024 RVVStackAlign = std::max(RVVStackAlign, ObjectAlign);
2025 }
2026
2027 uint64_t StackSize = Offset;
2028
2029 // Ensure the alignment of the RVV stack. Since we want the most-aligned
2030 // object right at the bottom (i.e., any padding at the top of the frame),
2031 // readjust all RVV objects down by the alignment padding.
2032 // Stack size and offsets are multiples of vscale, stack alignment is in
2033 // bytes, we can divide stack alignment by minimum vscale to get a maximum
2034 // stack alignment multiple of vscale.
2035 auto VScale =
2036 std::max<uint64_t>(ST.getRealMinVLen() / RISCV::RVVBitsPerBlock, 1);
2037 if (auto RVVStackAlignVScale = RVVStackAlign.value() / VScale) {
2038 if (auto AlignmentPadding =
2039 offsetToAlignment(StackSize, Align(RVVStackAlignVScale))) {
2040 StackSize += AlignmentPadding;
2041 for (int FI : ObjectsToAllocate)
2042 MFI.setObjectOffset(FI, MFI.getObjectOffset(FI) - AlignmentPadding);
2043 }
2044 }
2045
2046 return std::make_pair(StackSize, RVVStackAlign);
2047}
2048
2050 // For RVV spill, scalable stack offsets computing requires up to two scratch
2051 // registers
2052 static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2;
2053
2054 // For RVV spill, non-scalable stack offsets computing requires up to one
2055 // scratch register.
2056 static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1;
2057
2058 // ADDI instruction's destination register can be used for computing
2059 // offsets. So Scalable stack offsets require up to one scratch register.
2060 static constexpr unsigned ScavSlotsADDIScalableObject = 1;
2061
2062 static constexpr unsigned MaxScavSlotsNumKnown =
2063 std::max({ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject,
2064 ScavSlotsNumRVVSpillNonScalableObject});
2065
2066 unsigned MaxScavSlotsNum = 0;
2068 return false;
2069 for (const MachineBasicBlock &MBB : MF)
2070 for (const MachineInstr &MI : MBB) {
2071 bool IsRVVSpill = RISCV::isRVVSpill(MI);
2072 for (auto &MO : MI.operands()) {
2073 if (!MO.isFI())
2074 continue;
2075 bool IsScalableVectorID = MF.getFrameInfo().getStackID(MO.getIndex()) ==
2077 if (IsRVVSpill) {
2078 MaxScavSlotsNum = std::max(
2079 MaxScavSlotsNum, IsScalableVectorID
2080 ? ScavSlotsNumRVVSpillScalableObject
2081 : ScavSlotsNumRVVSpillNonScalableObject);
2082 } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) {
2083 MaxScavSlotsNum =
2084 std::max(MaxScavSlotsNum, ScavSlotsADDIScalableObject);
2085 }
2086 }
2087 if (MaxScavSlotsNum == MaxScavSlotsNumKnown)
2088 return MaxScavSlotsNumKnown;
2089 }
2090 return MaxScavSlotsNum;
2091}
2092
2093static bool hasRVVFrameObject(const MachineFunction &MF) {
2094 // Originally, the function will scan all the stack objects to check whether
2095 // if there is any scalable vector object on the stack or not. However, it
2096 // causes errors in the register allocator. In issue 53016, it returns false
2097 // before RA because there is no RVV stack objects. After RA, it returns true
2098 // because there are spilling slots for RVV values during RA. It will not
2099 // reserve BP during register allocation and generate BP access in the PEI
2100 // pass due to the inconsistent behavior of the function.
2101 //
2102 // The function is changed to use hasVInstructions() as the return value. It
2103 // is not precise, but it can make the register allocation correct.
2104 //
2105 // FIXME: Find a better way to make the decision or revisit the solution in
2106 // D103622.
2107 //
2108 // Refer to https://github.com/llvm/llvm-project/issues/53016.
2109 return MF.getSubtarget<RISCVSubtarget>().hasVInstructions();
2110}
2111
2113 const RISCVInstrInfo &TII) {
2114 unsigned FnSize = 0;
2115 for (auto &MBB : MF) {
2116 for (auto &MI : MBB) {
2117 // Far branches over 20-bit offset will be relaxed in branch relaxation
2118 // pass. In the worst case, conditional branches will be relaxed into
2119 // the following instruction sequence. Unconditional branches are
2120 // relaxed in the same way, with the exception that there is no first
2121 // branch instruction.
2122 //
2123 // foo
2124 // bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes
2125 // sd s11, 0(sp) # 4 bytes, or 2 bytes with Zca
2126 // jump .restore, s11 # 8 bytes
2127 // .rev_cond
2128 // bar
2129 // j .dest_bb # 4 bytes, or 2 bytes with Zca
2130 // .restore:
2131 // ld s11, 0(sp) # 4 bytes, or 2 bytes with Zca
2132 // .dest:
2133 // baz
2134 if (MI.isConditionalBranch())
2135 FnSize += TII.getInstSizeInBytes(MI);
2136 if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) {
2137 if (MF.getSubtarget<RISCVSubtarget>().hasStdExtZca())
2138 FnSize += 2 + 8 + 2 + 2;
2139 else
2140 FnSize += 4 + 8 + 4 + 4;
2141 continue;
2142 }
2143
2144 FnSize += TII.getInstSizeInBytes(MI);
2145 }
2146 }
2147 return FnSize;
2148}
2149
2151 MachineFunction &MF, RegScavenger *RS) const {
2152 const RISCVRegisterInfo *RegInfo =
2153 MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
2154 const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
2155 MachineFrameInfo &MFI = MF.getFrameInfo();
2156 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
2157 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2158
2159 int64_t RVVStackSize;
2160 Align RVVStackAlign;
2161 std::tie(RVVStackSize, RVVStackAlign) = assignRVVStackObjectOffsets(MF);
2162
2163 RVFI->setRVVStackSize(RVVStackSize);
2164 RVFI->setRVVStackAlign(RVVStackAlign);
2165
2166 if (hasRVVFrameObject(MF)) {
2167 // Ensure the entire stack is aligned to at least the RVV requirement: some
2168 // scalable-vector object alignments are not considered by the
2169 // target-independent code.
2170 MFI.ensureMaxAlignment(RVVStackAlign);
2171 }
2172
2173 unsigned ScavSlotsNum = 0;
2174
2175 // estimateStackSize has been observed to under-estimate the final stack
2176 // size, so give ourselves wiggle-room by checking for stack size
2177 // representable an 11-bit signed field rather than 12-bits.
2178 if (!isInt<11>(MFI.estimateStackSize(MF)))
2179 ScavSlotsNum = 1;
2180
2181 // Far branches over 20-bit offset require a spill slot for scratch register.
2182 bool IsLargeFunction = !isInt<20>(estimateFunctionSizeInBytes(MF, *TII));
2183 if (IsLargeFunction)
2184 ScavSlotsNum = std::max(ScavSlotsNum, 1u);
2185
2186 // RVV loads & stores have no capacity to hold the immediate address offsets
2187 // so we must always reserve an emergency spill slot if the MachineFunction
2188 // contains any RVV spills.
2189 ScavSlotsNum = std::max(ScavSlotsNum, getScavSlotsNumForRVV(MF));
2190
2191 for (unsigned I = 0; I < ScavSlotsNum; I++) {
2192 int FI = MFI.CreateSpillStackObject(RegInfo->getSpillSize(*RC),
2193 RegInfo->getSpillAlign(*RC));
2194 RS->addScavengingFrameIndex(FI);
2195
2196 if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1)
2197 RVFI->setBranchRelaxationScratchFrameIndex(FI);
2198 }
2199
2200 unsigned Size = RVFI->getReservedSpillsSize();
2201 for (const auto &Info : MFI.getCalleeSavedInfo()) {
2202 int FrameIdx = Info.getFrameIdx();
2203 if (FrameIdx < 0 || MFI.getStackID(FrameIdx) != TargetStackID::Default)
2204 continue;
2205
2206 Size += MFI.getObjectSize(FrameIdx);
2207 }
2208 RVFI->setCalleeSavedStackSize(Size);
2209}
2210
2211// Not preserve stack space within prologue for outgoing variables when the
2212// function contains variable size objects or there are vector objects accessed
2213// by the frame pointer.
2214// Let eliminateCallFramePseudoInstr preserve stack space for it.
2216 return !MF.getFrameInfo().hasVarSizedObjects() &&
2217 !(hasFP(MF) && hasRVVFrameObject(MF));
2218}
2219
2220// Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions.
2224 DebugLoc DL = MI->getDebugLoc();
2225
2226 if (!hasReservedCallFrame(MF)) {
2227 // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and
2228 // ADJCALLSTACKUP must be converted to instructions manipulating the stack
2229 // pointer. This is necessary when there is a variable length stack
2230 // allocation (e.g. alloca), which means it's not possible to allocate
2231 // space for outgoing arguments from within the function prologue.
2232 int64_t Amount = MI->getOperand(0).getImm();
2233
2234 if (Amount != 0) {
2235 // Ensure the stack remains aligned after adjustment.
2236 Amount = alignSPAdjust(Amount);
2237
2238 if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN)
2239 Amount = -Amount;
2240
2241 const RISCVTargetLowering *TLI =
2242 MF.getSubtarget<RISCVSubtarget>().getTargetLowering();
2243 int64_t ProbeSize = TLI->getStackProbeSize(MF, getStackAlign());
2244 if (TLI->hasInlineStackProbe(MF) && -Amount >= ProbeSize) {
2245 // When stack probing is enabled, the decrement of SP may need to be
2246 // probed. We can handle both the decrement and the probing in
2247 // allocateStack.
2248 bool DynAllocation =
2249 MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
2250 allocateStack(MBB, MI, MF, -Amount, -Amount,
2251 needsDwarfCFI(MF) && !hasFP(MF),
2252 /*NeedProbe=*/true, ProbeSize, DynAllocation,
2254 inlineStackProbe(MF, MBB);
2255 } else {
2256 const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
2259 }
2260 }
2261 }
2262
2263 return MBB.erase(MI);
2264}
2265
2266// We would like to split the SP adjustment to reduce prologue/epilogue
2267// as following instructions. In this way, the offset of the callee saved
2268// register could fit in a single store. Supposed that the first sp adjust
2269// amount is 2032.
2270// add sp,sp,-2032
2271// sw ra,2028(sp)
2272// sw s0,2024(sp)
2273// sw s1,2020(sp)
2274// sw s3,2012(sp)
2275// sw s4,2008(sp)
2276// add sp,sp,-64
2277uint64_t
2279 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2280 const MachineFrameInfo &MFI = MF.getFrameInfo();
2281 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
2282 uint64_t StackSize = getStackSizeWithRVVPadding(MF);
2283
2284 // Disable SplitSPAdjust if save-restore libcall, push/pop or QCI interrupts
2285 // are used. The callee-saved registers will be pushed by the save-restore
2286 // libcalls, so we don't have to split the SP adjustment in this case.
2287 if (RVFI->getReservedSpillsSize())
2288 return 0;
2289
2290 // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed
2291 // 12-bit and there exists a callee-saved register needing to be pushed.
2292 if (!isInt<12>(StackSize) && (CSI.size() > 0)) {
2293 // FirstSPAdjustAmount is chosen at most as (2048 - StackAlign) because
2294 // 2048 will cause sp = sp + 2048 in the epilogue to be split into multiple
2295 // instructions. Offsets smaller than 2048 can fit in a single load/store
2296 // instruction, and we have to stick with the stack alignment. 2048 has
2297 // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for
2298 // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment.
2299 const uint64_t StackAlign = getStackAlign().value();
2300
2301 // Amount of (2048 - StackAlign) will prevent callee saved and restored
2302 // instructions be compressed, so try to adjust the amount to the largest
2303 // offset that stack compression instructions accept when target supports
2304 // compression instructions.
2305 if (STI.hasStdExtZca()) {
2306 // The compression extensions may support the following instructions:
2307 // riscv32: c.lwsp rd, offset[7:2] => 2^(6 + 2)
2308 // c.swsp rs2, offset[7:2] => 2^(6 + 2)
2309 // c.flwsp rd, offset[7:2] => 2^(6 + 2)
2310 // c.fswsp rs2, offset[7:2] => 2^(6 + 2)
2311 // riscv64: c.ldsp rd, offset[8:3] => 2^(6 + 3)
2312 // c.sdsp rs2, offset[8:3] => 2^(6 + 3)
2313 // c.fldsp rd, offset[8:3] => 2^(6 + 3)
2314 // c.fsdsp rs2, offset[8:3] => 2^(6 + 3)
2315 const uint64_t RVCompressLen = STI.getXLen() * 8;
2316 // Compared with amount (2048 - StackAlign), StackSize needs to
2317 // satisfy the following conditions to avoid using more instructions
2318 // to adjust the sp after adjusting the amount, such as
2319 // StackSize meets the condition (StackSize <= 2048 + RVCompressLen),
2320 // case1: Amount is 2048 - StackAlign: use addi + addi to adjust sp.
2321 // case2: Amount is RVCompressLen: use addi + addi to adjust sp.
2322 auto CanCompress = [&](uint64_t CompressLen) -> bool {
2323 if (StackSize <= 2047 + CompressLen ||
2324 (StackSize > 2048 * 2 - StackAlign &&
2325 StackSize <= 2047 * 2 + CompressLen) ||
2326 StackSize > 2048 * 3 - StackAlign)
2327 return true;
2328
2329 return false;
2330 };
2331 // In the epilogue, addi sp, sp, 496 is used to recover the sp and it
2332 // can be compressed(C.ADDI16SP, offset can be [-512, 496]), but
2333 // addi sp, sp, 512 can not be compressed. So try to use 496 first.
2334 const uint64_t ADDI16SPCompressLen = 496;
2335 if (STI.is64Bit() && CanCompress(ADDI16SPCompressLen))
2336 return ADDI16SPCompressLen;
2337 if (CanCompress(RVCompressLen))
2338 return RVCompressLen;
2339 }
2340 return 2048 - StackAlign;
2341 }
2342 return 0;
2343}
2344
2347 std::vector<CalleeSavedInfo> &CSI) const {
2348 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2349 MachineFrameInfo &MFI = MF.getFrameInfo();
2350 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
2351
2352 // Preemptible Interrupts have two additional Callee-save Frame Indexes,
2353 // not tracked by `CSI`.
2354 if (RVFI->isSiFivePreemptibleInterrupt(MF)) {
2355 for (int I = 0; I < 2; ++I) {
2356 int FI = RVFI->getInterruptCSRFrameIndex(I);
2357 MFI.setIsCalleeSavedObjectIndex(FI, true);
2358 }
2359 }
2360
2361 // Early exit if no callee saved registers are modified!
2362 if (CSI.empty())
2363 return true;
2364
2365 if (RVFI->useQCIInterrupt(MF)) {
2366 RVFI->setQCIInterruptStackSize(QCIInterruptPushAmount);
2367 }
2368
2369 if (RVFI->isPushable(MF)) {
2370 // Determine how many GPRs we need to push and save it to RVFI.
2371 unsigned PushedRegNum = getNumPushPopRegs(CSI);
2372
2373 // `QC.C.MIENTER(.NEST)` will save `ra` and `s0`, so we should only push if
2374 // we want to push more than 2 registers. Otherwise, we should push if we
2375 // want to push more than 0 registers.
2376 unsigned OnlyPushIfMoreThan = RVFI->useQCIInterrupt(MF) ? 2 : 0;
2377 if (PushedRegNum > OnlyPushIfMoreThan) {
2378 RVFI->setRVPushRegs(PushedRegNum);
2379 RVFI->setRVPushStackSize(alignTo((STI.getXLen() / 8) * PushedRegNum, 16));
2380 }
2381 }
2382
2383 for (auto &CS : CSI) {
2384 MCRegister Reg = CS.getReg();
2385 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
2386 unsigned Size = RegInfo->getSpillSize(*RC);
2387
2388 if (RVFI->useQCIInterrupt(MF)) {
2389 const auto *FFI = llvm::find_if(FixedCSRFIQCIInterruptMap, [&](auto P) {
2390 return P.first == CS.getReg();
2391 });
2392 if (FFI != std::end(FixedCSRFIQCIInterruptMap)) {
2393 int64_t Offset = FFI->second * (int64_t)Size;
2394
2395 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, Offset);
2396 assert(FrameIdx < 0);
2397 CS.setFrameIdx(FrameIdx);
2398 continue;
2399 }
2400 }
2401
2402 if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) {
2403 const auto *FII = llvm::find_if(
2404 FixedCSRFIMap, [&](MCPhysReg P) { return P == CS.getReg(); });
2405 unsigned RegNum = std::distance(std::begin(FixedCSRFIMap), FII);
2406
2407 if (FII != std::end(FixedCSRFIMap)) {
2408 int64_t Offset;
2409 if (RVFI->getPushPopKind(MF) ==
2411 Offset = -int64_t(RVFI->getRVPushRegs() - RegNum) * Size;
2412 else
2413 Offset = -int64_t(RegNum + 1) * Size;
2414
2415 if (RVFI->useQCIInterrupt(MF))
2417
2418 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, Offset);
2419 assert(FrameIdx < 0);
2420 CS.setFrameIdx(FrameIdx);
2421 continue;
2422 }
2423 }
2424
2425 // For GPRPair registers, use 8-byte slots with required alignment by zilsd.
2426 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
2427 RISCV::GPRPairRegClass.contains(Reg)) {
2428 Align PairAlign = STI.getZilsdAlign();
2429 int FrameIdx = MFI.CreateStackObject(8, PairAlign, true);
2430 MFI.setIsCalleeSavedObjectIndex(FrameIdx, true);
2431 CS.setFrameIdx(FrameIdx);
2432 continue;
2433 }
2434
2435 // Not a fixed slot.
2436 Align Alignment = RegInfo->getSpillAlign(*RC);
2437 // We may not be able to satisfy the desired alignment specification of
2438 // the TargetRegisterClass if the stack alignment is smaller. Use the
2439 // min.
2440 Alignment = std::min(Alignment, getStackAlign());
2441 int FrameIdx = MFI.CreateStackObject(Size, Alignment, true);
2442 MFI.setIsCalleeSavedObjectIndex(FrameIdx, true);
2443 CS.setFrameIdx(FrameIdx);
2445 MFI.setStackID(FrameIdx, TargetStackID::ScalableVector);
2446 }
2447
2448 if (RVFI->useQCIInterrupt(MF)) {
2449 // Allocate a fixed object that covers the entire QCI stack allocation,
2450 // because there are gaps which are reserved for future use.
2451 MFI.CreateFixedSpillStackObject(
2452 QCIInterruptPushAmount, -static_cast<int64_t>(QCIInterruptPushAmount));
2453 }
2454
2455 if (RVFI->isPushable(MF)) {
2456 int64_t QCIOffset = RVFI->useQCIInterrupt(MF) ? QCIInterruptPushAmount : 0;
2457 // Allocate a fixed object that covers the full push.
2458 if (int64_t PushSize = RVFI->getRVPushStackSize())
2459 MFI.CreateFixedSpillStackObject(PushSize, -PushSize - QCIOffset);
2460 } else if (int LibCallRegs = getLibCallID(MF, CSI) + 1) {
2461 // If libcalls are used to spill and restore callee-saved registers, the
2462 // frame has two sections; the opaque section managed by the libcalls, and
2463 // the section managed by MachineFrameInfo which can also hold callee saved
2464 // registers in fixed stack slots, both of which have negative frame
2465 // indices. This gets even more complicated when incoming arguments are
2466 // passed via the stack, as these too have negative frame indices. An
2467 // example is detailed below:
2468 //
2469 // | incoming arg | <- FI[-3]
2470 // | libcallspill |
2471 // | calleespill | <- FI[-2]
2472 // | calleespill | <- FI[-1]
2473 // | this_frame | <- FI[0]
2474 //
2475 // For negative frame indices, the offset from the frame pointer will differ
2476 // depending on which of these groups the frame index applies to.
2477 // The following calculates the correct offset knowing the number of callee
2478 // saved registers spilt by the two methods.
2479 unsigned LibCallFrameSize =
2480 alignTo((STI.getXLen() / 8) * LibCallRegs, getStackAlign());
2481 RVFI->setLibCallStackSize(LibCallFrameSize);
2482 // Allocate a fixed object that covers the entire libcall area.
2483 MFI.CreateFixedSpillStackObject(LibCallFrameSize,
2484 -static_cast<int64_t>(LibCallFrameSize));
2485 }
2486
2487 return true;
2488}
2489
2493 if (CSI.empty())
2494 return true;
2495
2496 MachineFunction *MF = MBB.getParent();
2497 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2498 DebugLoc DL;
2499 if (MI != MBB.end() && !MI->isDebugInstr())
2500 DL = MI->getDebugLoc();
2501
2503 if (RVFI->useQCIInterrupt(*MF)) {
2504 // Emit QC.C.MIENTER(.NEST)
2505 BuildMI(
2506 MBB, MI, DL,
2507 TII.get(RVFI->getInterruptStackKind(*MF) ==
2509 ? RISCV::QC_C_MIENTER_NEST
2510 : RISCV::QC_C_MIENTER))
2512
2513 for (auto [Reg, _Offset] : FixedCSRFIQCIInterruptMap)
2514 MBB.addLiveIn(Reg);
2515 }
2516
2517 if (RVFI->isPushable(*MF)) {
2518 // Emit CM.PUSH with base StackAdj & evaluate Push stack
2519 unsigned PushedRegNum = RVFI->getRVPushRegs();
2520 if (PushedRegNum > 0) {
2521 // Use encoded number to represent registers to spill.
2522 unsigned Opcode = getPushOpcode(
2523 RVFI->getPushPopKind(*MF), hasFP(*MF) && !RVFI->useQCIInterrupt(*MF));
2524 unsigned RegEnc = RISCVZC::encodeRegListNumRegs(PushedRegNum);
2525 MachineInstrBuilder PushBuilder =
2526 BuildMI(MBB, MI, DL, TII.get(Opcode))
2528 PushBuilder.addImm(RegEnc);
2529 PushBuilder.addImm(0);
2530
2531 for (unsigned i = 0; i < PushedRegNum; i++)
2532 PushBuilder.addUse(FixedCSRFIMap[i], RegState::Implicit);
2533 }
2534 } else if (const char *SpillLibCall = getSpillLibCallName(*MF, CSI)) {
2535 // Add spill libcall via non-callee-saved register t0.
2536 MachineInstrBuilder NewMI =
2537 BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoCALLReg), RISCV::X5)
2538 .addExternalSymbol(SpillLibCall, RISCVII::MO_CALL)
2540 .addUse(RISCV::X2, RegState::Implicit)
2541 .addDef(RISCV::X2, RegState::ImplicitDefine)
2542 // The libcall may use t1 as a scratch register.
2544 // libgcc may emit an lui t2, 0 to set a landing pad label.
2546
2547 // Add registers spilled as implicit used.
2548 for (auto &CS : CSI)
2549 NewMI.addUse(CS.getReg(), RegState::Implicit);
2550 }
2551
2552 // Manually spill values not spilled by libcall & Push/Pop.
2553 const auto &UnmanagedCSI =
2554 getUnmanagedInterruptCSI(*MF, CSI, STI.preferAscendingLoadStore());
2555 const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, CSI);
2556
2557 auto storeRegsToStackSlots = [&](ArrayRef<CalleeSavedInfo> CSInfo) {
2558 for (auto &CS : CSInfo) {
2559 // Insert the spill to the stack frame.
2560 MCRegister Reg = CS.getReg();
2561 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
2562 TII.storeRegToStackSlot(MBB, MI, Reg, !MBB.isLiveIn(Reg),
2563 CS.getFrameIdx(), RC, Register(),
2565 }
2566 };
2567 storeRegsToStackSlots(UnmanagedCSI);
2568 storeRegsToStackSlots(RVVCSI);
2569
2570 return true;
2571}
2572
2573static unsigned getCalleeSavedRVVNumRegs(const Register &BaseReg) {
2574 return RISCV::VRRegClass.contains(BaseReg) ? 1
2575 : RISCV::VRM2RegClass.contains(BaseReg) ? 2
2576 : RISCV::VRM4RegClass.contains(BaseReg) ? 4
2577 : 8;
2578}
2579
2580void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI(
2583 const MachineFrameInfo &MFI = MF->getFrameInfo();
2584 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2585 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
2586
2587 const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, MFI.getCalleeSavedInfo());
2588 if (RVVCSI.empty())
2589 return;
2590
2591 uint64_t FixedSize = getStackSizeWithRVVPadding(*MF);
2592 if (!HasFP) {
2593 uint64_t ScalarLocalVarSize =
2594 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
2595 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
2596 FixedSize -= ScalarLocalVarSize;
2597 }
2598
2599 CFIInstBuilder CFIBuilder(MBB, MI, MachineInstr::FrameSetup);
2600 for (auto &CS : RVVCSI) {
2601 // Insert the spill to the stack frame.
2602 int FI = CS.getFrameIdx();
2603 MCRegister BaseReg = getRVVBaseRegister(TRI, CS.getReg());
2604 unsigned NumRegs = getCalleeSavedRVVNumRegs(CS.getReg());
2605 for (unsigned i = 0; i < NumRegs; ++i) {
2606 CFIBuilder.insertCFIInst(createDefCFAOffset(
2607 TRI, BaseReg + i,
2608 StackOffset::get(-FixedSize, MFI.getObjectOffset(FI) / 8 + i)));
2609 }
2610 }
2611}
2612
2613void RISCVFrameLowering::emitCalleeSavedRVVEpilogCFI(
2616 const MachineFrameInfo &MFI = MF->getFrameInfo();
2617 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
2618
2619 CFIInstBuilder CFIHelper(MBB, MI, MachineInstr::FrameDestroy);
2620 const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, MFI.getCalleeSavedInfo());
2621 for (auto &CS : RVVCSI) {
2622 MCRegister BaseReg = getRVVBaseRegister(TRI, CS.getReg());
2623 unsigned NumRegs = getCalleeSavedRVVNumRegs(CS.getReg());
2624 for (unsigned i = 0; i < NumRegs; ++i)
2625 CFIHelper.buildRestore(BaseReg + i);
2626 }
2627}
2628
2632 if (CSI.empty())
2633 return true;
2634
2635 MachineFunction *MF = MBB.getParent();
2636 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2637 DebugLoc DL;
2638 if (MI != MBB.end() && !MI->isDebugInstr())
2639 DL = MI->getDebugLoc();
2640
2641 // Manually restore values not restored by libcall & Push/Pop.
2642 // Reverse the restore order in epilog. In addition, the return
2643 // address will be restored first in the epilogue. It increases
2644 // the opportunity to avoid the load-to-use data hazard between
2645 // loading RA and return by RA. loadRegFromStackSlot can insert
2646 // multiple instructions.
2647 const auto &UnmanagedCSI =
2648 getUnmanagedInterruptCSI(*MF, CSI, STI.preferAscendingLoadStore());
2649 const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, CSI);
2650
2651 auto loadRegFromStackSlot = [&](ArrayRef<CalleeSavedInfo> CSInfo) {
2652 for (auto &CS : CSInfo) {
2653 MCRegister Reg = CS.getReg();
2654 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
2655 TII.loadRegFromStackSlot(MBB, MI, Reg, CS.getFrameIdx(), RC, Register(),
2656 RISCV::NoSubRegister,
2658 assert(MI != MBB.begin() &&
2659 "loadRegFromStackSlot didn't insert any code!");
2660 }
2661 };
2662 loadRegFromStackSlot(RVVCSI);
2663 loadRegFromStackSlot(UnmanagedCSI);
2664
2666 if (RVFI->useQCIInterrupt(*MF)) {
2667 // Don't emit anything here because restoration is handled by
2668 // QC.C.MILEAVERET which we already inserted to return.
2669 assert(MI->getOpcode() == RISCV::QC_C_MILEAVERET &&
2670 "Unexpected QCI Interrupt Return Instruction");
2671 }
2672
2673 if (RVFI->isPushable(*MF)) {
2674 unsigned PushedRegNum = RVFI->getRVPushRegs();
2675 if (PushedRegNum > 0) {
2676 unsigned Opcode = getPopOpcode(RVFI->getPushPopKind(*MF));
2677 unsigned RegEnc = RISCVZC::encodeRegListNumRegs(PushedRegNum);
2678 MachineInstrBuilder PopBuilder =
2679 BuildMI(MBB, MI, DL, TII.get(Opcode))
2681 // Use encoded number to represent registers to restore.
2682 PopBuilder.addImm(RegEnc);
2683 PopBuilder.addImm(0);
2684
2685 for (unsigned i = 0; i < RVFI->getRVPushRegs(); i++)
2687 }
2688 } else if (const char *RestoreLibCall = getRestoreLibCallName(*MF, CSI)) {
2689 assert(RVFI->useSaveRestoreLibCalls(*MF) &&
2690 "Expected restore with libcall to use save restore libcalls");
2691
2692 // Restore is not compatible with shadow stacks, so do the restore manually.
2693 // This ensures we still get the code size saving of `__riscv_save_<N>`.
2694 if (RVFI->hasShadowStack(*MF)) {
2695 loadRegFromStackSlot(getPushOrLibCallsSavedInfo(*MF, CSI));
2696 return true;
2697 }
2698
2699 // Add restore libcall via tail call. Use the software guarded variant when
2700 // cf-protection-branch is enabled because the restore function does not
2701 // contain an lpad.
2702 unsigned TailOpc =
2703 RVFI->hasCFProtectionBranch() ? RISCV::PseudoTAILX7 : RISCV::PseudoTAIL;
2704 MachineInstrBuilder NewMI =
2705 BuildMI(MBB, MI, DL, TII.get(TailOpc))
2706 .addExternalSymbol(RestoreLibCall, RISCVII::MO_CALL)
2708 .addDef(RISCV::X2, RegState::ImplicitDefine);
2709
2710 // Add registers restored as implicit defined.
2711 for (auto &CS : CSI)
2712 NewMI.addDef(CS.getReg(), RegState::ImplicitDefine);
2713
2714 // Remove trailing returns, since the terminator is now a tail call to the
2715 // restore function.
2716 if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
2717 NewMI.getInstr()->copyImplicitOps(*MF, *MI);
2718 MI->eraseFromParent();
2719 }
2720 }
2721 return true;
2722}
2723
2725 // Keep the conventional code flow when not optimizing.
2726 if (MF.getFunction().hasOptNone())
2727 return false;
2728
2729 // QCI and SiFive CLIC interrupt entry sequences must precede all handler
2730 // code.
2731 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
2732 if (RVFI->useQCIInterrupt(MF) || RVFI->useSiFiveInterrupt(MF))
2733 return false;
2734
2735 return true;
2736}
2737
2739 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2740 const MachineFunction *MF = MBB.getParent();
2741 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2742
2743 // Make sure VTYPE and VL are not live-in since we will use vsetvli in the
2744 // prologue to get the VLEN, and that will clobber these registers.
2745 //
2746 // We may do also check the stack contains objects with scalable vector type,
2747 // but this will require iterating over all the stack objects, but this may
2748 // not worth since the situation is rare, we could do further check in future
2749 // if we find it is necessary.
2750 if (STI.preferVsetvliOverReadVLENB() &&
2751 (MBB.isLiveIn(RISCV::VTYPE) || MBB.isLiveIn(RISCV::VL)))
2752 return false;
2753
2754 if (!RVFI->useSaveRestoreLibCalls(*MF))
2755 return true;
2756
2757 // Inserting a call to a __riscv_save libcall requires the use of the register
2758 // t0 (X5) to hold the return address. Therefore if this register is already
2759 // used we can't insert the call.
2760
2761 RegScavenger RS;
2762 RS.enterBasicBlock(*TmpMBB);
2763 return !RS.isRegUsed(RISCV::X5);
2764}
2765
2767 const MachineFunction *MF = MBB.getParent();
2768 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2769 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2770
2771 // If we have a shadow stack, we don't use `__riscv_restore` libcalls.
2772 if (RVFI->hasShadowStack(*MF))
2773 return true;
2774
2775 if (!RVFI->useSaveRestoreLibCalls(*MF))
2776 return true;
2777
2778 // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
2779 // This means if we still need to continue executing code within this function
2780 // the restore cannot take place in this basic block.
2781
2782 if (MBB.succ_size() > 1)
2783 return false;
2784
2785 MachineBasicBlock *SuccMBB =
2786 MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
2787
2788 // Doing a tail call should be safe if there are no successors, because either
2789 // we have a returning block or the end of the block is unreachable, so the
2790 // restore will be eliminated regardless.
2791 if (!SuccMBB)
2792 return true;
2793
2794 // The successor can only contain a return and debug instructions, since we
2795 // would effectively replace it with our own tail return at the end of this
2796 // block. The debug instructions would not execute on the tail-return path.
2797 return SuccMBB->isReturnBlock() &&
2798 llvm::count_if(SuccMBB->instrs(), [](const MachineInstr &MI) {
2799 return !MI.isDebugInstr();
2800 }) == 1;
2801}
2802
2806
2810
2811// Synthesize the probe loop.
2813 Register TargetReg, Register ScratchReg,
2814 bool IsRVV) {
2815 assert(TargetReg != RISCV::X2 && "New top of stack cannot already be in SP");
2816 assert(ScratchReg != RISCV::X2 && "Scratch register cannot be SP");
2817 assert(TargetReg != ScratchReg && "Target and scratch must be different");
2818
2819 MachineBasicBlock &MBB = *MBBI->getParent();
2820 MachineFunction &MF = *MBB.getParent();
2821
2823 const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
2824 bool IsRV64 = Subtarget.is64Bit();
2825 Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
2826 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
2827 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
2828
2829 MachineFunction::iterator MBBInsertPoint = std::next(MBB.getIterator());
2830 MachineBasicBlock *LoopTestMBB =
2831 MF.CreateMachineBasicBlock(MBB.getBasicBlock());
2832 MF.insert(MBBInsertPoint, LoopTestMBB);
2833 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(MBB.getBasicBlock());
2834 MF.insert(MBBInsertPoint, ExitMBB);
2836
2837 // ScratchReg = ProbeSize
2838 TII->movImm(MBB, MBBI, DL, ScratchReg, ProbeSize, Flags);
2839
2840 // LoopTest:
2841 // SUB SP, SP, ProbeSize
2842 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::SUB), SPReg)
2843 .addReg(SPReg)
2844 .addReg(ScratchReg)
2845 .setMIFlags(Flags);
2846
2847 // s[d|w] zero, 0(sp)
2848 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL,
2849 TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
2850 .addReg(RISCV::X0)
2851 .addReg(SPReg)
2852 .addImm(0)
2853 .setMIFlags(Flags);
2854
2855 if (IsRVV) {
2856 // SUB TargetReg, TargetReg, ProbeSize
2857 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::SUB),
2858 TargetReg)
2859 .addReg(TargetReg)
2860 .addReg(ScratchReg)
2861 .setMIFlags(Flags);
2862
2863 // BGE TargetReg, ProbeSize, LoopTest
2864 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::BGE))
2865 .addReg(TargetReg)
2866 .addReg(ScratchReg)
2867 .addMBB(LoopTestMBB)
2868 .setMIFlags(Flags);
2869
2870 } else {
2871 // BNE SP, TargetReg, LoopTest
2872 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::BNE))
2873 .addReg(SPReg)
2874 .addReg(TargetReg)
2875 .addMBB(LoopTestMBB)
2876 .setMIFlags(Flags);
2877 }
2878
2879 ExitMBB->splice(ExitMBB->end(), &MBB, std::next(MBBI), MBB.end());
2881
2882 LoopTestMBB->addSuccessor(ExitMBB);
2883 LoopTestMBB->addSuccessor(LoopTestMBB);
2884 MBB.addSuccessor(LoopTestMBB);
2885 // Update liveins.
2886 fullyRecomputeLiveIns({ExitMBB, LoopTestMBB});
2887}
2888
2889void RISCVFrameLowering::inlineStackProbe(MachineFunction &MF,
2890 MachineBasicBlock &MBB) const {
2891 // Get the instructions that need to be replaced. We emit at most two of
2892 // these. Remember them in order to avoid complications coming from the need
2893 // to traverse the block while potentially creating more blocks.
2894 SmallVector<MachineInstr *, 4> ToReplace;
2895 for (MachineInstr &MI : MBB) {
2896 unsigned Opc = MI.getOpcode();
2897 if (Opc == RISCV::PROBED_STACKALLOC ||
2898 Opc == RISCV::PROBED_STACKALLOC_RVV) {
2899 ToReplace.push_back(&MI);
2900 }
2901 }
2902
2903 for (MachineInstr *MI : ToReplace) {
2904 if (MI->getOpcode() == RISCV::PROBED_STACKALLOC ||
2905 MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV) {
2908 Register TargetReg = MI->getOperand(0).getReg();
2909
2910 Register ScratchReg =
2911 findScratchNonCalleeSaveRegister(&MBB, RISCV::X7, TargetReg);
2912
2913 assert(ScratchReg.isValid() &&
2914 "No available scratch register for stack probe loop");
2915
2916 emitStackProbeInline(MBBI, DL, TargetReg, ScratchReg,
2917 (MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV));
2919 }
2920 }
2921}
2922
2924 return 0;
2925}
2926
2929 return RISCV::X2;
2930}
2931
2932// On 64-bit systems the fixed stack can hold INT64_MAX bytes, since
2933// stack-offset calculation is done in 2s-complement.
2934// NOTE: In theory a register can hold any 64-bit number, so this constraint
2935// might be relaxed to UINT64_MAX in the future, if anyone actually needs
2936// that.
2938 return STI.is64Bit() ? INT64_MAX : UINT32_MAX;
2939}
static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs, const MachineBasicBlock &MBB)
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo & TII
uint64_t uint64_t
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
This file contains constants used for implementing Dwarf debug support.
static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL)
static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL)
IRTranslator LLVM IR MI
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
static uint64_t estimateFunctionSizeInBytes(const LoongArchInstrInfo *TII, const MachineFunction &MF)
static void emitStackProbeInline(MachineBasicBlock::iterator MBBI, DebugLoc DL, Register TargetReg)
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define P(N)
static constexpr uint64_t QCIInterruptPushAmount
static void emitSiFiveCLICStackSwap(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MachineInstr::MIFlag FrameFlag)
static unsigned getPushOpcode(RISCVMachineFunctionInfo::PushPopKind Kind, bool UpdateFP)
static void emitSiFiveCLICPreemptibleSaves(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL)
static MCRegister getRVVBaseRegister(const RISCVRegisterInfo &TRI, const Register &Reg)
static void createSiFivePreemptibleInterruptFrameEntries(MachineFunction &MF, RISCVMachineFunctionInfo &RVFI)
static constexpr MCPhysReg FPReg
static const char * getRestoreLibCallName(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static bool needsDwarfCFI(const MachineFunction &MF)
Returns true if DWARF CFI instructions ("frame moves") should be emitted.
static constexpr MCPhysReg SPReg
static const char * getSpillLibCallName(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static bool hasRVVFrameObject(const MachineFunction &MF)
static SmallVector< CalleeSavedInfo, 8 > getUnmanagedInterruptCSI(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI, bool ReverseOrder=false)
static void appendScalableVectorExpression(const TargetRegisterInfo &TRI, SmallVectorImpl< char > &Expr, StackOffset Offset, llvm::raw_string_ostream &Comment)
static SmallVector< CalleeSavedInfo, 8 > getQCISavedInfo(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static SmallVector< CalleeSavedInfo, 8 > getRVVCalleeSavedInfo(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static SmallVector< CalleeSavedInfo, 8 > getUnmanagedCSI(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI, bool ReverseOrder=false)
static void emitSiFiveCLICPreemptibleRestores(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, CFIInstBuilder &CFIBuilder, const DebugLoc &DL)
static bool isPop(unsigned Opcode)
static unsigned getCalleeSavedRVVNumRegs(const Register &BaseReg)
static MCCFIInstruction createDefCFAOffset(const TargetRegisterInfo &TRI, Register Reg, StackOffset Offset)
static unsigned getPopOpcode(RISCVMachineFunctionInfo::PushPopKind Kind)
static SmallVector< CalleeSavedInfo, 8 > getPushOrLibCallsSavedInfo(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static int getLibCallID(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static const std::pair< MCPhysReg, int8_t > FixedCSRFIQCIInterruptMap[]
static bool isPush(unsigned Opcode)
static constexpr MCPhysReg RAReg
static MCRegister getLargestFPRegisterOrZero(const RISCVSubtarget &STI, const TargetRegisterInfo &TRI, MCRegister Reg)
static const MCPhysReg FixedCSRFIMap[]
static int getSiFiveCLICScratchFrameIndex(const MachineFunction &MF)
static unsigned getNumPushPopRegs(const std::vector< CalleeSavedInfo > &CSI)
static MCRegister getPhysicalGPR(const TargetRegisterInfo &TRI, MCRegister Reg)
static unsigned getScavSlotsNumForRVV(MachineFunction &MF)
MachineRegisterInfo & MRI
This file declares the machine register scavenger class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
LLVM_ABI StringRef getValueAsString() const
Return the attribute's value as a string.
bool test(unsigned Idx) const
Returns true if bit Idx is set.
Definition BitVector.h:483
BitVector & reset()
Reset all bits in the bitvector.
Definition BitVector.h:410
BitVector & set()
Set all bits in the bitvector.
Definition BitVector.h:367
iterator_range< const_set_bits_iterator > set_bits() const
Definition BitVector.h:160
Helper class for creating CFI instructions and inserting them into MIR.
void buildEscape(StringRef Bytes, StringRef Comment="") const
void buildDefCFAOffset(int64_t Offset, MCSymbol *Label=nullptr) const
void buildRestore(MCRegister Reg) const
void buildDefCFARegister(MCRegister Reg) const
void buildOffset(MCRegister Reg, int64_t Offset) const
void insertCFIInst(const MCCFIInstruction &CFIInst) const
void buildDefCFA(MCRegister Reg, int64_t Offset) const
void setInsertPoint(MachineBasicBlock::iterator IP)
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
MCRegister getReg() const
A debug info location.
Definition DebugLoc.h:126
Diagnostic information for unsupported feature in backend.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:766
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
bool hasOptNone() const
Do not optimize this function (-O0).
Definition Function.h:686
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
Definition MCDwarf.h:756
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
constexpr unsigned id() const
Definition MCRegister.h:82
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
bool isReturnBlock() const
Convenience function that returns true if the block ends in a return instruction.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
LLVM_ABI void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
bool adjustsStack() const
Return true if this function adjusts the stack – e.g., when calling another function.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
LLVM_ABI void ensureMaxAlignment(Align Alignment)
Make sure the function's frame is at least Align bytes aligned.
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Align getMaxAlign() const
Return alignment of this function's frame.
void setObjectOffset(int ObjectIdx, int64_t SPOffset)
Set the stack frame offset of the specified object.
uint64_t getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
int64_t getOffsetAdjustment() const
Return the correction for frame offsets.
LLVM_ABI uint64_t estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isMaxCallFrameSizeComputed() const
LLVM_ABI int CreateSpillStackObject(uint64_t Size, Align Alignment, TargetStackID::Value StackID=TargetStackID::Default)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
uint8_t getStackID(int ObjectIdx) const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
void setStackSize(uint64_t Size)
Set the size of the stack.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
bool needsFrameMoves() const
True if this function needs frame moves for debug or exceptions.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
bool disableFramePointerElim() const
Returns true if frame pointer elimination should be disabled for this function.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
LLVM_ABI void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI)
Copy implicit register operands from specified instruction to this instruction.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool isReserved(MCRegister PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
LLVM_ABI const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
uint64_t getFirstSPAdjustAmount(const MachineFunction &MF) const
bool enableShrinkWrapping(const MachineFunction &MF) const override
Returns true if the target will correctly handle shrink wrapping.
uint64_t getStackThreshold() const override
getStackThreshold - Return the maximum stack size
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
bool hasBP(const MachineFunction &MF) const
void allocateStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineFunction &MF, uint64_t Offset, uint64_t RealStackSize, bool EmitCFI, bool NeedProbe, uint64_t ProbeSize, bool DynAllocation, MachineInstr::MIFlag Flag) const
bool canUseAsEpilogue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a epilogue for the target.
bool hasFPImpl(const MachineFunction &MF) const override
Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB, Register PreferredReg, Register DontUseReg=Register()) const
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
Register getInitialCFARegister(const MachineFunction &MF) const override
Return initial CFA register value i.e.
const RISCVSubtarget & STI
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - This method should return the base register and offset used to reference a f...
bool isSupportedStackID(TargetStackID::Value ID) const override
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
TargetStackID::Value getStackIDForScalableVectors() const override
Returns the StackID that scalable vectors should be associated with.
int getInitialCFAOffset(const MachineFunction &MF) const override
Return initial CFA offset value i.e.
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
RISCVFrameLowering(const RISCVSubtarget &STI)
uint64_t getStackSizeWithRVVPadding(const MachineFunction &MF) const
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
bool isPushable(const MachineFunction &MF) const
InterruptStackKind getInterruptStackKind(const MachineFunction &MF) const
bool useSiFiveInterrupt(const MachineFunction &MF) const
bool isSiFivePreemptibleInterrupt(const MachineFunction &MF) const
PushPopKind getPushPopKind(const MachineFunction &MF) const
bool hasShadowStack(const MachineFunction &MF) const
bool useSaveRestoreLibCalls(const MachineFunction &MF) const
bool useQCIInterrupt(const MachineFunction &MF) const
TargetRegisterClass const * getLargestFPRegClass() const
bool hasVInstructions() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
Represents a location in source code.
Definition SMLoc.h:22
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition SmallSet.h:134
bool contains(const T &V) const
Check if the SmallSet contains the given element.
Definition SmallSet.h:229
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:28
void append(StringRef RHS)
Append from a StringRef.
Definition SmallString.h:70
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
Definition TypeSize.h:31
int64_t getFixed() const
Returns the fixed component of the stack.
Definition TypeSize.h:47
int64_t getScalable() const
Returns the scalable component of the stack.
Definition TypeSize.h:50
static StackOffset get(int64_t Fixed, int64_t Scalable)
Definition TypeSize.h:42
static StackOffset getScalable(int64_t Scalable)
Definition TypeSize.h:41
static StackOffset getFixed(int64_t Fixed)
Definition TypeSize.h:40
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
unsigned getStackAlignment() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
TargetFrameLowering(StackDirection D, Align StackAl, int LAO, Align TransAl=Align(1), bool StackReal=true)
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
int alignSPAdjust(int SPAdj) const
alignSPAdjust - This method aligns the stack adjustment to the correct alignment.
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool hasStackRealignment(const MachineFunction &MF) const
True if stack realignment is required and still possible.
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
self_iterator getIterator()
Definition ilist_node.h:123
A raw_ostream that writes to an std::string.
#define INT64_MAX
Definition DataTypes.h:71
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
Definition CallingConv.h:50
MCRegister getBPReg()
MCRegister getSCSPReg()
static VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
LLVM_ABI unsigned encodeVTYPE(VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic, bool AltFmt=false)
static unsigned encodeRegListNumRegs(unsigned NumRegs)
static constexpr unsigned RVVBitsPerBlock
bool isRVVSpill(const MachineInstr &MI)
static constexpr unsigned RVVBytesPerBlock
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
@ Offset
Definition DWP.cpp:577
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition STLExtras.h:846
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Dead
Unused definition.
@ Kill
The last use of a register.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
Definition Error.cpp:173
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
auto make_first_range(ContainerTy &&c)
Given a container of pairs, return a range over the first elements.
Definition STLExtras.h:1415
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
uint64_t offsetToAlignment(uint64_t Value, Align Alignment)
Returns the offset to the next integer (mod 2**64) that is greater than or equal to Value and is a mu...
Definition Alignment.h:191
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:2035
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
void appendLEB128(SmallVectorImpl< U > &Buffer, T Value)
Definition LEB128.h:280
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:202
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
static bool isRVVRegClass(const TargetRegisterClass *RC)
void adjustReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator II, const DebugLoc &DL, Register DestReg, Register SrcReg, StackOffset Offset, MachineInstr::MIFlag Flag, MaybeAlign RequiredAlign) const