LLVM  12.0.0git
SILowerSGPRSpills.cpp
Go to the documentation of this file.
1 //===-- SILowerSGPRSPills.cpp ---------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Handle SGPR spills. This pass takes the place of PrologEpilogInserter for all
10 // SGPR spills, so must insert CSR SGPR spills as well as expand them.
11 //
12 // This pass must never create new SGPR virtual registers.
13 //
14 // FIXME: Must stop RegScavenger spills in later passes.
15 //
16 //===----------------------------------------------------------------------===//
17 
18 #include "AMDGPU.h"
19 #include "AMDGPUSubtarget.h"
20 #include "SIInstrInfo.h"
21 #include "SIMachineFunctionInfo.h"
30 #include "llvm/InitializePasses.h"
32 
33 using namespace llvm;
34 
35 #define DEBUG_TYPE "si-lower-sgpr-spills"
36 
38 
39 namespace {
40 
41 static cl::opt<bool> EnableSpillVGPRToAGPR(
42  "amdgpu-spill-vgpr-to-agpr",
43  cl::desc("Enable spilling VGPRs to AGPRs"),
45  cl::init(true));
46 
47 class SILowerSGPRSpills : public MachineFunctionPass {
48 private:
49  const SIRegisterInfo *TRI = nullptr;
50  const SIInstrInfo *TII = nullptr;
51  VirtRegMap *VRM = nullptr;
52  LiveIntervals *LIS = nullptr;
53 
54  // Save and Restore blocks of the current function. Typically there is a
55  // single save block, unless Windows EH funclets are involved.
56  MBBVector SaveBlocks;
57  MBBVector RestoreBlocks;
58 
59 public:
60  static char ID;
61 
62  SILowerSGPRSpills() : MachineFunctionPass(ID) {}
63 
64  void calculateSaveRestoreBlocks(MachineFunction &MF);
65  bool spillCalleeSavedRegs(MachineFunction &MF);
66 
67  bool runOnMachineFunction(MachineFunction &MF) override;
68 
69  void getAnalysisUsage(AnalysisUsage &AU) const override {
70  AU.setPreservesAll();
72  }
73 };
74 
75 } // end anonymous namespace
76 
77 char SILowerSGPRSpills::ID = 0;
78 
79 INITIALIZE_PASS_BEGIN(SILowerSGPRSpills, DEBUG_TYPE,
80  "SI lower SGPR spill instructions", false, false)
82 INITIALIZE_PASS_END(SILowerSGPRSpills, DEBUG_TYPE,
83  "SI lower SGPR spill instructions", false, false)
84 
85 char &llvm::SILowerSGPRSpillsID = SILowerSGPRSpills::ID;
86 
87 /// Insert restore code for the callee-saved registers used in the function.
88 static void insertCSRSaves(MachineBasicBlock &SaveBlock,
90  LiveIntervals *LIS) {
91  MachineFunction &MF = *SaveBlock.getParent();
95 
96  MachineBasicBlock::iterator I = SaveBlock.begin();
97  if (!TFI->spillCalleeSavedRegisters(SaveBlock, I, CSI, TRI)) {
98  for (const CalleeSavedInfo &CS : CSI) {
99  // Insert the spill to the stack frame.
100  unsigned Reg = CS.getReg();
101 
102  MachineInstrSpan MIS(I, &SaveBlock);
103  const TargetRegisterClass *RC =
104  TRI->getMinimalPhysRegClass(Reg, MVT::i32);
105 
106  TII.storeRegToStackSlot(SaveBlock, I, Reg, true, CS.getFrameIdx(), RC,
107  TRI);
108 
109  if (LIS) {
110  assert(std::distance(MIS.begin(), I) == 1);
111  MachineInstr &Inst = *std::prev(I);
112 
113  LIS->InsertMachineInstrInMaps(Inst);
114  LIS->removeAllRegUnitsForPhysReg(Reg);
115  }
116  }
117  }
118 }
119 
120 /// Insert restore code for the callee-saved registers used in the function.
121 static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
123  LiveIntervals *LIS) {
124  MachineFunction &MF = *RestoreBlock.getParent();
125  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
127  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
128 
129  // Restore all registers immediately before the return and any
130  // terminators that precede it.
132 
133  // FIXME: Just emit the readlane/writelane directly
134  if (!TFI->restoreCalleeSavedRegisters(RestoreBlock, I, CSI, TRI)) {
135  for (const CalleeSavedInfo &CI : reverse(CSI)) {
136  unsigned Reg = CI.getReg();
137  const TargetRegisterClass *RC =
138  TRI->getMinimalPhysRegClass(Reg, MVT::i32);
139 
140  TII.loadRegFromStackSlot(RestoreBlock, I, Reg, CI.getFrameIdx(), RC, TRI);
141  assert(I != RestoreBlock.begin() &&
142  "loadRegFromStackSlot didn't insert any code!");
143  // Insert in reverse order. loadRegFromStackSlot can insert
144  // multiple instructions.
145 
146  if (LIS) {
147  MachineInstr &Inst = *std::prev(I);
148  LIS->InsertMachineInstrInMaps(Inst);
149  LIS->removeAllRegUnitsForPhysReg(Reg);
150  }
151  }
152  }
153 }
154 
155 /// Compute the sets of entry and return blocks for saving and restoring
156 /// callee-saved registers, and placing prolog and epilog code.
157 void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) {
158  const MachineFrameInfo &MFI = MF.getFrameInfo();
159 
160  // Even when we do not change any CSR, we still want to insert the
161  // prologue and epilogue of the function.
162  // So set the save points for those.
163 
164  // Use the points found by shrink-wrapping, if any.
165  if (MFI.getSavePoint()) {
166  SaveBlocks.push_back(MFI.getSavePoint());
167  assert(MFI.getRestorePoint() && "Both restore and save must be set");
168  MachineBasicBlock *RestoreBlock = MFI.getRestorePoint();
169  // If RestoreBlock does not have any successor and is not a return block
170  // then the end point is unreachable and we do not need to insert any
171  // epilogue.
172  if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
173  RestoreBlocks.push_back(RestoreBlock);
174  return;
175  }
176 
177  // Save refs to entry and return blocks.
178  SaveBlocks.push_back(&MF.front());
179  for (MachineBasicBlock &MBB : MF) {
180  if (MBB.isEHFuncletEntry())
181  SaveBlocks.push_back(&MBB);
182  if (MBB.isReturnBlock())
183  RestoreBlocks.push_back(&MBB);
184  }
185 }
186 
187 bool SILowerSGPRSpills::spillCalleeSavedRegs(MachineFunction &MF) {
189  const Function &F = MF.getFunction();
190  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
191  const SIFrameLowering *TFI = ST.getFrameLowering();
192  MachineFrameInfo &MFI = MF.getFrameInfo();
193  RegScavenger *RS = nullptr;
194 
195  // Determine which of the registers in the callee save list should be saved.
196  BitVector SavedRegs;
197  TFI->determineCalleeSavesSGPR(MF, SavedRegs, RS);
198 
199  // Add the code to save and restore the callee saved registers.
200  if (!F.hasFnAttribute(Attribute::Naked)) {
201  // FIXME: This is a lie. The CalleeSavedInfo is incomplete, but this is
202  // necessary for verifier liveness checks.
203  MFI.setCalleeSavedInfoValid(true);
204 
205  std::vector<CalleeSavedInfo> CSI;
206  const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
207 
208  for (unsigned I = 0; CSRegs[I]; ++I) {
209  unsigned Reg = CSRegs[I];
210  if (SavedRegs.test(Reg)) {
211  const TargetRegisterClass *RC =
212  TRI->getMinimalPhysRegClass(Reg, MVT::i32);
213  int JunkFI = MFI.CreateStackObject(TRI->getSpillSize(*RC),
214  TRI->getSpillAlign(*RC), true);
215 
216  CSI.push_back(CalleeSavedInfo(Reg, JunkFI));
217  }
218  }
219 
220  if (!CSI.empty()) {
221  for (MachineBasicBlock *SaveBlock : SaveBlocks)
222  insertCSRSaves(*SaveBlock, CSI, LIS);
223 
224  for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
225  insertCSRRestores(*RestoreBlock, CSI, LIS);
226  return true;
227  }
228  }
229 
230  return false;
231 }
232 
233 // Find lowest available VGPR and use it as VGPR reserved for SGPR spills.
235  const GCNSubtarget &ST) {
237  MachineFrameInfo &FrameInfo = MF.getFrameInfo();
239  Register LowestAvailableVGPR, ReservedVGPR;
240  ArrayRef<MCPhysReg> AllVGPR32s = ST.getRegisterInfo()->getAllVGPR32(MF);
241  for (MCPhysReg Reg : AllVGPR32s) {
242  if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) {
243  LowestAvailableVGPR = Reg;
244  break;
245  }
246  }
247 
248  if (!LowestAvailableVGPR)
249  return false;
250 
251  ReservedVGPR = FuncInfo->VGPRReservedForSGPRSpill;
252  const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
253  int i = 0;
254 
255  for (MachineBasicBlock &MBB : MF) {
256  for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
257  if (Reg.VGPR == ReservedVGPR) {
258  MBB.removeLiveIn(ReservedVGPR);
259  MBB.addLiveIn(LowestAvailableVGPR);
260  Optional<int> FI;
261  if (FuncInfo->isCalleeSavedReg(CSRegs, LowestAvailableVGPR))
262  FI = FrameInfo.CreateSpillStackObject(4, Align(4));
263 
264  FuncInfo->setSGPRSpillVGPRs(LowestAvailableVGPR, FI, i);
265  }
266  ++i;
267  }
269  }
270 
271  return true;
272 }
273 
274 bool SILowerSGPRSpills::runOnMachineFunction(MachineFunction &MF) {
275  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
276  TII = ST.getInstrInfo();
277  TRI = &TII->getRegisterInfo();
278 
279  VRM = getAnalysisIfAvailable<VirtRegMap>();
280 
281  assert(SaveBlocks.empty() && RestoreBlocks.empty());
282 
283  // First, expose any CSR SGPR spills. This is mostly the same as what PEI
284  // does, but somewhat simpler.
285  calculateSaveRestoreBlocks(MF);
286  bool HasCSRs = spillCalleeSavedRegs(MF);
287 
288  MachineFrameInfo &MFI = MF.getFrameInfo();
289  if (!MFI.hasStackObjects() && !HasCSRs) {
290  SaveBlocks.clear();
291  RestoreBlocks.clear();
292  return false;
293  }
294 
297  const bool SpillVGPRToAGPR = ST.hasMAIInsts() && FuncInfo->hasSpilledVGPRs()
298  && EnableSpillVGPRToAGPR;
299 
300  bool MadeChange = false;
301 
302  const bool SpillToAGPR = EnableSpillVGPRToAGPR && ST.hasMAIInsts();
303 
304  // TODO: CSR VGPRs will never be spilled to AGPRs. These can probably be
305  // handled as SpilledToReg in regular PrologEpilogInserter.
306  if ((TRI->spillSGPRToVGPR() && (HasCSRs || FuncInfo->hasSpilledSGPRs())) ||
307  SpillVGPRToAGPR) {
308  // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs
309  // are spilled to VGPRs, in which case we can eliminate the stack usage.
310  //
311  // This operates under the assumption that only other SGPR spills are users
312  // of the frame index.
313 
314  lowerShiftReservedVGPR(MF, ST);
315 
316  for (MachineBasicBlock &MBB : MF) {
318  for (auto I = MBB.begin(), E = MBB.end(); I != E; I = Next) {
319  MachineInstr &MI = *I;
320  Next = std::next(I);
321 
322  if (SpillToAGPR && TII->isVGPRSpill(MI)) {
323  // Try to eliminate stack used by VGPR spills before frame
324  // finalization.
325  unsigned FIOp = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
326  AMDGPU::OpName::vaddr);
327  int FI = MI.getOperand(FIOp).getIndex();
328  Register VReg =
329  TII->getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
330  if (FuncInfo->allocateVGPRSpillToAGPR(MF, FI,
331  TRI->isAGPR(MRI, VReg))) {
332  TRI->eliminateFrameIndex(MI, 0, FIOp, nullptr);
333  continue;
334  }
335  }
336 
337  if (!TII->isSGPRSpill(MI))
338  continue;
339 
340  int FI = TII->getNamedOperand(MI, AMDGPU::OpName::addr)->getIndex();
342  if (FuncInfo->allocateSGPRSpillToVGPR(MF, FI)) {
343  bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(MI, FI, nullptr);
344  (void)Spilled;
345  assert(Spilled && "failed to spill SGPR to VGPR when allocated");
346  }
347  }
348  }
349 
350  for (MachineBasicBlock &MBB : MF) {
351  for (auto SSpill : FuncInfo->getSGPRSpillVGPRs())
352  MBB.addLiveIn(SSpill.VGPR);
353 
354  for (MCPhysReg Reg : FuncInfo->getVGPRSpillAGPRs())
355  MBB.addLiveIn(Reg);
356 
357  for (MCPhysReg Reg : FuncInfo->getAGPRSpillVGPRs())
358  MBB.addLiveIn(Reg);
359 
361  }
362 
363  MadeChange = true;
364  } else if (FuncInfo->VGPRReservedForSGPRSpill) {
365  FuncInfo->removeVGPRForSGPRSpill(FuncInfo->VGPRReservedForSGPRSpill, MF);
366  }
367 
368  SaveBlocks.clear();
369  RestoreBlocks.clear();
370 
371  return MadeChange;
372 }
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR)
Reserve AGPRs or VGPRs to support spilling for FrameIndex FI.
AMDGPU specific subclass of TargetSubtarget.
virtual void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const
Store the specified register of the given register class to the specified stack frame index...
void setCalleeSavedInfoValid(bool v)
This class represents lattice values for constants.
Definition: AllocatorList.h:23
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
bool hasStackObjects() const
Return true if there are any stack objects in this function.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned Reg
bool test(unsigned Idx) const
Definition: BitVector.h:502
const SIInstrInfo * getInstrInfo() const override
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:330
unsigned const TargetRegisterInfo * TRI
F(f)
bool removeVGPRForSGPRSpill(Register ReservedVGPR, MachineFunction &MF)
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
void removeLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Remove the specified register from the live in set.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineBasicBlock & MBB
MachineInstrSpan provides an interface to get an iteration range containing the instruction it was in...
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
ArrayRef< MCPhysReg > getAGPRSpillVGPRs() const
bool isReturnBlock() const
Convenience function that returns true if the block ends in a return instruction. ...
ArrayRef< MCPhysReg > getAllVGPR32(const MachineFunction &MF) const
Return all VGPR32 which satisfy the waves per execution unit requirement of the subtarget.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
MachineBasicBlock * getRestorePoint() const
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:456
bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg)
SlotIndex InsertMachineInstrInMaps(MachineInstr &MI)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
virtual void eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj, unsigned FIOperandNum, RegScavenger *RS=nullptr) const =0
This method must be overriden to eliminate abstract frame indices from instructions which may use the...
virtual const TargetInstrInfo * getInstrInfo() const
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
bool isAllocatable(MCRegister PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn&#39;t been...
bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI)
Reserve a slice of a VGPR to support spilling for FrameIndex FI.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:19
TargetInstrInfo - Interface to description of machine instruction set.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:434
unsigned const MachineRegisterInfo * MRI
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:298
virtual bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
void removeAllRegUnitsForPhysReg(unsigned Reg)
Remove associated live ranges for the register units associated with Reg.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
static void insertCSRRestores(MachineBasicBlock &RestoreBlock, MutableArrayRef< CalleeSavedInfo > CSI, LiveIntervals *LIS)
Insert restore code for the callee-saved registers used in the function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Align getSpillAlign(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class...
virtual void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const
Load the specified register of the given register class from the specified stack frame index...
const SIFrameLowering * getFrameLowering() const override
Represent the analysis usage information of a pass.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
static void insertCSRSaves(MachineBasicBlock &SaveBlock, ArrayRef< CalleeSavedInfo > CSI, LiveIntervals *LIS)
Insert restore code for the callee-saved registers used in the function.
bool isPhysRegUsed(MCRegister PhysReg) const
Return true if the specified register is modified or read in this function.
#define DEBUG_TYPE
void setSGPRSpillVGPRs(Register NewVGPR, Optional< int > newFI, int Index)
ArrayRef< SGPRSpillVGPRCSR > getSGPRSpillVGPRs() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const MachineBasicBlock & front() const
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:883
Information about stack frame layout on the target.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
void setPreservesAll()
Set by analyses that do not transform their input at all.
char & SILowerSGPRSpillsID
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:62
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Interface definition for SIInstrInfo.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
bool isEHFuncletEntry() const
Returns true if this is the entry block of an EH funclet.
void push_back(MachineInstr *MI)
#define I(x, y, z)
Definition: MD5.cpp:59
virtual const TargetFrameLowering * getFrameLowering() const
static bool lowerShiftReservedVGPR(MachineFunction &MF, const GCNSubtarget &ST)
uint8_t getStackID(int ObjectIdx) const
MachineBasicBlock * getSavePoint() const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
virtual bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
const MCPhysReg * getCalleeSavedRegs() const
Returns list of callee saved registers.
bool hasMAIInsts() const
INITIALIZE_PASS_BEGIN(SILowerSGPRSpills, DEBUG_TYPE, "SI lower SGPR spill instructions", false, false) INITIALIZE_PASS_END(SILowerSGPRSpills
IRTranslator LLVM IR MI
inst_range instructions(Function *F)
Definition: InstIterator.h:133
ArrayRef< MCPhysReg > getVGPRSpillAGPRs() const
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Definition: STLExtras.h:341
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:466
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
MachineBasicBlock::iterator begin()
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
const SIRegisterInfo * getRegisterInfo() const override