LLVM  9.0.0svn
SIFormMemoryClauses.cpp
Go to the documentation of this file.
1 //===-- SIFormMemoryClauses.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass creates bundles of SMEM and VMEM instructions forming memory
11 /// clauses if XNACK is enabled. Def operands of clauses are marked as early
12 /// clobber to make sure we will not override any source within a clause.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "AMDGPU.h"
17 #include "AMDGPUSubtarget.h"
18 #include "GCNRegPressure.h"
19 #include "SIInstrInfo.h"
20 #include "SIMachineFunctionInfo.h"
21 #include "SIRegisterInfo.h"
23 #include "llvm/ADT/DenseMap.h"
26 
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "si-form-memory-clauses"
30 
31 // Clauses longer then 15 instructions would overflow one of the counters
32 // and stall. They can stall even earlier if there are outstanding counters.
33 static cl::opt<unsigned>
34 MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15),
35  cl::desc("Maximum length of a memory clause, instructions"));
36 
37 namespace {
38 
39 class SIFormMemoryClauses : public MachineFunctionPass {
41 
42 public:
43  static char ID;
44 
45 public:
46  SIFormMemoryClauses() : MachineFunctionPass(ID) {
48  }
49 
50  bool runOnMachineFunction(MachineFunction &MF) override;
51 
52  StringRef getPassName() const override {
53  return "SI Form memory clauses";
54  }
55 
56  void getAnalysisUsage(AnalysisUsage &AU) const override {
58  AU.setPreservesAll();
60  }
61 
62 private:
63  template <typename Callable>
64  void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const;
65 
66  bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
67  bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
68  void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
69  bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
71 
72  const GCNSubtarget *ST;
73  const SIRegisterInfo *TRI;
74  const MachineRegisterInfo *MRI;
76 
77  unsigned LastRecordedOccupancy;
78  unsigned MaxVGPRs;
79  unsigned MaxSGPRs;
80 };
81 
82 } // End anonymous namespace.
83 
84 INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE,
85  "SI Form memory clauses", false, false)
87 INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE,
88  "SI Form memory clauses", false, false)
89 
90 
91 char SIFormMemoryClauses::ID = 0;
92 
93 char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID;
94 
96  return new SIFormMemoryClauses();
97 }
98 
99 static bool isVMEMClauseInst(const MachineInstr &MI) {
100  return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI);
101 }
102 
103 static bool isSMEMClauseInst(const MachineInstr &MI) {
104  return SIInstrInfo::isSMRD(MI);
105 }
106 
107 // There no sense to create store clauses, they do not define anything,
108 // thus there is nothing to set early-clobber.
109 static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
110  if (MI.isDebugValue() || MI.isBundled())
111  return false;
112  if (!MI.mayLoad() || MI.mayStore())
113  return false;
114  if (AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1 ||
115  AMDGPU::getAtomicRetOp(MI.getOpcode()) != -1)
116  return false;
117  if (IsVMEMClause && !isVMEMClauseInst(MI))
118  return false;
119  if (!IsVMEMClause && !isSMEMClauseInst(MI))
120  return false;
121  // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
122  for (const MachineOperand &ResMO : MI.defs()) {
123  unsigned ResReg = ResMO.getReg();
124  for (const MachineOperand &MO : MI.uses()) {
125  if (!MO.isReg() || MO.isDef())
126  continue;
127  if (MO.getReg() == ResReg)
128  return false;
129  }
130  break; // Only check the first def.
131  }
132  return true;
133 }
134 
135 static unsigned getMopState(const MachineOperand &MO) {
136  unsigned S = 0;
137  if (MO.isImplicit())
138  S |= RegState::Implicit;
139  if (MO.isDead())
140  S |= RegState::Dead;
141  if (MO.isUndef())
142  S |= RegState::Undef;
143  if (MO.isKill())
144  S |= RegState::Kill;
145  if (MO.isEarlyClobber())
148  S |= RegState::Renamable;
149  return S;
150 }
151 
152 template <typename Callable>
153 void SIFormMemoryClauses::forAllLanes(unsigned Reg, LaneBitmask LaneMask,
154  Callable Func) const {
155  if (LaneMask.all() || TargetRegisterInfo::isPhysicalRegister(Reg) ||
156  LaneMask == MRI->getMaxLaneMaskForVReg(Reg)) {
157  Func(0);
158  return;
159  }
160 
161  const TargetRegisterClass *RC = MRI->getRegClass(Reg);
162  unsigned E = TRI->getNumSubRegIndices();
164  for (unsigned Idx = 1; Idx < E; ++Idx) {
165  // Is this index even compatible with the given class?
166  if (TRI->getSubClassWithSubReg(RC, Idx) != RC)
167  continue;
168  LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
169  // Early exit if we found a perfect match.
170  if (SubRegMask == LaneMask) {
171  Func(Idx);
172  return;
173  }
174 
175  if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
176  continue;
177 
178  CoveringSubregs.push_back(Idx);
179  }
180 
181  llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) {
182  LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
183  LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
184  unsigned NA = MaskA.getNumLanes();
185  unsigned NB = MaskB.getNumLanes();
186  if (NA != NB)
187  return NA > NB;
188  return MaskA.getHighestLane() > MaskB.getHighestLane();
189  });
190 
191  for (unsigned Idx : CoveringSubregs) {
192  LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
193  if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
194  continue;
195 
196  Func(Idx);
197  LaneMask &= ~SubRegMask;
198  if (LaneMask.none())
199  return;
200  }
201 
202  llvm_unreachable("Failed to find all subregs to cover lane mask");
203 }
204 
205 // Returns false if there is a use of a def already in the map.
206 // In this case we must break the clause.
207 bool SIFormMemoryClauses::canBundle(const MachineInstr &MI,
208  RegUse &Defs, RegUse &Uses) const {
209  // Check interference with defs.
210  for (const MachineOperand &MO : MI.operands()) {
211  // TODO: Prologue/Epilogue Insertion pass does not process bundled
212  // instructions.
213  if (MO.isFI())
214  return false;
215 
216  if (!MO.isReg())
217  continue;
218 
219  unsigned Reg = MO.getReg();
220 
221  // If it is tied we will need to write same register as we read.
222  if (MO.isTied())
223  return false;
224 
225  RegUse &Map = MO.isDef() ? Uses : Defs;
226  auto Conflict = Map.find(Reg);
227  if (Conflict == Map.end())
228  continue;
229 
231  return false;
232 
233  LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
234  if ((Conflict->second.second & Mask).any())
235  return false;
236  }
237 
238  return true;
239 }
240 
241 // Since all defs in the clause are early clobber we can run out of registers.
242 // Function returns false if pressure would hit the limit if instruction is
243 // bundled into a memory clause.
244 bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI,
245  GCNDownwardRPTracker &RPT) {
246  // NB: skip advanceBeforeNext() call. Since all defs will be marked
247  // early-clobber they will all stay alive at least to the end of the
248  // clause. Therefor we should not decrease pressure even if load
249  // pointer becomes dead and could otherwise be reused for destination.
250  RPT.advanceToNext();
251  GCNRegPressure MaxPressure = RPT.moveMaxPressure();
252  unsigned Occupancy = MaxPressure.getOccupancy(*ST);
253  if (Occupancy >= MFI->getMinAllowedOccupancy() &&
254  MaxPressure.getVGPRNum() <= MaxVGPRs &&
255  MaxPressure.getSGPRNum() <= MaxSGPRs) {
256  LastRecordedOccupancy = Occupancy;
257  return true;
258  }
259  return false;
260 }
261 
262 // Collect register defs and uses along with their lane masks and states.
263 void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
264  RegUse &Defs, RegUse &Uses) const {
265  for (const MachineOperand &MO : MI.operands()) {
266  if (!MO.isReg())
267  continue;
268  unsigned Reg = MO.getReg();
269  if (!Reg)
270  continue;
271 
273  TRI->getSubRegIndexLaneMask(MO.getSubReg()) :
275  RegUse &Map = MO.isDef() ? Defs : Uses;
276 
277  auto Loc = Map.find(Reg);
278  unsigned State = getMopState(MO);
279  if (Loc == Map.end()) {
280  Map[Reg] = std::make_pair(State, Mask);
281  } else {
282  Loc->second.first |= State;
283  Loc->second.second |= Mask;
284  }
285  }
286 }
287 
288 // Check register def/use conflicts, occupancy limits and collect def/use maps.
289 // Return true if instruction can be bundled with previous. It it cannot
290 // def/use maps are not updated.
291 bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
292  RegUse &Defs, RegUse &Uses,
293  GCNDownwardRPTracker &RPT) {
294  if (!canBundle(MI, Defs, Uses))
295  return false;
296 
297  if (!checkPressure(MI, RPT))
298  return false;
299 
300  collectRegUses(MI, Defs, Uses);
301  return true;
302 }
303 
304 bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
305  if (skipFunction(MF.getFunction()))
306  return false;
307 
308  ST = &MF.getSubtarget<GCNSubtarget>();
309  if (!ST->isXNACKEnabled())
310  return false;
311 
312  const SIInstrInfo *TII = ST->getInstrInfo();
313  TRI = ST->getRegisterInfo();
314  MRI = &MF.getRegInfo();
315  MFI = MF.getInfo<SIMachineFunctionInfo>();
316  LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
317  SlotIndexes *Ind = LIS->getSlotIndexes();
318  bool Changed = false;
319 
320  MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
321  MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
322  unsigned FuncMaxClause = AMDGPU::getIntegerAttribute(
323  MF.getFunction(), "amdgpu-max-memory-clause", MaxClause);
324 
325  for (MachineBasicBlock &MBB : MF) {
327  for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
328  MachineInstr &MI = *I;
329  Next = std::next(I);
330 
331  bool IsVMEM = isVMEMClauseInst(MI);
332 
333  if (!isValidClauseInst(MI, IsVMEM))
334  continue;
335 
336  RegUse Defs, Uses;
337  GCNDownwardRPTracker RPT(*LIS);
338  RPT.reset(MI);
339 
340  if (!processRegUses(MI, Defs, Uses, RPT))
341  continue;
342 
343  unsigned Length = 1;
344  for ( ; Next != E && Length < FuncMaxClause; ++Next) {
345  if (!isValidClauseInst(*Next, IsVMEM))
346  break;
347 
348  // A load from pointer which was loaded inside the same bundle is an
349  // impossible clause because we will need to write and read the same
350  // register inside. In this case processRegUses will return false.
351  if (!processRegUses(*Next, Defs, Uses, RPT))
352  break;
353 
354  ++Length;
355  }
356  if (Length < 2)
357  continue;
358 
359  Changed = true;
360  MFI->limitOccupancy(LastRecordedOccupancy);
361 
362  auto B = BuildMI(MBB, I, DebugLoc(), TII->get(TargetOpcode::BUNDLE));
363  Ind->insertMachineInstrInMaps(*B);
364 
365  for (auto BI = I; BI != Next; ++BI) {
366  BI->bundleWithPred();
367  Ind->removeSingleMachineInstrFromMaps(*BI);
368 
369  for (MachineOperand &MO : BI->defs())
370  if (MO.readsReg())
371  MO.setIsInternalRead(true);
372  }
373 
374  for (auto &&R : Defs) {
375  forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
376  unsigned S = R.second.first | RegState::EarlyClobber;
377  if (!SubReg)
378  S &= ~(RegState::Undef | RegState::Dead);
379  B.addDef(R.first, S, SubReg);
380  });
381  }
382 
383  for (auto &&R : Uses) {
384  forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
385  B.addUse(R.first, R.second.first & ~RegState::Kill, SubReg);
386  });
387  }
388 
389  for (auto &&R : Defs) {
390  unsigned Reg = R.first;
391  Uses.erase(Reg);
393  continue;
394  LIS->removeInterval(Reg);
395  LIS->createAndComputeVirtRegInterval(Reg);
396  }
397 
398  for (auto &&R : Uses) {
399  unsigned Reg = R.first;
401  continue;
402  LIS->removeInterval(Reg);
403  LIS->createAndComputeVirtRegInterval(Reg);
404  }
405  }
406  }
407 
408  return Changed;
409 }
char & SIFormMemoryClausesID
unsigned getNumLanes() const
Definition: LaneBitmask.h:75
void bundleWithPred()
Bundle this instruction with its predecessor.
Interface definition for SIRegisterInfo.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
AMDGPU specific subclass of TargetSubtarget.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:493
FunctionPass * createSIFormMemoryClausesPass()
void initializeSIFormMemoryClausesPass(PassRegistry &)
SI Form memory clauses
decltype(MaxPressure) moveMaxPressure()
bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs=nullptr)
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
unsigned Reg
unsigned getHighestLane() const
Definition: LaneBitmask.h:78
static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause)
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
LLVM_READONLY int getAtomicNoRetOp(uint16_t Opcode)
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:460
static bool isSMRD(const MachineInstr &MI)
Definition: SIInstrInfo.h:452
bool isEarlyClobber() const
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
LLVM_READONLY int getAtomicRetOp(uint16_t Opcode)
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:488
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
unsigned SubReg
static constexpr LaneBitmask getAll()
Definition: LaneBitmask.h:83
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:410
unsigned getOccupancy(const GCNSubtarget &ST) const
SlotIndexes pass.
Definition: SlotIndexes.h:314
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:821
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
unsigned const MachineRegisterInfo * MRI
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
static cl::opt< unsigned > MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15), cl::desc("Maximum length of a memory clause, instructions"))
bool isBundled() const
Return true if this instruction part of a bundle.
Definition: MachineInstr.h:357
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
unsigned getSGPRNum() const
Represent the analysis usage information of a pass.
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
Definition: MachineInstr.h:482
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
constexpr bool all() const
Definition: LaneBitmask.h:53
static unsigned getMopState(const MachineOperand &MO)
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1122
Iterator for intrusive lists based on ilist_node.
bool isDebugValue() const
MachineOperand class - Representation of each machine instruction operand.
unsigned getVGPRNum() const
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
const Function & getFunction() const
Return the LLVM function that this machine code represents.
void setPreservesAll()
Set by analyses that do not transform their input at all.
bool isRenamable() const
isRenamable - Returns true if this register may be renamed, i.e.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Provides AMDGPU specific target descriptions.
Representation of each machine instruction.
Definition: MachineInstr.h:63
#define DEBUG_TYPE
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Interface definition for SIInstrInfo.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
static bool isVMEMClauseInst(const MachineInstr &MI)
int getIntegerAttribute(const Function &F, StringRef Name, int Default)
#define I(x, y, z)
Definition: MD5.cpp:58
static bool isSMEMClauseInst(const MachineInstr &MI)
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:340
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Definition: MachineInstr.h:808
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE, "SI Form memory clauses", false, false) INITIALIZE_PASS_END(SIFormMemoryClauses
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
Register getReg() const
getReg - Returns the register number.
bool isImplicit() const