LLVM  10.0.0svn
MLxExpansionPass.cpp
Go to the documentation of this file.
1 //===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
10 // multiple and add / sub instructions) when special VMLx hazards are detected.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "ARM.h"
15 #include "ARMBaseInstrInfo.h"
16 #include "ARMSubtarget.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/Statistic.h"
25 #include "llvm/Support/Debug.h"
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "mlx-expansion"
30 
31 static cl::opt<bool>
32 ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
33 static cl::opt<unsigned>
34 ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
35 
36 STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
37 
38 namespace {
39  struct MLxExpansion : public MachineFunctionPass {
40  static char ID;
41  MLxExpansion() : MachineFunctionPass(ID) {}
42 
43  bool runOnMachineFunction(MachineFunction &Fn) override;
44 
45  StringRef getPassName() const override {
46  return "ARM MLA / MLS expansion pass";
47  }
48 
49  private:
50  const ARMBaseInstrInfo *TII;
51  const TargetRegisterInfo *TRI;
53 
54  bool isLikeA9;
55  bool isSwift;
56  unsigned MIIdx;
57  MachineInstr* LastMIs[4];
59 
60  void clearStack();
61  void pushStack(MachineInstr *MI);
62  MachineInstr *getAccDefMI(MachineInstr *MI) const;
63  unsigned getDefReg(MachineInstr *MI) const;
64  bool hasLoopHazard(MachineInstr *MI) const;
65  bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
66  bool FindMLxHazard(MachineInstr *MI);
67  void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
68  unsigned MulOpc, unsigned AddSubOpc,
69  bool NegAcc, bool HasLane);
70  bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
71  };
72  char MLxExpansion::ID = 0;
73 }
74 
75 void MLxExpansion::clearStack() {
76  std::fill(LastMIs, LastMIs + 4, nullptr);
77  MIIdx = 0;
78 }
79 
80 void MLxExpansion::pushStack(MachineInstr *MI) {
81  LastMIs[MIIdx] = MI;
82  if (++MIIdx == 4)
83  MIIdx = 0;
84 }
85 
86 MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
87  // Look past COPY and INSERT_SUBREG instructions to find the
88  // real definition MI. This is important for _sfp instructions.
89  Register Reg = MI->getOperand(1).getReg();
91  return nullptr;
92 
93  MachineBasicBlock *MBB = MI->getParent();
94  MachineInstr *DefMI = MRI->getVRegDef(Reg);
95  while (true) {
96  if (DefMI->getParent() != MBB)
97  break;
98  if (DefMI->isCopyLike()) {
99  Reg = DefMI->getOperand(1).getReg();
100  if (Register::isVirtualRegister(Reg)) {
101  DefMI = MRI->getVRegDef(Reg);
102  continue;
103  }
104  } else if (DefMI->isInsertSubreg()) {
105  Reg = DefMI->getOperand(2).getReg();
106  if (Register::isVirtualRegister(Reg)) {
107  DefMI = MRI->getVRegDef(Reg);
108  continue;
109  }
110  }
111  break;
112  }
113  return DefMI;
114 }
115 
116 unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
117  Register Reg = MI->getOperand(0).getReg();
118  if (Register::isPhysicalRegister(Reg) || !MRI->hasOneNonDBGUse(Reg))
119  return Reg;
120 
121  MachineBasicBlock *MBB = MI->getParent();
122  MachineInstr *UseMI = &*MRI->use_instr_nodbg_begin(Reg);
123  if (UseMI->getParent() != MBB)
124  return Reg;
125 
126  while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
127  Reg = UseMI->getOperand(0).getReg();
128  if (Register::isPhysicalRegister(Reg) || !MRI->hasOneNonDBGUse(Reg))
129  return Reg;
130  UseMI = &*MRI->use_instr_nodbg_begin(Reg);
131  if (UseMI->getParent() != MBB)
132  return Reg;
133  }
134 
135  return Reg;
136 }
137 
138 /// hasLoopHazard - Check whether an MLx instruction is chained to itself across
139 /// a single-MBB loop.
140 bool MLxExpansion::hasLoopHazard(MachineInstr *MI) const {
141  Register Reg = MI->getOperand(1).getReg();
143  return false;
144 
145  MachineBasicBlock *MBB = MI->getParent();
146  MachineInstr *DefMI = MRI->getVRegDef(Reg);
147  while (true) {
148 outer_continue:
149  if (DefMI->getParent() != MBB)
150  break;
151 
152  if (DefMI->isPHI()) {
153  for (unsigned i = 1, e = DefMI->getNumOperands(); i < e; i += 2) {
154  if (DefMI->getOperand(i + 1).getMBB() == MBB) {
155  Register SrcReg = DefMI->getOperand(i).getReg();
156  if (Register::isVirtualRegister(SrcReg)) {
157  DefMI = MRI->getVRegDef(SrcReg);
158  goto outer_continue;
159  }
160  }
161  }
162  } else if (DefMI->isCopyLike()) {
163  Reg = DefMI->getOperand(1).getReg();
164  if (Register::isVirtualRegister(Reg)) {
165  DefMI = MRI->getVRegDef(Reg);
166  continue;
167  }
168  } else if (DefMI->isInsertSubreg()) {
169  Reg = DefMI->getOperand(2).getReg();
170  if (Register::isVirtualRegister(Reg)) {
171  DefMI = MRI->getVRegDef(Reg);
172  continue;
173  }
174  }
175 
176  break;
177  }
178 
179  return DefMI == MI;
180 }
181 
182 bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
183  // FIXME: Detect integer instructions properly.
184  const MCInstrDesc &MCID = MI->getDesc();
185  unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
186  if (MI->mayStore())
187  return false;
188  unsigned Opcode = MCID.getOpcode();
189  if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
190  return false;
191  if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
192  return MI->readsRegister(Reg, TRI);
193  return false;
194 }
195 
196 static bool isFpMulInstruction(unsigned Opcode) {
197  switch (Opcode) {
198  case ARM::VMULS:
199  case ARM::VMULfd:
200  case ARM::VMULfq:
201  case ARM::VMULD:
202  case ARM::VMULslfd:
203  case ARM::VMULslfq:
204  return true;
205  default:
206  return false;
207  }
208 }
209 
210 bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
211  if (NumExpand >= ExpandLimit)
212  return false;
213 
214  if (ForceExapnd)
215  return true;
216 
217  MachineInstr *DefMI = getAccDefMI(MI);
218  if (TII->isFpMLxInstruction(DefMI->getOpcode())) {
219  // r0 = vmla
220  // r3 = vmla r0, r1, r2
221  // takes 16 - 17 cycles
222  //
223  // r0 = vmla
224  // r4 = vmul r1, r2
225  // r3 = vadd r0, r4
226  // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
227  IgnoreStall.insert(DefMI);
228  return true;
229  }
230 
231  // On Swift, we mostly care about hazards from multiplication instructions
232  // writing the accumulator and the pipelining of loop iterations by out-of-
233  // order execution.
234  if (isSwift)
235  return isFpMulInstruction(DefMI->getOpcode()) || hasLoopHazard(MI);
236 
237  if (IgnoreStall.count(MI))
238  return false;
239 
240  // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
241  // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
242  // preserves the in-order retirement of the instructions.
243  // Look at the next few instructions, if *most* of them can cause hazards,
244  // then the scheduler can't *fix* this, we'd better break up the VMLA.
245  unsigned Limit1 = isLikeA9 ? 1 : 4;
246  unsigned Limit2 = isLikeA9 ? 1 : 4;
247  for (unsigned i = 1; i <= 4; ++i) {
248  int Idx = ((int)MIIdx - i + 4) % 4;
249  MachineInstr *NextMI = LastMIs[Idx];
250  if (!NextMI)
251  continue;
252 
253  if (TII->canCauseFpMLxStall(NextMI->getOpcode())) {
254  if (i <= Limit1)
255  return true;
256  }
257 
258  // Look for VMLx RAW hazard.
259  if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI))
260  return true;
261  }
262 
263  return false;
264 }
265 
266 /// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
267 /// of MUL + ADD / SUB instructions.
268 void
269 MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
270  unsigned MulOpc, unsigned AddSubOpc,
271  bool NegAcc, bool HasLane) {
272  Register DstReg = MI->getOperand(0).getReg();
273  bool DstDead = MI->getOperand(0).isDead();
274  Register AccReg = MI->getOperand(1).getReg();
275  Register Src1Reg = MI->getOperand(2).getReg();
276  Register Src2Reg = MI->getOperand(3).getReg();
277  bool Src1Kill = MI->getOperand(2).isKill();
278  bool Src2Kill = MI->getOperand(3).isKill();
279  unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
280  unsigned NextOp = HasLane ? 5 : 4;
281  ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
282  Register PredReg = MI->getOperand(++NextOp).getReg();
283 
284  const MCInstrDesc &MCID1 = TII->get(MulOpc);
285  const MCInstrDesc &MCID2 = TII->get(AddSubOpc);
286  const MachineFunction &MF = *MI->getParent()->getParent();
287  Register TmpReg =
288  MRI->createVirtualRegister(TII->getRegClass(MCID1, 0, TRI, MF));
289 
290  MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID1, TmpReg)
291  .addReg(Src1Reg, getKillRegState(Src1Kill))
292  .addReg(Src2Reg, getKillRegState(Src2Kill));
293  if (HasLane)
294  MIB.addImm(LaneImm);
295  MIB.addImm(Pred).addReg(PredReg);
296 
297  MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID2)
298  .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
299 
300  if (NegAcc) {
301  bool AccKill = MRI->hasOneNonDBGUse(AccReg);
302  MIB.addReg(TmpReg, getKillRegState(true))
303  .addReg(AccReg, getKillRegState(AccKill));
304  } else {
305  MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
306  }
307  MIB.addImm(Pred).addReg(PredReg);
308 
309  LLVM_DEBUG({
310  dbgs() << "Expanding: " << *MI;
311  dbgs() << " to:\n";
313  MII = std::prev(MII);
314  MachineInstr &MI2 = *MII;
315  MII = std::prev(MII);
316  MachineInstr &MI1 = *MII;
317  dbgs() << " " << MI1;
318  dbgs() << " " << MI2;
319  });
320 
321  MI->eraseFromParent();
322  ++NumExpand;
323 }
324 
325 bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
326  bool Changed = false;
327 
328  clearStack();
329  IgnoreStall.clear();
330 
331  unsigned Skip = 0;
332  MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
333  while (MII != E) {
334  MachineInstr *MI = &*MII++;
335 
336  if (MI->isPosition() || MI->isImplicitDef() || MI->isCopy())
337  continue;
338 
339  const MCInstrDesc &MCID = MI->getDesc();
340  if (MI->isBarrier()) {
341  clearStack();
342  Skip = 0;
343  continue;
344  }
345 
346  unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
347  if (Domain == ARMII::DomainGeneral) {
348  if (++Skip == 2)
349  // Assume dual issues of non-VFP / NEON instructions.
350  pushStack(nullptr);
351  } else {
352  Skip = 0;
353 
354  unsigned MulOpc, AddSubOpc;
355  bool NegAcc, HasLane;
356  if (!TII->isFpMLxInstruction(MCID.getOpcode(),
357  MulOpc, AddSubOpc, NegAcc, HasLane) ||
358  !FindMLxHazard(MI))
359  pushStack(MI);
360  else {
361  ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
362  Changed = true;
363  }
364  }
365  }
366 
367  return Changed;
368 }
369 
370 bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
371  if (skipFunction(Fn.getFunction()))
372  return false;
373 
374  TII = static_cast<const ARMBaseInstrInfo *>(Fn.getSubtarget().getInstrInfo());
376  MRI = &Fn.getRegInfo();
377  const ARMSubtarget *STI = &Fn.getSubtarget<ARMSubtarget>();
378  if (!STI->expandMLx())
379  return false;
380  isLikeA9 = STI->isLikeA9() || STI->isSwift();
381  isSwift = STI->isSwift();
382 
383  bool Modified = false;
384  for (MachineBasicBlock &MBB : Fn)
385  Modified |= ExpandFPMLxInstructions(MBB);
386 
387  return Modified;
388 }
389 
391  return new MLxExpansion();
392 }
FunctionPass * createMLxExpansionPass()
MachineBasicBlock * getMBB() const
This class represents lattice values for constants.
Definition: AllocatorList.h:23
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:385
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:63
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:178
unsigned Reg
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
bool isCopyLike() const
Return true if the instruction behaves like a copy.
bool isPHI() const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:414
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:411
static cl::opt< unsigned > ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden)
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:408
static cl::opt< bool > ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden)
virtual const TargetInstrInfo * getInstrInfo() const
reverse_iterator rend()
reverse_iterator rbegin()
unsigned getKillRegState(bool B)
unsigned getDeadRegState(bool B)
unsigned getDefRegState(bool B)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:838
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
unsigned const MachineRegisterInfo * MRI
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineInstrBuilder & UseMI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
bool isCopy() const
bool isImplicitDef() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
static bool hasRAWHazard(MachineInstr *DefMI, MachineInstr *MI, const TargetRegisterInfo &TRI)
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr reads the specified register.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
bool isInsertSubreg() const
MachineInstrBuilder MachineInstrBuilder & DefMI
int64_t getImm() const
const Function & getFunction() const
Return the LLVM function that this machine code represents.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
bool expandMLx() const
Definition: ARMSubtarget.h:653
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:256
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:64
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
bool isSwift() const
Definition: ARMSubtarget.h:600
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:69
bool isPosition() const
unsigned getOpcode() const
Return the opcode number for this descriptor.
Definition: MCInstrDesc.h:218
IRTranslator LLVM IR MI
bool isBarrier(QueryType Type=AnyInBundle) const
Returns true if the specified instruction stops control flow from executing the instruction immediate...
Definition: MachineInstr.h:658
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
Register getReg() const
getReg - Returns the register number.
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
static bool isFpMulInstruction(unsigned Opcode)
bool isLikeA9() const
Definition: ARMSubtarget.h:602
Wrapper class representing virtual and physical registers.
Definition: Register.h:19