LLVM  14.0.0git
AArch64SIMDInstrOpt.cpp
Go to the documentation of this file.
1 //
2 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3 // See https://llvm.org/LICENSE.txt for license information.
4 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5 //
6 //===----------------------------------------------------------------------===//
7 //
8 // This file contains a pass that performs optimization on SIMD instructions
9 // with high latency by splitting them into more efficient series of
10 // instructions.
11 //
12 // 1. Rewrite certain SIMD instructions with vector element due to their
13 // inefficiency on some targets.
14 //
15 // For example:
16 // fmla v0.4s, v1.4s, v2.s[1]
17 //
18 // Is rewritten into:
19 // dup v3.4s, v2.s[1]
20 // fmla v0.4s, v1.4s, v3.4s
21 //
22 // 2. Rewrite interleaved memory access instructions due to their
23 // inefficiency on some targets.
24 //
25 // For example:
26 // st2 {v0.4s, v1.4s}, addr
27 //
28 // Is rewritten into:
29 // zip1 v2.4s, v0.4s, v1.4s
30 // zip2 v3.4s, v0.4s, v1.4s
31 // stp q2, q3, addr
32 //
33 //===----------------------------------------------------------------------===//
34 
35 #include "AArch64InstrInfo.h"
36 #include "llvm/ADT/SmallVector.h"
37 #include "llvm/ADT/Statistic.h"
38 #include "llvm/ADT/StringRef.h"
49 #include "llvm/MC/MCInstrDesc.h"
50 #include "llvm/MC/MCSchedule.h"
51 #include "llvm/Pass.h"
52 #include <unordered_map>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "aarch64-simdinstr-opt"
57 
58 STATISTIC(NumModifiedInstr,
59  "Number of SIMD instructions modified");
60 
61 #define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \
62  "AArch64 SIMD instructions optimization pass"
63 
64 namespace {
65 
66 struct AArch64SIMDInstrOpt : public MachineFunctionPass {
67  static char ID;
68 
69  const TargetInstrInfo *TII;
71  TargetSchedModel SchedModel;
72 
73  // The two maps below are used to cache decisions instead of recomputing:
74  // This is used to cache instruction replacement decisions within function
75  // units and across function units.
76  std::map<std::pair<unsigned, std::string>, bool> SIMDInstrTable;
77  // This is used to cache the decision of whether to leave the interleaved
78  // store instructions replacement pass early or not for a particular target.
79  std::unordered_map<std::string, bool> InterlEarlyExit;
80 
81  typedef enum {
82  VectorElem,
83  Interleave
84  } Subpass;
85 
86  // Instruction represented by OrigOpc is replaced by instructions in ReplOpc.
87  struct InstReplInfo {
88  unsigned OrigOpc;
89  std::vector<unsigned> ReplOpc;
90  const TargetRegisterClass RC;
91  };
92 
93 #define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC) \
94  {OpcOrg, {OpcR0, OpcR1, OpcR2}, RC}
95 #define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, \
96  OpcR7, OpcR8, OpcR9, RC) \
97  {OpcOrg, \
98  {OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, OpcR8, OpcR9}, RC}
99 
100  // The Instruction Replacement Table:
101  std::vector<InstReplInfo> IRT = {
102  // ST2 instructions
103  RuleST2(AArch64::ST2Twov2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
104  AArch64::STPQi, AArch64::FPR128RegClass),
105  RuleST2(AArch64::ST2Twov4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
106  AArch64::STPQi, AArch64::FPR128RegClass),
107  RuleST2(AArch64::ST2Twov2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
108  AArch64::STPDi, AArch64::FPR64RegClass),
109  RuleST2(AArch64::ST2Twov8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
110  AArch64::STPQi, AArch64::FPR128RegClass),
111  RuleST2(AArch64::ST2Twov4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
112  AArch64::STPDi, AArch64::FPR64RegClass),
113  RuleST2(AArch64::ST2Twov16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
114  AArch64::STPQi, AArch64::FPR128RegClass),
115  RuleST2(AArch64::ST2Twov8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
116  AArch64::STPDi, AArch64::FPR64RegClass),
117  // ST4 instructions
118  RuleST4(AArch64::ST4Fourv2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
119  AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64,
120  AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
121  AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
122  RuleST4(AArch64::ST4Fourv4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
123  AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32,
124  AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
125  AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
126  RuleST4(AArch64::ST4Fourv2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
127  AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32,
128  AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
129  AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),
130  RuleST4(AArch64::ST4Fourv8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
131  AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16,
132  AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
133  AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
134  RuleST4(AArch64::ST4Fourv4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
135  AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16,
136  AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
137  AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),
138  RuleST4(AArch64::ST4Fourv16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
139  AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8,
140  AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
141  AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
142  RuleST4(AArch64::ST4Fourv8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
143  AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8,
144  AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
145  AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass)
146  };
147 
148  // A costly instruction is replaced in this work by N efficient instructions
149  // The maximum of N is curently 10 and it is for ST4 case.
150  static const unsigned MaxNumRepl = 10;
151 
152  AArch64SIMDInstrOpt() : MachineFunctionPass(ID) {
154  }
155 
156  /// Based only on latency of instructions, determine if it is cost efficient
157  /// to replace the instruction InstDesc by the instructions stored in the
158  /// array InstDescRepl.
159  /// Return true if replacement is expected to be faster.
160  bool shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc,
161  SmallVectorImpl<const MCInstrDesc*> &ReplInstrMCID);
162 
163  /// Determine if we need to exit the instruction replacement optimization
164  /// passes early. This makes sure that no compile time is spent in this pass
165  /// for targets with no need for any of these optimizations.
166  /// Return true if early exit of the pass is recommended.
167  bool shouldExitEarly(MachineFunction *MF, Subpass SP);
168 
169  /// Check whether an equivalent DUP instruction has already been
170  /// created or not.
171  /// Return true when the DUP instruction already exists. In this case,
172  /// DestReg will point to the destination of the already created DUP.
173  bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg,
174  unsigned LaneNumber, unsigned *DestReg) const;
175 
176  /// Certain SIMD instructions with vector element operand are not efficient.
177  /// Rewrite them into SIMD instructions with vector operands. This rewrite
178  /// is driven by the latency of the instructions.
179  /// Return true if the SIMD instruction is modified.
180  bool optimizeVectElement(MachineInstr &MI);
181 
182  /// Process The REG_SEQUENCE instruction, and extract the source
183  /// operands of the ST2/4 instruction from it.
184  /// Example of such instructions.
185  /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;
186  /// Return true when the instruction is processed successfully.
187  bool processSeqRegInst(MachineInstr *DefiningMI, unsigned* StReg,
188  unsigned* StRegKill, unsigned NumArg) const;
189 
190  /// Load/Store Interleaving instructions are not always beneficial.
191  /// Replace them by ZIP instructionand classical load/store.
192  /// Return true if the SIMD instruction is modified.
193  bool optimizeLdStInterleave(MachineInstr &MI);
194 
195  /// Return the number of useful source registers for this
196  /// instruction (2 for ST2 and 4 for ST4).
197  unsigned determineSrcReg(MachineInstr &MI) const;
198 
199  bool runOnMachineFunction(MachineFunction &Fn) override;
200 
201  StringRef getPassName() const override {
203  }
204 };
205 
206 char AArch64SIMDInstrOpt::ID = 0;
207 
208 } // end anonymous namespace
209 
210 INITIALIZE_PASS(AArch64SIMDInstrOpt, "aarch64-simdinstr-opt",
212 
213 /// Based only on latency of instructions, determine if it is cost efficient
214 /// to replace the instruction InstDesc by the instructions stored in the
215 /// array InstDescRepl.
216 /// Return true if replacement is expected to be faster.
217 bool AArch64SIMDInstrOpt::
218 shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc,
219  SmallVectorImpl<const MCInstrDesc*> &InstDescRepl) {
220  // Check if replacement decision is already available in the cached table.
221  // if so, return it.
222  std::string Subtarget = std::string(SchedModel.getSubtargetInfo()->getCPU());
223  auto InstID = std::make_pair(InstDesc->getOpcode(), Subtarget);
224  auto It = SIMDInstrTable.find(InstID);
225  if (It != SIMDInstrTable.end())
226  return It->second;
227 
228  unsigned SCIdx = InstDesc->getSchedClass();
229  const MCSchedClassDesc *SCDesc =
230  SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx);
231 
232  // If a target does not define resources for the instructions
233  // of interest, then return false for no replacement.
234  const MCSchedClassDesc *SCDescRepl;
235  if (!SCDesc->isValid() || SCDesc->isVariant())
236  {
237  SIMDInstrTable[InstID] = false;
238  return false;
239  }
240  for (auto IDesc : InstDescRepl)
241  {
242  SCDescRepl = SchedModel.getMCSchedModel()->getSchedClassDesc(
243  IDesc->getSchedClass());
244  if (!SCDescRepl->isValid() || SCDescRepl->isVariant())
245  {
246  SIMDInstrTable[InstID] = false;
247  return false;
248  }
249  }
250 
251  // Replacement cost.
252  unsigned ReplCost = 0;
253  for (auto IDesc :InstDescRepl)
254  ReplCost += SchedModel.computeInstrLatency(IDesc->getOpcode());
255 
256  if (SchedModel.computeInstrLatency(InstDesc->getOpcode()) > ReplCost)
257  {
258  SIMDInstrTable[InstID] = true;
259  return true;
260  }
261  else
262  {
263  SIMDInstrTable[InstID] = false;
264  return false;
265  }
266 }
267 
268 /// Determine if we need to exit this pass for a kind of instruction replacement
269 /// early. This makes sure that no compile time is spent in this pass for
270 /// targets with no need for any of these optimizations beyond performing this
271 /// check.
272 /// Return true if early exit of this pass for a kind of instruction
273 /// replacement is recommended for a target.
274 bool AArch64SIMDInstrOpt::shouldExitEarly(MachineFunction *MF, Subpass SP) {
275  const MCInstrDesc* OriginalMCID;
277 
278  switch (SP) {
279  // For this optimization, check by comparing the latency of a representative
280  // instruction to that of the replacement instructions.
281  // TODO: check for all concerned instructions.
282  case VectorElem:
283  OriginalMCID = &TII->get(AArch64::FMLAv4i32_indexed);
284  ReplInstrMCID.push_back(&TII->get(AArch64::DUPv4i32lane));
285  ReplInstrMCID.push_back(&TII->get(AArch64::FMLAv4f32));
286  if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID))
287  return false;
288  break;
289 
290  // For this optimization, check for all concerned instructions.
291  case Interleave:
292  std::string Subtarget =
293  std::string(SchedModel.getSubtargetInfo()->getCPU());
294  auto It = InterlEarlyExit.find(Subtarget);
295  if (It != InterlEarlyExit.end())
296  return It->second;
297 
298  for (auto &I : IRT) {
299  OriginalMCID = &TII->get(I.OrigOpc);
300  for (auto &Repl : I.ReplOpc)
301  ReplInstrMCID.push_back(&TII->get(Repl));
302  if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) {
303  InterlEarlyExit[Subtarget] = false;
304  return false;
305  }
306  ReplInstrMCID.clear();
307  }
308  InterlEarlyExit[Subtarget] = true;
309  break;
310  }
311 
312  return true;
313 }
314 
315 /// Check whether an equivalent DUP instruction has already been
316 /// created or not.
317 /// Return true when the DUP instruction already exists. In this case,
318 /// DestReg will point to the destination of the already created DUP.
319 bool AArch64SIMDInstrOpt::reuseDUP(MachineInstr &MI, unsigned DupOpcode,
320  unsigned SrcReg, unsigned LaneNumber,
321  unsigned *DestReg) const {
322  for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin();
323  MII != MIE;) {
324  MII--;
325  MachineInstr *CurrentMI = &*MII;
326 
327  if (CurrentMI->getOpcode() == DupOpcode &&
328  CurrentMI->getNumOperands() == 3 &&
329  CurrentMI->getOperand(1).getReg() == SrcReg &&
330  CurrentMI->getOperand(2).getImm() == LaneNumber) {
331  *DestReg = CurrentMI->getOperand(0).getReg();
332  return true;
333  }
334  }
335 
336  return false;
337 }
338 
339 /// Certain SIMD instructions with vector element operand are not efficient.
340 /// Rewrite them into SIMD instructions with vector operands. This rewrite
341 /// is driven by the latency of the instructions.
342 /// The instruction of concerns are for the time being FMLA, FMLS, FMUL,
343 /// and FMULX and hence they are hardcoded.
344 ///
345 /// For example:
346 /// fmla v0.4s, v1.4s, v2.s[1]
347 ///
348 /// Is rewritten into
349 /// dup v3.4s, v2.s[1] // DUP not necessary if redundant
350 /// fmla v0.4s, v1.4s, v3.4s
351 ///
352 /// Return true if the SIMD instruction is modified.
353 bool AArch64SIMDInstrOpt::optimizeVectElement(MachineInstr &MI) {
354  const MCInstrDesc *MulMCID, *DupMCID;
355  const TargetRegisterClass *RC = &AArch64::FPR128RegClass;
356 
357  switch (MI.getOpcode()) {
358  default:
359  return false;
360 
361  // 4X32 instructions
362  case AArch64::FMLAv4i32_indexed:
363  DupMCID = &TII->get(AArch64::DUPv4i32lane);
364  MulMCID = &TII->get(AArch64::FMLAv4f32);
365  break;
366  case AArch64::FMLSv4i32_indexed:
367  DupMCID = &TII->get(AArch64::DUPv4i32lane);
368  MulMCID = &TII->get(AArch64::FMLSv4f32);
369  break;
370  case AArch64::FMULXv4i32_indexed:
371  DupMCID = &TII->get(AArch64::DUPv4i32lane);
372  MulMCID = &TII->get(AArch64::FMULXv4f32);
373  break;
374  case AArch64::FMULv4i32_indexed:
375  DupMCID = &TII->get(AArch64::DUPv4i32lane);
376  MulMCID = &TII->get(AArch64::FMULv4f32);
377  break;
378 
379  // 2X64 instructions
380  case AArch64::FMLAv2i64_indexed:
381  DupMCID = &TII->get(AArch64::DUPv2i64lane);
382  MulMCID = &TII->get(AArch64::FMLAv2f64);
383  break;
384  case AArch64::FMLSv2i64_indexed:
385  DupMCID = &TII->get(AArch64::DUPv2i64lane);
386  MulMCID = &TII->get(AArch64::FMLSv2f64);
387  break;
388  case AArch64::FMULXv2i64_indexed:
389  DupMCID = &TII->get(AArch64::DUPv2i64lane);
390  MulMCID = &TII->get(AArch64::FMULXv2f64);
391  break;
392  case AArch64::FMULv2i64_indexed:
393  DupMCID = &TII->get(AArch64::DUPv2i64lane);
394  MulMCID = &TII->get(AArch64::FMULv2f64);
395  break;
396 
397  // 2X32 instructions
398  case AArch64::FMLAv2i32_indexed:
399  RC = &AArch64::FPR64RegClass;
400  DupMCID = &TII->get(AArch64::DUPv2i32lane);
401  MulMCID = &TII->get(AArch64::FMLAv2f32);
402  break;
403  case AArch64::FMLSv2i32_indexed:
404  RC = &AArch64::FPR64RegClass;
405  DupMCID = &TII->get(AArch64::DUPv2i32lane);
406  MulMCID = &TII->get(AArch64::FMLSv2f32);
407  break;
408  case AArch64::FMULXv2i32_indexed:
409  RC = &AArch64::FPR64RegClass;
410  DupMCID = &TII->get(AArch64::DUPv2i32lane);
411  MulMCID = &TII->get(AArch64::FMULXv2f32);
412  break;
413  case AArch64::FMULv2i32_indexed:
414  RC = &AArch64::FPR64RegClass;
415  DupMCID = &TII->get(AArch64::DUPv2i32lane);
416  MulMCID = &TII->get(AArch64::FMULv2f32);
417  break;
418  }
419 
421  ReplInstrMCID.push_back(DupMCID);
422  ReplInstrMCID.push_back(MulMCID);
423  if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()),
424  ReplInstrMCID))
425  return false;
426 
427  const DebugLoc &DL = MI.getDebugLoc();
428  MachineBasicBlock &MBB = *MI.getParent();
430 
431  // Get the operands of the current SIMD arithmetic instruction.
432  Register MulDest = MI.getOperand(0).getReg();
433  Register SrcReg0 = MI.getOperand(1).getReg();
434  unsigned Src0IsKill = getKillRegState(MI.getOperand(1).isKill());
435  Register SrcReg1 = MI.getOperand(2).getReg();
436  unsigned Src1IsKill = getKillRegState(MI.getOperand(2).isKill());
437  unsigned DupDest;
438 
439  // Instructions of interest have either 4 or 5 operands.
440  if (MI.getNumOperands() == 5) {
441  Register SrcReg2 = MI.getOperand(3).getReg();
442  unsigned Src2IsKill = getKillRegState(MI.getOperand(3).isKill());
443  unsigned LaneNumber = MI.getOperand(4).getImm();
444  // Create a new DUP instruction. Note that if an equivalent DUP instruction
445  // has already been created before, then use that one instead of creating
446  // a new one.
447  if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) {
448  DupDest = MRI.createVirtualRegister(RC);
449  BuildMI(MBB, MI, DL, *DupMCID, DupDest)
450  .addReg(SrcReg2, Src2IsKill)
451  .addImm(LaneNumber);
452  }
453  BuildMI(MBB, MI, DL, *MulMCID, MulDest)
454  .addReg(SrcReg0, Src0IsKill)
455  .addReg(SrcReg1, Src1IsKill)
456  .addReg(DupDest, Src2IsKill);
457  } else if (MI.getNumOperands() == 4) {
458  unsigned LaneNumber = MI.getOperand(3).getImm();
459  if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) {
460  DupDest = MRI.createVirtualRegister(RC);
461  BuildMI(MBB, MI, DL, *DupMCID, DupDest)
462  .addReg(SrcReg1, Src1IsKill)
463  .addImm(LaneNumber);
464  }
465  BuildMI(MBB, MI, DL, *MulMCID, MulDest)
466  .addReg(SrcReg0, Src0IsKill)
467  .addReg(DupDest, Src1IsKill);
468  } else {
469  return false;
470  }
471 
472  ++NumModifiedInstr;
473  return true;
474 }
475 
476 /// Load/Store Interleaving instructions are not always beneficial.
477 /// Replace them by ZIP instructions and classical load/store.
478 ///
479 /// For example:
480 /// st2 {v0.4s, v1.4s}, addr
481 ///
482 /// Is rewritten into:
483 /// zip1 v2.4s, v0.4s, v1.4s
484 /// zip2 v3.4s, v0.4s, v1.4s
485 /// stp q2, q3, addr
486 //
487 /// For example:
488 /// st4 {v0.4s, v1.4s, v2.4s, v3.4s}, addr
489 ///
490 /// Is rewritten into:
491 /// zip1 v4.4s, v0.4s, v2.4s
492 /// zip2 v5.4s, v0.4s, v2.4s
493 /// zip1 v6.4s, v1.4s, v3.4s
494 /// zip2 v7.4s, v1.4s, v3.4s
495 /// zip1 v8.4s, v4.4s, v6.4s
496 /// zip2 v9.4s, v4.4s, v6.4s
497 /// zip1 v10.4s, v5.4s, v7.4s
498 /// zip2 v11.4s, v5.4s, v7.4s
499 /// stp q8, q9, addr
500 /// stp q10, q11, addr+32
501 ///
502 /// Currently only instructions related to ST2 and ST4 are considered.
503 /// Other may be added later.
504 /// Return true if the SIMD instruction is modified.
505 bool AArch64SIMDInstrOpt::optimizeLdStInterleave(MachineInstr &MI) {
506 
507  unsigned SeqReg, AddrReg;
508  unsigned StReg[4], StRegKill[4];
509  MachineInstr *DefiningMI;
510  const DebugLoc &DL = MI.getDebugLoc();
511  MachineBasicBlock &MBB = *MI.getParent();
514 
515  // If current instruction matches any of the rewriting rules, then
516  // gather information about parameters of the new instructions.
517  bool Match = false;
518  for (auto &I : IRT) {
519  if (MI.getOpcode() == I.OrigOpc) {
520  SeqReg = MI.getOperand(0).getReg();
521  AddrReg = MI.getOperand(1).getReg();
522  DefiningMI = MRI->getUniqueVRegDef(SeqReg);
523  unsigned NumReg = determineSrcReg(MI);
524  if (!processSeqRegInst(DefiningMI, StReg, StRegKill, NumReg))
525  return false;
526 
527  for (auto &Repl : I.ReplOpc) {
528  ReplInstrMCID.push_back(&TII->get(Repl));
529  // Generate destination registers but only for non-store instruction.
530  if (Repl != AArch64::STPQi && Repl != AArch64::STPDi)
531  ZipDest.push_back(MRI->createVirtualRegister(&I.RC));
532  }
533  Match = true;
534  break;
535  }
536  }
537 
538  if (!Match)
539  return false;
540 
541  // Determine if it is profitable to replace MI by the series of instructions
542  // represented in ReplInstrMCID.
543  if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()),
544  ReplInstrMCID))
545  return false;
546 
547  // Generate the replacement instructions composed of ZIP1, ZIP2, and STP (at
548  // this point, the code generation is hardcoded and does not rely on the IRT
549  // table used above given that code generation for ST2 replacement is somewhat
550  // different than for ST4 replacement. We could have added more info into the
551  // table related to how we build new instructions but we may be adding more
552  // complexity with that).
553  switch (MI.getOpcode()) {
554  default:
555  return false;
556 
557  case AArch64::ST2Twov16b:
558  case AArch64::ST2Twov8b:
559  case AArch64::ST2Twov8h:
560  case AArch64::ST2Twov4h:
561  case AArch64::ST2Twov4s:
562  case AArch64::ST2Twov2s:
563  case AArch64::ST2Twov2d:
564  // ZIP instructions
565  BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0])
566  .addReg(StReg[0])
567  .addReg(StReg[1]);
568  BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1])
569  .addReg(StReg[0], StRegKill[0])
570  .addReg(StReg[1], StRegKill[1]);
571  // STP instructions
572  BuildMI(MBB, MI, DL, *ReplInstrMCID[2])
573  .addReg(ZipDest[0])
574  .addReg(ZipDest[1])
575  .addReg(AddrReg)
576  .addImm(0);
577  break;
578 
579  case AArch64::ST4Fourv16b:
580  case AArch64::ST4Fourv8b:
581  case AArch64::ST4Fourv8h:
582  case AArch64::ST4Fourv4h:
583  case AArch64::ST4Fourv4s:
584  case AArch64::ST4Fourv2s:
585  case AArch64::ST4Fourv2d:
586  // ZIP instructions
587  BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0])
588  .addReg(StReg[0])
589  .addReg(StReg[2]);
590  BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1])
591  .addReg(StReg[0], StRegKill[0])
592  .addReg(StReg[2], StRegKill[2]);
593  BuildMI(MBB, MI, DL, *ReplInstrMCID[2], ZipDest[2])
594  .addReg(StReg[1])
595  .addReg(StReg[3]);
596  BuildMI(MBB, MI, DL, *ReplInstrMCID[3], ZipDest[3])
597  .addReg(StReg[1], StRegKill[1])
598  .addReg(StReg[3], StRegKill[3]);
599  BuildMI(MBB, MI, DL, *ReplInstrMCID[4], ZipDest[4])
600  .addReg(ZipDest[0])
601  .addReg(ZipDest[2]);
602  BuildMI(MBB, MI, DL, *ReplInstrMCID[5], ZipDest[5])
603  .addReg(ZipDest[0])
604  .addReg(ZipDest[2]);
605  BuildMI(MBB, MI, DL, *ReplInstrMCID[6], ZipDest[6])
606  .addReg(ZipDest[1])
607  .addReg(ZipDest[3]);
608  BuildMI(MBB, MI, DL, *ReplInstrMCID[7], ZipDest[7])
609  .addReg(ZipDest[1])
610  .addReg(ZipDest[3]);
611  // stp instructions
612  BuildMI(MBB, MI, DL, *ReplInstrMCID[8])
613  .addReg(ZipDest[4])
614  .addReg(ZipDest[5])
615  .addReg(AddrReg)
616  .addImm(0);
617  BuildMI(MBB, MI, DL, *ReplInstrMCID[9])
618  .addReg(ZipDest[6])
619  .addReg(ZipDest[7])
620  .addReg(AddrReg)
621  .addImm(2);
622  break;
623  }
624 
625  ++NumModifiedInstr;
626  return true;
627 }
628 
629 /// Process The REG_SEQUENCE instruction, and extract the source
630 /// operands of the ST2/4 instruction from it.
631 /// Example of such instruction.
632 /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;
633 /// Return true when the instruction is processed successfully.
634 bool AArch64SIMDInstrOpt::processSeqRegInst(MachineInstr *DefiningMI,
635  unsigned* StReg, unsigned* StRegKill, unsigned NumArg) const {
636  assert (DefiningMI != NULL);
637  if (DefiningMI->getOpcode() != AArch64::REG_SEQUENCE)
638  return false;
639 
640  for (unsigned i=0; i<NumArg; i++) {
641  StReg[i] = DefiningMI->getOperand(2*i+1).getReg();
642  StRegKill[i] = getKillRegState(DefiningMI->getOperand(2*i+1).isKill());
643 
644  // Sanity check for the other arguments.
645  if (DefiningMI->getOperand(2*i+2).isImm()) {
646  switch (DefiningMI->getOperand(2*i+2).getImm()) {
647  default:
648  return false;
649 
650  case AArch64::dsub0:
651  case AArch64::dsub1:
652  case AArch64::dsub2:
653  case AArch64::dsub3:
654  case AArch64::qsub0:
655  case AArch64::qsub1:
656  case AArch64::qsub2:
657  case AArch64::qsub3:
658  break;
659  }
660  }
661  else
662  return false;
663  }
664  return true;
665 }
666 
667 /// Return the number of useful source registers for this instruction
668 /// (2 for ST2 and 4 for ST4).
669 unsigned AArch64SIMDInstrOpt::determineSrcReg(MachineInstr &MI) const {
670  switch (MI.getOpcode()) {
671  default:
672  llvm_unreachable("Unsupported instruction for this pass");
673 
674  case AArch64::ST2Twov16b:
675  case AArch64::ST2Twov8b:
676  case AArch64::ST2Twov8h:
677  case AArch64::ST2Twov4h:
678  case AArch64::ST2Twov4s:
679  case AArch64::ST2Twov2s:
680  case AArch64::ST2Twov2d:
681  return 2;
682 
683  case AArch64::ST4Fourv16b:
684  case AArch64::ST4Fourv8b:
685  case AArch64::ST4Fourv8h:
686  case AArch64::ST4Fourv4h:
687  case AArch64::ST4Fourv4s:
688  case AArch64::ST4Fourv2s:
689  case AArch64::ST4Fourv2d:
690  return 4;
691  }
692 }
693 
694 bool AArch64SIMDInstrOpt::runOnMachineFunction(MachineFunction &MF) {
695  if (skipFunction(MF.getFunction()))
696  return false;
697 
698  TII = MF.getSubtarget().getInstrInfo();
699  MRI = &MF.getRegInfo();
700  const TargetSubtargetInfo &ST = MF.getSubtarget();
701  const AArch64InstrInfo *AAII =
702  static_cast<const AArch64InstrInfo *>(ST.getInstrInfo());
703  if (!AAII)
704  return false;
705  SchedModel.init(&ST);
706  if (!SchedModel.hasInstrSchedModel())
707  return false;
708 
709  bool Changed = false;
710  for (auto OptimizationKind : {VectorElem, Interleave}) {
711  if (!shouldExitEarly(&MF, OptimizationKind)) {
713  for (MachineBasicBlock &MBB : MF) {
714  for (MachineBasicBlock::iterator MII = MBB.begin(), MIE = MBB.end();
715  MII != MIE;) {
716  MachineInstr &MI = *MII;
717  bool InstRewrite;
718  if (OptimizationKind == VectorElem)
719  InstRewrite = optimizeVectElement(MI) ;
720  else
721  InstRewrite = optimizeLdStInterleave(MI);
722  if (InstRewrite) {
723  // Add MI to the list of instructions to be removed given that it
724  // has been replaced.
725  RemoveMIs.push_back(&MI);
726  Changed = true;
727  }
728  ++MII;
729  }
730  }
731  for (MachineInstr *MI : RemoveMIs)
732  MI->eraseFromParent();
733  }
734  }
735 
736  return Changed;
737 }
738 
739 /// Returns an instance of the high cost ASIMD instruction replacement
740 /// optimization pass.
742  return new AArch64SIMDInstrOpt();
743 }
i
i
Definition: README.txt:29
llvm::MCInstrDesc::getOpcode
unsigned getOpcode() const
Return the opcode number for this descriptor.
Definition: MCInstrDesc.h:221
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:105
MachineInstr.h
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
This file implements support for optimizing divisions by a constant.
Definition: AllocatorList.h:23
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:158
MCInstrDesc.h
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
StringRef.h
Pass.h
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:92
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
Statistic.h
llvm::MachineRegisterInfo::getUniqueVRegDef
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
Definition: MachineRegisterInfo.cpp:411
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
MachineBasicBlock.h
TargetInstrInfo.h
llvm::initializeAArch64SIMDInstrOptPass
void initializeAArch64SIMDInstrOptPass(PassRegistry &)
llvm::DiagnosticPredicateTy::Match
@ Match
llvm::createAArch64SIMDInstrOptPass
FunctionPass * createAArch64SIMDInstrOptPass()
Returns an instance of the high cost ASIMD instruction replacement optimization pass.
Definition: AArch64SIMDInstrOpt.cpp:741
llvm::MCSchedClassDesc::isValid
bool isValid() const
Definition: MCSchedule.h:127
MachineRegisterInfo.h
llvm::MachineOperand::isKill
bool isKill() const
Definition: MachineOperand.h:390
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:644
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
AArch64InstrInfo.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::AArch64InstrInfo
Definition: AArch64InstrInfo.h:38
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:537
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:499
llvm::MCSchedClassDesc
Summarize the scheduling resources required for an instruction of a particular scheduling class.
Definition: MCSchedule.h:109
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:195
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
RuleST4
#define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, OpcR8, OpcR9, RC)
Definition: AArch64SIMDInstrOpt.cpp:95
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:634
TargetSchedule.h
MCSchedule.h
llvm::TargetSchedModel
Provide an instruction scheduling machine model to CodeGen passes.
Definition: TargetSchedule.h:30
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
I
#define I(x, y, z)
Definition: MD5.cpp:59
MachineFunctionPass.h
INITIALIZE_PASS
INITIALIZE_PASS(AArch64SIMDInstrOpt, "aarch64-simdinstr-opt", AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false) bool AArch64SIMDInstrOpt
Based only on latency of instructions, determine if it is cost efficient to replace the instruction I...
Definition: AArch64SIMDInstrOpt.cpp:210
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:225
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:360
llvm::MachineFunction
Definition: MachineFunction.h:234
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:489
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:134
TargetSubtargetInfo.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::TargetSubtargetInfo
TargetSubtargetInfo - Generic base class for all target subtargets.
Definition: TargetSubtargetInfo.h:59
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:600
RuleST2
#define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC)
Definition: AArch64SIMDInstrOpt.cpp:93
llvm::MCSchedClassDesc::isVariant
bool isVariant() const
Definition: MCSchedule.h:130
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:585
llvm::MachineOperand::isImm
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Definition: MachineOperand.h:323
llvm::getKillRegState
unsigned getKillRegState(bool B)
Definition: MachineInstrBuilder.h:508
SmallVector.h
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:268
MachineInstrBuilder.h
AARCH64_VECTOR_BY_ELEMENT_OPT_NAME
#define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME
Definition: AArch64SIMDInstrOpt.cpp:61
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
llvm::MachineInstr::getNumOperands
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:492
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
MachineOperand.h
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
MachineFunction.h
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:270
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37