LLVM  15.0.0git
AArch64MacroFusion.cpp
Go to the documentation of this file.
1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file contains the AArch64 implementation of the DAG scheduling
10 /// mutation to pair instructions back to back.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64MacroFusion.h"
15 #include "AArch64Subtarget.h"
16 #include "llvm/CodeGen/MacroFusion.h"
17 #include "llvm/CodeGen/TargetInstrInfo.h"
18 
19 using namespace llvm;
20 
21 /// CMN, CMP, TST followed by Bcc
22 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
23  const MachineInstr &SecondMI, bool CmpOnly) {
24  if (SecondMI.getOpcode() != AArch64::Bcc)
25  return false;
26 
27  // Assume the 1st instr to be a wildcard if it is unspecified.
28  if (FirstMI == nullptr)
29  return true;
30 
31  // If we're in CmpOnly mode, we only fuse arithmetic instructions that
32  // discard their result.
33  if (CmpOnly && !(FirstMI->getOperand(0).getReg() == AArch64::XZR ||
34  FirstMI->getOperand(0).getReg() == AArch64::WZR)) {
35  return false;
36  }
37 
38  switch (FirstMI->getOpcode()) {
39  case AArch64::ADDSWri:
40  case AArch64::ADDSWrr:
41  case AArch64::ADDSXri:
42  case AArch64::ADDSXrr:
43  case AArch64::ANDSWri:
44  case AArch64::ANDSWrr:
45  case AArch64::ANDSXri:
46  case AArch64::ANDSXrr:
47  case AArch64::SUBSWri:
48  case AArch64::SUBSWrr:
49  case AArch64::SUBSXri:
50  case AArch64::SUBSXrr:
51  case AArch64::BICSWrr:
52  case AArch64::BICSXrr:
53  return true;
54  case AArch64::ADDSWrs:
55  case AArch64::ADDSXrs:
56  case AArch64::ANDSWrs:
57  case AArch64::ANDSXrs:
58  case AArch64::SUBSWrs:
59  case AArch64::SUBSXrs:
60  case AArch64::BICSWrs:
61  case AArch64::BICSXrs:
62  // Shift value can be 0 making these behave like the "rr" variant...
63  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
64  }
65 
66  return false;
67 }
68 
69 /// ALU operations followed by CBZ/CBNZ.
70 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
71  const MachineInstr &SecondMI) {
72  if (SecondMI.getOpcode() != AArch64::CBZW &&
73  SecondMI.getOpcode() != AArch64::CBZX &&
74  SecondMI.getOpcode() != AArch64::CBNZW &&
75  SecondMI.getOpcode() != AArch64::CBNZX)
76  return false;
77 
78  // Assume the 1st instr to be a wildcard if it is unspecified.
79  if (FirstMI == nullptr)
80  return true;
81 
82  switch (FirstMI->getOpcode()) {
83  case AArch64::ADDWri:
84  case AArch64::ADDWrr:
85  case AArch64::ADDXri:
86  case AArch64::ADDXrr:
87  case AArch64::ANDWri:
88  case AArch64::ANDWrr:
89  case AArch64::ANDXri:
90  case AArch64::ANDXrr:
91  case AArch64::EORWri:
92  case AArch64::EORWrr:
93  case AArch64::EORXri:
94  case AArch64::EORXrr:
95  case AArch64::ORRWri:
96  case AArch64::ORRWrr:
97  case AArch64::ORRXri:
98  case AArch64::ORRXrr:
99  case AArch64::SUBWri:
100  case AArch64::SUBWrr:
101  case AArch64::SUBXri:
102  case AArch64::SUBXrr:
103  return true;
104  case AArch64::ADDWrs:
105  case AArch64::ADDXrs:
106  case AArch64::ANDWrs:
107  case AArch64::ANDXrs:
108  case AArch64::SUBWrs:
109  case AArch64::SUBXrs:
110  case AArch64::BICWrs:
111  case AArch64::BICXrs:
112  // Shift value can be 0 making these behave like the "rr" variant...
113  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
114  }
115 
116  return false;
117 }
118 
119 /// AES crypto encoding or decoding.
120 static bool isAESPair(const MachineInstr *FirstMI,
121  const MachineInstr &SecondMI) {
122  // Assume the 1st instr to be a wildcard if it is unspecified.
123  switch (SecondMI.getOpcode()) {
124  // AES encode.
125  case AArch64::AESMCrr:
126  case AArch64::AESMCrrTied:
127  return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
128  // AES decode.
129  case AArch64::AESIMCrr:
130  case AArch64::AESIMCrrTied:
131  return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
132  }
133 
134  return false;
135 }
136 
137 /// AESE/AESD/PMULL + EOR.
138 static bool isCryptoEORPair(const MachineInstr *FirstMI,
139  const MachineInstr &SecondMI) {
140  if (SecondMI.getOpcode() != AArch64::EORv16i8)
141  return false;
142 
143  // Assume the 1st instr to be a wildcard if it is unspecified.
144  if (FirstMI == nullptr)
145  return true;
146 
147  switch (FirstMI->getOpcode()) {
148  case AArch64::AESErr:
149  case AArch64::AESDrr:
150  case AArch64::PMULLv16i8:
151  case AArch64::PMULLv8i8:
152  case AArch64::PMULLv1i64:
153  case AArch64::PMULLv2i64:
154  return true;
155  }
156 
157  return false;
158 }
159 
160 static bool isAdrpAddPair(const MachineInstr *FirstMI,
161  const MachineInstr &SecondMI) {
162  // Assume the 1st instr to be a wildcard if it is unspecified.
163  if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
164  SecondMI.getOpcode() == AArch64::ADDXri)
165  return true;
166  return false;
167 }
168 
169 /// Literal generation.
170 static bool isLiteralsPair(const MachineInstr *FirstMI,
171  const MachineInstr &SecondMI) {
172  // Assume the 1st instr to be a wildcard if it is unspecified.
173  // 32 bit immediate.
174  if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
175  (SecondMI.getOpcode() == AArch64::MOVKWi &&
176  SecondMI.getOperand(3).getImm() == 16))
177  return true;
178 
179  // Lower half of 64 bit immediate.
180  if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
181  (SecondMI.getOpcode() == AArch64::MOVKXi &&
182  SecondMI.getOperand(3).getImm() == 16))
183  return true;
184 
185  // Upper half of 64 bit immediate.
186  if ((FirstMI == nullptr ||
187  (FirstMI->getOpcode() == AArch64::MOVKXi &&
188  FirstMI->getOperand(3).getImm() == 32)) &&
189  (SecondMI.getOpcode() == AArch64::MOVKXi &&
190  SecondMI.getOperand(3).getImm() == 48))
191  return true;
192 
193  return false;
194 }
195 
196 /// Fuse address generation and loads or stores.
197 static bool isAddressLdStPair(const MachineInstr *FirstMI,
198  const MachineInstr &SecondMI) {
199  switch (SecondMI.getOpcode()) {
200  case AArch64::STRBBui:
201  case AArch64::STRBui:
202  case AArch64::STRDui:
203  case AArch64::STRHHui:
204  case AArch64::STRHui:
205  case AArch64::STRQui:
206  case AArch64::STRSui:
207  case AArch64::STRWui:
208  case AArch64::STRXui:
209  case AArch64::LDRBBui:
210  case AArch64::LDRBui:
211  case AArch64::LDRDui:
212  case AArch64::LDRHHui:
213  case AArch64::LDRHui:
214  case AArch64::LDRQui:
215  case AArch64::LDRSui:
216  case AArch64::LDRWui:
217  case AArch64::LDRXui:
218  case AArch64::LDRSBWui:
219  case AArch64::LDRSBXui:
220  case AArch64::LDRSHWui:
221  case AArch64::LDRSHXui:
222  case AArch64::LDRSWui:
223  // Assume the 1st instr to be a wildcard if it is unspecified.
224  if (FirstMI == nullptr)
225  return true;
226 
227  switch (FirstMI->getOpcode()) {
228  case AArch64::ADR:
229  return SecondMI.getOperand(2).getImm() == 0;
230  case AArch64::ADRP:
231  return true;
232  }
233  }
234 
235  return false;
236 }
237 
238 /// Compare and conditional select.
239 static bool isCCSelectPair(const MachineInstr *FirstMI,
240  const MachineInstr &SecondMI) {
241  // 32 bits
242  if (SecondMI.getOpcode() == AArch64::CSELWr) {
243  // Assume the 1st instr to be a wildcard if it is unspecified.
244  if (FirstMI == nullptr)
245  return true;
246 
247  if (FirstMI->definesRegister(AArch64::WZR))
248  switch (FirstMI->getOpcode()) {
249  case AArch64::SUBSWrs:
250  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
251  case AArch64::SUBSWrx:
252  return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
253  case AArch64::SUBSWrr:
254  case AArch64::SUBSWri:
255  return true;
256  }
257  }
258 
259  // 64 bits
260  if (SecondMI.getOpcode() == AArch64::CSELXr) {
261  // Assume the 1st instr to be a wildcard if it is unspecified.
262  if (FirstMI == nullptr)
263  return true;
264 
265  if (FirstMI->definesRegister(AArch64::XZR))
266  switch (FirstMI->getOpcode()) {
267  case AArch64::SUBSXrs:
268  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
269  case AArch64::SUBSXrx:
270  case AArch64::SUBSXrx64:
271  return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
272  case AArch64::SUBSXrr:
273  case AArch64::SUBSXri:
274  return true;
275  }
276  }
277 
278  return false;
279 }
280 
281 // Arithmetic and logic.
282 static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
283  const MachineInstr &SecondMI) {
284  if (AArch64InstrInfo::hasShiftedReg(SecondMI))
285  return false;
286 
287  switch (SecondMI.getOpcode()) {
288  // Arithmetic
289  case AArch64::ADDWrr:
290  case AArch64::ADDXrr:
291  case AArch64::SUBWrr:
292  case AArch64::SUBXrr:
293  case AArch64::ADDWrs:
294  case AArch64::ADDXrs:
295  case AArch64::SUBWrs:
296  case AArch64::SUBXrs:
297  // Logic
298  case AArch64::ANDWrr:
299  case AArch64::ANDXrr:
300  case AArch64::BICWrr:
301  case AArch64::BICXrr:
302  case AArch64::EONWrr:
303  case AArch64::EONXrr:
304  case AArch64::EORWrr:
305  case AArch64::EORXrr:
306  case AArch64::ORNWrr:
307  case AArch64::ORNXrr:
308  case AArch64::ORRWrr:
309  case AArch64::ORRXrr:
310  case AArch64::ANDWrs:
311  case AArch64::ANDXrs:
312  case AArch64::BICWrs:
313  case AArch64::BICXrs:
314  case AArch64::EONWrs:
315  case AArch64::EONXrs:
316  case AArch64::EORWrs:
317  case AArch64::EORXrs:
318  case AArch64::ORNWrs:
319  case AArch64::ORNXrs:
320  case AArch64::ORRWrs:
321  case AArch64::ORRXrs:
322  // Assume the 1st instr to be a wildcard if it is unspecified.
323  if (FirstMI == nullptr)
324  return true;
325 
326  // Arithmetic
327  switch (FirstMI->getOpcode()) {
328  case AArch64::ADDWrr:
329  case AArch64::ADDXrr:
330  case AArch64::ADDSWrr:
331  case AArch64::ADDSXrr:
332  case AArch64::SUBWrr:
333  case AArch64::SUBXrr:
334  case AArch64::SUBSWrr:
335  case AArch64::SUBSXrr:
336  return true;
337  case AArch64::ADDWrs:
338  case AArch64::ADDXrs:
339  case AArch64::ADDSWrs:
340  case AArch64::ADDSXrs:
341  case AArch64::SUBWrs:
342  case AArch64::SUBXrs:
343  case AArch64::SUBSWrs:
344  case AArch64::SUBSXrs:
345  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
346  }
347  break;
348 
349  // Arithmetic, setting flags.
350  case AArch64::ADDSWrr:
351  case AArch64::ADDSXrr:
352  case AArch64::SUBSWrr:
353  case AArch64::SUBSXrr:
354  case AArch64::ADDSWrs:
355  case AArch64::ADDSXrs:
356  case AArch64::SUBSWrs:
357  case AArch64::SUBSXrs:
358  // Assume the 1st instr to be a wildcard if it is unspecified.
359  if (FirstMI == nullptr)
360  return true;
361 
362  // Arithmetic, not setting flags.
363  switch (FirstMI->getOpcode()) {
364  case AArch64::ADDWrr:
365  case AArch64::ADDXrr:
366  case AArch64::SUBWrr:
367  case AArch64::SUBXrr:
368  return true;
369  case AArch64::ADDWrs:
370  case AArch64::ADDXrs:
371  case AArch64::SUBWrs:
372  case AArch64::SUBXrs:
373  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
374  }
375  break;
376  }
377 
378  return false;
379 }
380 
381 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
382 /// together. Given SecondMI, when FirstMI is unspecified, then check if
383 /// SecondMI may be part of a fused pair at all.
384 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
385  const TargetSubtargetInfo &TSI,
386  const MachineInstr *FirstMI,
387  const MachineInstr &SecondMI) {
388  const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
389 
390  // All checking functions assume that the 1st instr is a wildcard if it is
391  // unspecified.
392  if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
393  bool CmpOnly = !ST.hasArithmeticBccFusion();
394  if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly))
395  return true;
396  }
397  if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
398  return true;
399  if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
400  return true;
401  if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
402  return true;
403  if (ST.hasFuseAdrpAdd() && isAdrpAddPair(FirstMI, SecondMI))
404  return true;
405  if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
406  return true;
407  if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
408  return true;
409  if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
410  return true;
411  if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
412  return true;
413 
414  return false;
415 }
416 
417 std::unique_ptr<ScheduleDAGMutation>
418 llvm::createAArch64MacroFusionDAGMutation() {
419  return createMacroFusionDAGMutation(shouldScheduleAdjacent);
420 }
isArithmeticBccPair
static bool isArithmeticBccPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI, bool CmpOnly)
CMN, CMP, TST followed by Bcc.
Definition: AArch64MacroFusion.cpp:22
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
isAddressLdStPair
static bool isAddressLdStPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Fuse address generation and loads or stores.
Definition: AArch64MacroFusion.cpp:197
isArithmeticCbzPair
static bool isArithmeticCbzPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
ALU operations followed by CBZ/CBNZ.
Definition: AArch64MacroFusion.cpp:70
TargetInstrInfo.h
isArithmeticLogicPair
static bool isArithmeticLogicPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Definition: AArch64MacroFusion.cpp:282
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:546
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:501
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
llvm::isAESPair
static bool isAESPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Definition: ARMMacroFusion.cpp:22
isAdrpAddPair
static bool isAdrpAddPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Definition: AArch64MacroFusion.cpp:160
llvm::MachineInstr::definesRegister
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr fully defines the specified register.
Definition: MachineInstr.h:1376
llvm::createMacroFusionDAGMutation
std::unique_ptr< ScheduleDAGMutation > createMacroFusionDAGMutation(ShouldSchedulePredTy shouldScheduleAdjacent)
Create a DAG scheduling mutation to pair instructions back to back for instructions that benefit acco...
Definition: MacroFusion.cpp:200
AArch64MacroFusion.h
llvm::shouldScheduleAdjacent
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, const TargetSubtargetInfo &TSI, const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Check if the instr pair, FirstMI and SecondMI, should be fused together.
Definition: ARMMacroFusion.cpp:51
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
MacroFusion.h
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:359
llvm::isLiteralsPair
static bool isLiteralsPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Definition: ARMMacroFusion.cpp:38
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:491
llvm::AArch64ISD::ADR
@ ADR
Definition: AArch64ISelLowering.h:64
llvm::TargetSubtargetInfo
TargetSubtargetInfo - Generic base class for all target subtargets.
Definition: TargetSubtargetInfo.h:60
isCryptoEORPair
static bool isCryptoEORPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
AESE/AESD/PMULL + EOR.
Definition: AArch64MacroFusion.cpp:138
llvm::createAArch64MacroFusionDAGMutation
std::unique_ptr< ScheduleDAGMutation > createAArch64MacroFusionDAGMutation()
Note that you have to add: DAG.addMutation(createAArch64MacroFusionDAGMutation()); to AArch64PassConf...
Definition: AArch64MacroFusion.cpp:418
llvm::AArch64ISD::ADRP
@ ADRP
Definition: AArch64ISelLowering.h:63
isCCSelectPair
static bool isCCSelectPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Compare and conditional select.
Definition: AArch64MacroFusion.cpp:239
AArch64Subtarget.h
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
Generated on Wed Jun 29 2022 17:45:20 for LLVM by   doxygen 1.8.17