LLVM  17.0.0git
AArch64MacroFusion.cpp
Go to the documentation of this file.
1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file contains the AArch64 implementation of the DAG scheduling
10 /// mutation to pair instructions back to back.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64MacroFusion.h"
15 #include "AArch64Subtarget.h"
16 #include "llvm/CodeGen/MacroFusion.h"
17 #include "llvm/CodeGen/TargetInstrInfo.h"
18 
19 using namespace llvm;
20 
21 /// CMN, CMP, TST followed by Bcc
22 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
23  const MachineInstr &SecondMI, bool CmpOnly) {
24  if (SecondMI.getOpcode() != AArch64::Bcc)
25  return false;
26 
27  // Assume the 1st instr to be a wildcard if it is unspecified.
28  if (FirstMI == nullptr)
29  return true;
30 
31  // If we're in CmpOnly mode, we only fuse arithmetic instructions that
32  // discard their result.
33  if (CmpOnly && FirstMI->getOperand(0).isReg() &&
34  !(FirstMI->getOperand(0).getReg() == AArch64::XZR ||
35  FirstMI->getOperand(0).getReg() == AArch64::WZR)) {
36  return false;
37  }
38 
39  switch (FirstMI->getOpcode()) {
40  case AArch64::ADDSWri:
41  case AArch64::ADDSWrr:
42  case AArch64::ADDSXri:
43  case AArch64::ADDSXrr:
44  case AArch64::ANDSWri:
45  case AArch64::ANDSWrr:
46  case AArch64::ANDSXri:
47  case AArch64::ANDSXrr:
48  case AArch64::SUBSWri:
49  case AArch64::SUBSWrr:
50  case AArch64::SUBSXri:
51  case AArch64::SUBSXrr:
52  case AArch64::BICSWrr:
53  case AArch64::BICSXrr:
54  return true;
55  case AArch64::ADDSWrs:
56  case AArch64::ADDSXrs:
57  case AArch64::ANDSWrs:
58  case AArch64::ANDSXrs:
59  case AArch64::SUBSWrs:
60  case AArch64::SUBSXrs:
61  case AArch64::BICSWrs:
62  case AArch64::BICSXrs:
63  // Shift value can be 0 making these behave like the "rr" variant...
64  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
65  }
66 
67  return false;
68 }
69 
70 /// ALU operations followed by CBZ/CBNZ.
71 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
72  const MachineInstr &SecondMI) {
73  if (SecondMI.getOpcode() != AArch64::CBZW &&
74  SecondMI.getOpcode() != AArch64::CBZX &&
75  SecondMI.getOpcode() != AArch64::CBNZW &&
76  SecondMI.getOpcode() != AArch64::CBNZX)
77  return false;
78 
79  // Assume the 1st instr to be a wildcard if it is unspecified.
80  if (FirstMI == nullptr)
81  return true;
82 
83  switch (FirstMI->getOpcode()) {
84  case AArch64::ADDWri:
85  case AArch64::ADDWrr:
86  case AArch64::ADDXri:
87  case AArch64::ADDXrr:
88  case AArch64::ANDWri:
89  case AArch64::ANDWrr:
90  case AArch64::ANDXri:
91  case AArch64::ANDXrr:
92  case AArch64::EORWri:
93  case AArch64::EORWrr:
94  case AArch64::EORXri:
95  case AArch64::EORXrr:
96  case AArch64::ORRWri:
97  case AArch64::ORRWrr:
98  case AArch64::ORRXri:
99  case AArch64::ORRXrr:
100  case AArch64::SUBWri:
101  case AArch64::SUBWrr:
102  case AArch64::SUBXri:
103  case AArch64::SUBXrr:
104  return true;
105  case AArch64::ADDWrs:
106  case AArch64::ADDXrs:
107  case AArch64::ANDWrs:
108  case AArch64::ANDXrs:
109  case AArch64::SUBWrs:
110  case AArch64::SUBXrs:
111  case AArch64::BICWrs:
112  case AArch64::BICXrs:
113  // Shift value can be 0 making these behave like the "rr" variant...
114  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
115  }
116 
117  return false;
118 }
119 
120 /// AES crypto encoding or decoding.
121 static bool isAESPair(const MachineInstr *FirstMI,
122  const MachineInstr &SecondMI) {
123  // Assume the 1st instr to be a wildcard if it is unspecified.
124  switch (SecondMI.getOpcode()) {
125  // AES encode.
126  case AArch64::AESMCrr:
127  case AArch64::AESMCrrTied:
128  return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
129  // AES decode.
130  case AArch64::AESIMCrr:
131  case AArch64::AESIMCrrTied:
132  return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
133  }
134 
135  return false;
136 }
137 
138 /// AESE/AESD/PMULL + EOR.
139 static bool isCryptoEORPair(const MachineInstr *FirstMI,
140  const MachineInstr &SecondMI) {
141  if (SecondMI.getOpcode() != AArch64::EORv16i8)
142  return false;
143 
144  // Assume the 1st instr to be a wildcard if it is unspecified.
145  if (FirstMI == nullptr)
146  return true;
147 
148  switch (FirstMI->getOpcode()) {
149  case AArch64::AESErr:
150  case AArch64::AESDrr:
151  case AArch64::PMULLv16i8:
152  case AArch64::PMULLv8i8:
153  case AArch64::PMULLv1i64:
154  case AArch64::PMULLv2i64:
155  return true;
156  }
157 
158  return false;
159 }
160 
161 static bool isAdrpAddPair(const MachineInstr *FirstMI,
162  const MachineInstr &SecondMI) {
163  // Assume the 1st instr to be a wildcard if it is unspecified.
164  if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
165  SecondMI.getOpcode() == AArch64::ADDXri)
166  return true;
167  return false;
168 }
169 
170 /// Literal generation.
171 static bool isLiteralsPair(const MachineInstr *FirstMI,
172  const MachineInstr &SecondMI) {
173  // Assume the 1st instr to be a wildcard if it is unspecified.
174  // 32 bit immediate.
175  if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
176  (SecondMI.getOpcode() == AArch64::MOVKWi &&
177  SecondMI.getOperand(3).getImm() == 16))
178  return true;
179 
180  // Lower half of 64 bit immediate.
181  if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
182  (SecondMI.getOpcode() == AArch64::MOVKXi &&
183  SecondMI.getOperand(3).getImm() == 16))
184  return true;
185 
186  // Upper half of 64 bit immediate.
187  if ((FirstMI == nullptr ||
188  (FirstMI->getOpcode() == AArch64::MOVKXi &&
189  FirstMI->getOperand(3).getImm() == 32)) &&
190  (SecondMI.getOpcode() == AArch64::MOVKXi &&
191  SecondMI.getOperand(3).getImm() == 48))
192  return true;
193 
194  return false;
195 }
196 
197 /// Fuse address generation and loads or stores.
198 static bool isAddressLdStPair(const MachineInstr *FirstMI,
199  const MachineInstr &SecondMI) {
200  switch (SecondMI.getOpcode()) {
201  case AArch64::STRBBui:
202  case AArch64::STRBui:
203  case AArch64::STRDui:
204  case AArch64::STRHHui:
205  case AArch64::STRHui:
206  case AArch64::STRQui:
207  case AArch64::STRSui:
208  case AArch64::STRWui:
209  case AArch64::STRXui:
210  case AArch64::LDRBBui:
211  case AArch64::LDRBui:
212  case AArch64::LDRDui:
213  case AArch64::LDRHHui:
214  case AArch64::LDRHui:
215  case AArch64::LDRQui:
216  case AArch64::LDRSui:
217  case AArch64::LDRWui:
218  case AArch64::LDRXui:
219  case AArch64::LDRSBWui:
220  case AArch64::LDRSBXui:
221  case AArch64::LDRSHWui:
222  case AArch64::LDRSHXui:
223  case AArch64::LDRSWui:
224  // Assume the 1st instr to be a wildcard if it is unspecified.
225  if (FirstMI == nullptr)
226  return true;
227 
228  switch (FirstMI->getOpcode()) {
229  case AArch64::ADR:
230  return SecondMI.getOperand(2).getImm() == 0;
231  case AArch64::ADRP:
232  return true;
233  }
234  }
235 
236  return false;
237 }
238 
239 /// Compare and conditional select.
240 static bool isCCSelectPair(const MachineInstr *FirstMI,
241  const MachineInstr &SecondMI) {
242  // 32 bits
243  if (SecondMI.getOpcode() == AArch64::CSELWr) {
244  // Assume the 1st instr to be a wildcard if it is unspecified.
245  if (FirstMI == nullptr)
246  return true;
247 
248  if (FirstMI->definesRegister(AArch64::WZR))
249  switch (FirstMI->getOpcode()) {
250  case AArch64::SUBSWrs:
251  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
252  case AArch64::SUBSWrx:
253  return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
254  case AArch64::SUBSWrr:
255  case AArch64::SUBSWri:
256  return true;
257  }
258  }
259 
260  // 64 bits
261  if (SecondMI.getOpcode() == AArch64::CSELXr) {
262  // Assume the 1st instr to be a wildcard if it is unspecified.
263  if (FirstMI == nullptr)
264  return true;
265 
266  if (FirstMI->definesRegister(AArch64::XZR))
267  switch (FirstMI->getOpcode()) {
268  case AArch64::SUBSXrs:
269  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
270  case AArch64::SUBSXrx:
271  case AArch64::SUBSXrx64:
272  return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
273  case AArch64::SUBSXrr:
274  case AArch64::SUBSXri:
275  return true;
276  }
277  }
278 
279  return false;
280 }
281 
282 // Arithmetic and logic.
283 static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
284  const MachineInstr &SecondMI) {
285  if (AArch64InstrInfo::hasShiftedReg(SecondMI))
286  return false;
287 
288  switch (SecondMI.getOpcode()) {
289  // Arithmetic
290  case AArch64::ADDWrr:
291  case AArch64::ADDXrr:
292  case AArch64::SUBWrr:
293  case AArch64::SUBXrr:
294  case AArch64::ADDWrs:
295  case AArch64::ADDXrs:
296  case AArch64::SUBWrs:
297  case AArch64::SUBXrs:
298  // Logic
299  case AArch64::ANDWrr:
300  case AArch64::ANDXrr:
301  case AArch64::BICWrr:
302  case AArch64::BICXrr:
303  case AArch64::EONWrr:
304  case AArch64::EONXrr:
305  case AArch64::EORWrr:
306  case AArch64::EORXrr:
307  case AArch64::ORNWrr:
308  case AArch64::ORNXrr:
309  case AArch64::ORRWrr:
310  case AArch64::ORRXrr:
311  case AArch64::ANDWrs:
312  case AArch64::ANDXrs:
313  case AArch64::BICWrs:
314  case AArch64::BICXrs:
315  case AArch64::EONWrs:
316  case AArch64::EONXrs:
317  case AArch64::EORWrs:
318  case AArch64::EORXrs:
319  case AArch64::ORNWrs:
320  case AArch64::ORNXrs:
321  case AArch64::ORRWrs:
322  case AArch64::ORRXrs:
323  // Assume the 1st instr to be a wildcard if it is unspecified.
324  if (FirstMI == nullptr)
325  return true;
326 
327  // Arithmetic
328  switch (FirstMI->getOpcode()) {
329  case AArch64::ADDWrr:
330  case AArch64::ADDXrr:
331  case AArch64::ADDSWrr:
332  case AArch64::ADDSXrr:
333  case AArch64::SUBWrr:
334  case AArch64::SUBXrr:
335  case AArch64::SUBSWrr:
336  case AArch64::SUBSXrr:
337  return true;
338  case AArch64::ADDWrs:
339  case AArch64::ADDXrs:
340  case AArch64::ADDSWrs:
341  case AArch64::ADDSXrs:
342  case AArch64::SUBWrs:
343  case AArch64::SUBXrs:
344  case AArch64::SUBSWrs:
345  case AArch64::SUBSXrs:
346  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
347  }
348  break;
349 
350  // Arithmetic, setting flags.
351  case AArch64::ADDSWrr:
352  case AArch64::ADDSXrr:
353  case AArch64::SUBSWrr:
354  case AArch64::SUBSXrr:
355  case AArch64::ADDSWrs:
356  case AArch64::ADDSXrs:
357  case AArch64::SUBSWrs:
358  case AArch64::SUBSXrs:
359  // Assume the 1st instr to be a wildcard if it is unspecified.
360  if (FirstMI == nullptr)
361  return true;
362 
363  // Arithmetic, not setting flags.
364  switch (FirstMI->getOpcode()) {
365  case AArch64::ADDWrr:
366  case AArch64::ADDXrr:
367  case AArch64::SUBWrr:
368  case AArch64::SUBXrr:
369  return true;
370  case AArch64::ADDWrs:
371  case AArch64::ADDXrs:
372  case AArch64::SUBWrs:
373  case AArch64::SUBXrs:
374  return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
375  }
376  break;
377  }
378 
379  return false;
380 }
381 
382 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
383 /// together. Given SecondMI, when FirstMI is unspecified, then check if
384 /// SecondMI may be part of a fused pair at all.
385 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
386  const TargetSubtargetInfo &TSI,
387  const MachineInstr *FirstMI,
388  const MachineInstr &SecondMI) {
389  const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
390 
391  // All checking functions assume that the 1st instr is a wildcard if it is
392  // unspecified.
393  if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
394  bool CmpOnly = !ST.hasArithmeticBccFusion();
395  if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly))
396  return true;
397  }
398  if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
399  return true;
400  if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
401  return true;
402  if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
403  return true;
404  if (ST.hasFuseAdrpAdd() && isAdrpAddPair(FirstMI, SecondMI))
405  return true;
406  if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
407  return true;
408  if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
409  return true;
410  if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
411  return true;
412  if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
413  return true;
414 
415  return false;
416 }
417 
418 std::unique_ptr<ScheduleDAGMutation>
419 llvm::createAArch64MacroFusionDAGMutation() {
420  return createMacroFusionDAGMutation(shouldScheduleAdjacent);
421 }
isArithmeticBccPair
static bool isArithmeticBccPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI, bool CmpOnly)
CMN, CMP, TST followed by Bcc.
Definition: AArch64MacroFusion.cpp:22
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
isAddressLdStPair
static bool isAddressLdStPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Fuse address generation and loads or stores.
Definition: AArch64MacroFusion.cpp:198
isArithmeticCbzPair
static bool isArithmeticCbzPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
ALU operations followed by CBZ/CBNZ.
Definition: AArch64MacroFusion.cpp:71
TargetInstrInfo.h
isArithmeticLogicPair
static bool isArithmeticLogicPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Definition: AArch64MacroFusion.cpp:283
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:98
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:553
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
llvm::isAESPair
static bool isAESPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Definition: ARMMacroFusion.cpp:22
isAdrpAddPair
static bool isAdrpAddPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Definition: AArch64MacroFusion.cpp:161
llvm::MachineInstr::definesRegister
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr fully defines the specified register.
Definition: MachineInstr.h:1428
llvm::createMacroFusionDAGMutation
std::unique_ptr< ScheduleDAGMutation > createMacroFusionDAGMutation(ShouldSchedulePredTy shouldScheduleAdjacent)
Create a DAG scheduling mutation to pair instructions back to back for instructions that benefit acco...
Definition: MacroFusion.cpp:200
AArch64MacroFusion.h
llvm::shouldScheduleAdjacent
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, const TargetSubtargetInfo &TSI, const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Check if the instr pair, FirstMI and SecondMI, should be fused together.
Definition: ARMMacroFusion.cpp:51
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:326
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
MacroFusion.h
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:366
llvm::isLiteralsPair
static bool isLiteralsPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Definition: ARMMacroFusion.cpp:38
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:516
llvm::AArch64ISD::ADR
@ ADR
Definition: AArch64ISelLowering.h:76
llvm::TargetSubtargetInfo
TargetSubtargetInfo - Generic base class for all target subtargets.
Definition: TargetSubtargetInfo.h:62
isCryptoEORPair
static bool isCryptoEORPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
AESE/AESD/PMULL + EOR.
Definition: AArch64MacroFusion.cpp:139
llvm::createAArch64MacroFusionDAGMutation
std::unique_ptr< ScheduleDAGMutation > createAArch64MacroFusionDAGMutation()
Note that you have to add: DAG.addMutation(createAArch64MacroFusionDAGMutation()); to AArch64PassConf...
Definition: AArch64MacroFusion.cpp:419
llvm::AArch64ISD::ADRP
@ ADRP
Definition: AArch64ISelLowering.h:75
isCCSelectPair
static bool isCCSelectPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)
Compare and conditional select.
Definition: AArch64MacroFusion.cpp:240
AArch64Subtarget.h
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
Generated on Sat Jan 28 2023 09:53:47 for LLVM by   doxygen 1.8.17