Line data Source code
1 : //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : /// \file This file contains the AArch64 implementation of the DAG scheduling
11 : /// mutation to pair instructions back to back.
12 : //
13 : //===----------------------------------------------------------------------===//
14 :
15 : #include "AArch64Subtarget.h"
16 : #include "llvm/CodeGen/MacroFusion.h"
17 : #include "llvm/CodeGen/TargetInstrInfo.h"
18 :
19 : using namespace llvm;
20 :
21 : namespace {
22 :
23 : /// CMN, CMP, TST followed by Bcc
24 3359 : static bool isArithmeticBccPair(const MachineInstr *FirstMI,
25 : const MachineInstr &SecondMI) {
26 6718 : if (SecondMI.getOpcode() != AArch64::Bcc)
27 : return false;
28 :
29 : // Assume the 1st instr to be a wildcard if it is unspecified.
30 90 : if (FirstMI == nullptr)
31 : return true;
32 :
33 90 : switch (FirstMI->getOpcode()) {
34 : case AArch64::ADDSWri:
35 : case AArch64::ADDSWrr:
36 : case AArch64::ADDSXri:
37 : case AArch64::ADDSXrr:
38 : case AArch64::ANDSWri:
39 : case AArch64::ANDSWrr:
40 : case AArch64::ANDSXri:
41 : case AArch64::ANDSXrr:
42 : case AArch64::SUBSWri:
43 : case AArch64::SUBSWrr:
44 : case AArch64::SUBSXri:
45 : case AArch64::SUBSXrr:
46 : case AArch64::BICSWrr:
47 : case AArch64::BICSXrr:
48 : return true;
49 0 : case AArch64::ADDSWrs:
50 : case AArch64::ADDSXrs:
51 : case AArch64::ANDSWrs:
52 : case AArch64::ANDSXrs:
53 : case AArch64::SUBSWrs:
54 : case AArch64::SUBSXrs:
55 : case AArch64::BICSWrs:
56 : case AArch64::BICSXrs:
57 : // Shift value can be 0 making these behave like the "rr" variant...
58 0 : return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
59 : }
60 :
61 9 : return false;
62 : }
63 :
64 : /// ALU operations followed by CBZ/CBNZ.
65 3259 : static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
66 : const MachineInstr &SecondMI) {
67 3259 : if (SecondMI.getOpcode() != AArch64::CBZW &&
68 3249 : SecondMI.getOpcode() != AArch64::CBZX &&
69 6451 : SecondMI.getOpcode() != AArch64::CBNZW &&
70 : SecondMI.getOpcode() != AArch64::CBNZX)
71 : return false;
72 :
73 : // Assume the 1st instr to be a wildcard if it is unspecified.
74 71 : if (FirstMI == nullptr)
75 : return true;
76 :
77 74 : switch (FirstMI->getOpcode()) {
78 : case AArch64::ADDWri:
79 : case AArch64::ADDWrr:
80 : case AArch64::ADDXri:
81 : case AArch64::ADDXrr:
82 : case AArch64::ANDWri:
83 : case AArch64::ANDWrr:
84 : case AArch64::ANDXri:
85 : case AArch64::ANDXrr:
86 : case AArch64::EORWri:
87 : case AArch64::EORWrr:
88 : case AArch64::EORXri:
89 : case AArch64::EORXrr:
90 : case AArch64::ORRWri:
91 : case AArch64::ORRWrr:
92 : case AArch64::ORRXri:
93 : case AArch64::ORRXrr:
94 : case AArch64::SUBWri:
95 : case AArch64::SUBWrr:
96 : case AArch64::SUBXri:
97 : case AArch64::SUBXrr:
98 : return true;
99 0 : case AArch64::ADDWrs:
100 : case AArch64::ADDXrs:
101 : case AArch64::ANDWrs:
102 : case AArch64::ANDXrs:
103 : case AArch64::SUBWrs:
104 : case AArch64::SUBXrs:
105 : case AArch64::BICWrs:
106 : case AArch64::BICXrs:
107 : // Shift value can be 0 making these behave like the "rr" variant...
108 0 : return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
109 : }
110 :
111 33 : return false;
112 : }
113 :
114 : /// AES crypto encoding or decoding.
115 132640 : static bool isAESPair(const MachineInstr *FirstMI,
116 : const MachineInstr &SecondMI) {
117 : // Assume the 1st instr to be a wildcard if it is unspecified.
118 265280 : switch (SecondMI.getOpcode()) {
119 : // AES encode.
120 366 : case AArch64::AESMCrr:
121 : case AArch64::AESMCrrTied:
122 366 : return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
123 : // AES decode.
124 294 : case AArch64::AESIMCrr:
125 : case AArch64::AESIMCrrTied:
126 294 : return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
127 : }
128 :
129 : return false;
130 : }
131 :
132 : /// AESE/AESD/PMULL + EOR.
133 : static bool isCryptoEORPair(const MachineInstr *FirstMI,
134 : const MachineInstr &SecondMI) {
135 6364 : if (SecondMI.getOpcode() != AArch64::EORv16i8)
136 : return false;
137 :
138 : // Assume the 1st instr to be a wildcard if it is unspecified.
139 0 : if (FirstMI == nullptr)
140 : return true;
141 :
142 0 : switch (FirstMI->getOpcode()) {
143 : case AArch64::AESErr:
144 : case AArch64::AESDrr:
145 : case AArch64::PMULLv16i8:
146 : case AArch64::PMULLv8i8:
147 : case AArch64::PMULLv1i64:
148 : case AArch64::PMULLv2i64:
149 : return true;
150 : }
151 :
152 : return false;
153 : }
154 :
155 : /// Literal generation.
156 4655 : static bool isLiteralsPair(const MachineInstr *FirstMI,
157 : const MachineInstr &SecondMI) {
158 : // Assume the 1st instr to be a wildcard if it is unspecified.
159 :
160 : // PC relative address.
161 4655 : if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
162 4377 : SecondMI.getOpcode() == AArch64::ADDXri)
163 : return true;
164 :
165 : // 32 bit immediate.
166 4613 : if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
167 4277 : (SecondMI.getOpcode() == AArch64::MOVKWi &&
168 8 : SecondMI.getOperand(3).getImm() == 16))
169 : return true;
170 :
171 : // Lower half of 64 bit immediate.
172 4605 : if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
173 4273 : (SecondMI.getOpcode() == AArch64::MOVKXi &&
174 16 : SecondMI.getOperand(3).getImm() == 16))
175 : return true;
176 :
177 : // Upper half of 64 bit immediate.
178 332 : if ((FirstMI == nullptr ||
179 332 : (FirstMI->getOpcode() == AArch64::MOVKXi &&
180 4601 : FirstMI->getOperand(3).getImm() == 32)) &&
181 4269 : (SecondMI.getOpcode() == AArch64::MOVKXi &&
182 12 : SecondMI.getOperand(3).getImm() == 48))
183 8 : return true;
184 :
185 : return false;
186 : }
187 :
188 : /// Fuse address generation and loads or stores.
189 3102 : static bool isAddressLdStPair(const MachineInstr *FirstMI,
190 : const MachineInstr &SecondMI) {
191 6204 : switch (SecondMI.getOpcode()) {
192 531 : case AArch64::STRBBui:
193 : case AArch64::STRBui:
194 : case AArch64::STRDui:
195 : case AArch64::STRHHui:
196 : case AArch64::STRHui:
197 : case AArch64::STRQui:
198 : case AArch64::STRSui:
199 : case AArch64::STRWui:
200 : case AArch64::STRXui:
201 : case AArch64::LDRBBui:
202 : case AArch64::LDRBui:
203 : case AArch64::LDRDui:
204 : case AArch64::LDRHHui:
205 : case AArch64::LDRHui:
206 : case AArch64::LDRQui:
207 : case AArch64::LDRSui:
208 : case AArch64::LDRWui:
209 : case AArch64::LDRXui:
210 : case AArch64::LDRSBWui:
211 : case AArch64::LDRSBXui:
212 : case AArch64::LDRSHWui:
213 : case AArch64::LDRSHXui:
214 : case AArch64::LDRSWui:
215 : // Assume the 1st instr to be a wildcard if it is unspecified.
216 531 : if (FirstMI == nullptr)
217 : return true;
218 :
219 344 : switch (FirstMI->getOpcode()) {
220 0 : case AArch64::ADR:
221 0 : return SecondMI.getOperand(2).getImm() == 0;
222 : case AArch64::ADRP:
223 : return true;
224 : }
225 : }
226 :
227 : return false;
228 : }
229 :
230 : /// Compare and conditional select.
231 2776 : static bool isCCSelectPair(const MachineInstr *FirstMI,
232 : const MachineInstr &SecondMI) {
233 : // 32 bits
234 5552 : if (SecondMI.getOpcode() == AArch64::CSELWr) {
235 : // Assume the 1st instr to be a wildcard if it is unspecified.
236 12 : if (FirstMI == nullptr)
237 : return true;
238 :
239 6 : if (FirstMI->definesRegister(AArch64::WZR))
240 12 : switch (FirstMI->getOpcode()) {
241 0 : case AArch64::SUBSWrs:
242 0 : return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
243 0 : case AArch64::SUBSWrx:
244 0 : return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
245 : case AArch64::SUBSWrr:
246 : case AArch64::SUBSWri:
247 : return true;
248 : }
249 : }
250 :
251 : // 64 bits
252 5528 : if (SecondMI.getOpcode() == AArch64::CSELXr) {
253 : // Assume the 1st instr to be a wildcard if it is unspecified.
254 12 : if (FirstMI == nullptr)
255 : return true;
256 :
257 6 : if (FirstMI->definesRegister(AArch64::XZR))
258 12 : switch (FirstMI->getOpcode()) {
259 0 : case AArch64::SUBSXrs:
260 0 : return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
261 0 : case AArch64::SUBSXrx:
262 : case AArch64::SUBSXrx64:
263 0 : return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
264 6 : case AArch64::SUBSXrr:
265 : case AArch64::SUBSXri:
266 6 : return true;
267 : }
268 : }
269 :
270 : return false;
271 : }
272 :
273 : /// Check if the instr pair, FirstMI and SecondMI, should be fused
274 : /// together. Given SecondMI, when FirstMI is unspecified, then check if
275 : /// SecondMI may be part of a fused pair at all.
276 132779 : static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
277 : const TargetSubtargetInfo &TSI,
278 : const MachineInstr *FirstMI,
279 : const MachineInstr &SecondMI) {
280 : const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
281 :
282 : // All checking functions assume that the 1st instr is a wildcard if it is
283 : // unspecified.
284 132779 : if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI))
285 : return true;
286 132698 : if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
287 : return true;
288 132660 : if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
289 : return true;
290 132006 : if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
291 : return true;
292 132006 : if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
293 : return true;
294 131940 : if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
295 : return true;
296 131663 : if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
297 24 : return true;
298 :
299 : return false;
300 : }
301 :
302 : } // end namespace
303 :
304 :
305 : namespace llvm {
306 :
307 23726 : std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
308 23726 : return createMacroFusionDAGMutation(shouldScheduleAdjacent);
309 : }
310 :
311 : } // end namespace llvm
|
